2019-05-27 08:55:01 +02:00
/* SPDX-License-Identifier: GPL-2.0-or-later */
2005-09-26 16:04:21 +10:00
/ *
* PowerPC v e r s i o n
* Copyright ( C ) 1 9 9 5 - 1 9 9 6 G a r y T h o m a s ( g d t @linuxppc.org)
* Rewritten b y C o r t D o u g a n ( c o r t @cs.nmt.edu) for PReP
* Copyright ( C ) 1 9 9 6 C o r t D o u g a n < c o r t @cs.nmt.edu>
* Adapted f o r P o w e r M a c i n t o s h b y P a u l M a c k e r r a s .
* Low- l e v e l e x c e p t i o n h a n d l e r s a n d M M U s u p p o r t
* rewritten b y P a u l M a c k e r r a s .
* Copyright ( C ) 1 9 9 6 P a u l M a c k e r r a s .
*
* This f i l e c o n t a i n s l o w - l e v e l a s s e m b l e r r o u t i n e s f o r m a n a g i n g
* the P o w e r P C M M U h a s h t a b l e . ( P P C 8 x x p r o c e s s o r s d o n ' t u s e a
* hash t a b l e , s o t h i s f i l e i s n o t u s e d o n t h e m . )
* /
2005-10-10 22:20:10 +10:00
# include < a s m / r e g . h >
2005-09-26 16:04:21 +10:00
# include < a s m / p a g e . h >
# include < a s m / p g t a b l e . h >
# include < a s m / c p u t a b l e . h >
# include < a s m / p p c _ a s m . h >
# include < a s m / t h r e a d _ i n f o . h >
# include < a s m / a s m - o f f s e t s . h >
2016-01-13 23:33:46 -05:00
# include < a s m / e x p o r t . h >
2018-07-05 16:25:01 +00:00
# include < a s m / f e a t u r e - f i x u p s . h >
2018-11-09 17:33:24 +00:00
# include < a s m / c o d e - p a t c h i n g - a s m . h >
2005-09-26 16:04:21 +10:00
# ifdef C O N F I G _ S M P
2007-05-14 17:11:58 -05:00
.section .bss
.align 2
mmu_hash_lock :
.space 4
2005-09-26 16:04:21 +10:00
# endif / * C O N F I G _ S M P * /
/ *
* Load a P T E i n t o t h e h a s h t a b l e , i f p o s s i b l e .
* The a d d r e s s i s i n r4 , a n d r3 c o n t a i n s a n a c c e s s f l a g :
2020-03-10 17:29:12 +00:00
* _ PAGE_ R W ( 0 x00 2 ) i f a w r i t e .
2005-09-26 16:04:21 +10:00
* r9 c o n t a i n s t h e S R R 1 v a l u e , f r o m w h i c h w e u s e t h e M S R _ P R b i t .
2009-07-14 20:52:54 +00:00
* SPRG_ T H R E A D c o n t a i n s t h e p h y s i c a l a d d r e s s o f t h e c u r r e n t t a s k ' s t h r e a d .
2005-09-26 16:04:21 +10:00
*
* Returns t o t h e c a l l e r i f t h e a c c e s s i s i l l e g a l o r t h e r e i s n o
* mapping f o r t h e a d d r e s s . O t h e r w i s e i t p l a c e s a n a p p r o p r i a t e P T E
* in t h e h a s h t a b l e a n d r e t u r n s f r o m t h e e x c e p t i o n .
2019-02-21 10:37:57 +00:00
* Uses r0 , r3 - r6 , r8 , r10 , c t r , l r .
2005-09-26 16:04:21 +10:00
* /
.text
_ GLOBAL( h a s h _ p a g e )
# ifdef C O N F I G _ S M P
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
lis r8 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @h
ori r8 , r8 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @l
2005-09-26 16:04:21 +10:00
lis r0 ,0 x0 f f f
b 1 0 f
11 : lwz r6 ,0 ( r8 )
cmpwi 0 ,r6 ,0
bne 1 1 b
10 : lwarx r6 ,0 ,r8
cmpwi 0 ,r6 ,0
bne- 1 1 b
stwcx. r0 ,0 ,r8
bne- 1 0 b
isync
# endif
/* Get PTE (linux-style) and check access */
lis r0 ,K E R N E L B A S E @h /* check if kernel address */
cmplw 0 ,r4 ,r0
ori r3 ,r3 ,_ P A G E _ U S E R | _ P A G E _ P R E S E N T / * t e s t l o w a d d r e s s e s a s u s e r * /
2019-03-11 08:30:27 +00:00
mfspr r5 , S P R N _ S P R G _ P G D I R / * p h y s p a g e - t a b l e r o o t * /
2005-09-26 16:04:21 +10:00
blt+ 1 1 2 f / * a s s u m e u s e r m o r e l i k e l y * /
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
lis r5 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @ha /* if kernel address, use */
addi r5 ,r5 ,( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @l /* kernel page table */
2020-03-10 17:29:12 +00:00
rlwimi r3 ,r9 ,3 2 - 1 4 ,3 1 ,3 1 / * M S R _ P R - > _ P A G E _ U S E R * /
2019-03-11 08:30:27 +00:00
112 :
2008-09-24 11:01:24 -05:00
# ifndef C O N F I G _ P T E _ 6 4 B I T
2005-09-26 16:04:21 +10:00
rlwimi r5 ,r4 ,1 2 ,2 0 ,2 9 / * i n s e r t t o p 1 0 b i t s o f a d d r e s s * /
lwz r8 ,0 ( r5 ) / * g e t p m d e n t r y * /
rlwinm. r8 ,r8 ,0 ,0 ,1 9 / * e x t r a c t a d d r e s s o f p t e p a g e * /
2008-09-24 11:01:24 -05:00
# else
rlwinm r8 ,r4 ,1 3 ,1 9 ,2 9 / * C o m p u t e p g d i r / p m d o f f s e t * /
lwzx r8 ,r8 ,r5 / * G e t L 1 e n t r y * /
rlwinm. r8 ,r8 ,0 ,0 ,2 0 / * e x t r a c t p t b a s e a d d r e s s * /
# endif
2005-09-26 16:04:21 +10:00
# ifdef C O N F I G _ S M P
beq- h a s h _ p a g e _ o u t / * r e t u r n i f n o m a p p i n g * /
# else
/ * XXX i t s e e m s l i k e t h e 6 0 1 w i l l g i v e a m a c h i n e f a u l t o n t h e
rfi i f i t s a l i g n m e n t i s w r o n g ( b o t t o m 4 b i t s o f a d d r e s s a r e
8 or 0 x c ) a n d w e h a v e h a d a n o t - t a k e n c o n d i t i o n a l b r a n c h
to t h e a d d r e s s f o l l o w i n g t h e r f i . * /
beqlr-
# endif
2008-09-24 11:01:24 -05:00
# ifndef C O N F I G _ P T E _ 6 4 B I T
2005-09-26 16:04:21 +10:00
rlwimi r8 ,r4 ,2 2 ,2 0 ,2 9 / * i n s e r t n e x t 1 0 b i t s o f a d d r e s s * /
2008-09-24 11:01:24 -05:00
# else
rlwimi r8 ,r4 ,2 3 ,2 0 ,2 8 / * c o m p u t e p t e a d d r e s s * /
# endif
2020-03-10 17:29:12 +00:00
rlwinm r0 ,r3 ,6 ,2 4 ,2 4 / * _ P A G E _ R W a c c e s s - > _ P A G E _ D I R T Y * /
2005-09-26 16:04:21 +10:00
ori r0 ,r0 ,_ P A G E _ A C C E S S E D | _ P A G E _ H A S H P T E
/ *
* Update t h e l i n u x P T E a t o m i c a l l y . W e d o t h e l w a r x u p - f r o n t
* because a l m o s t a l w a y s , t h e r e w o n ' t b e a p e r m i s s i o n v i o l a t i o n
* and t h e r e w o n ' t a l r e a d y b e a n H P T E , a n d t h u s w e w i l l h a v e
* to u p d a t e t h e P T E t o s e t _ P A G E _ H A S H P T E . - - p a u l u s .
2008-09-24 11:01:24 -05:00
*
* If P T E _ 6 4 B I T i s s e t , t h e l o w w o r d i s t h e f l a g s w o r d ; use that
* word f o r l o c k i n g s i n c e i t c o n t a i n s a l l t h e i n t e r e s t i n g b i t s .
2005-09-26 16:04:21 +10:00
* /
2008-09-24 11:01:24 -05:00
# if ( P T E _ F L A G S _ O F F S E T ! = 0 )
addi r8 ,r8 ,P T E _ F L A G S _ O F F S E T
# endif
2005-09-26 16:04:21 +10:00
retry :
2008-09-24 11:01:24 -05:00
lwarx r6 ,0 ,r8 / * g e t l i n u x - s t y l e p t e , f l a g w o r d * /
2005-09-26 16:04:21 +10:00
andc. r5 ,r3 ,r6 / * c h e c k a c c e s s & ~ p e r m i s s i o n * /
# ifdef C O N F I G _ S M P
bne- h a s h _ p a g e _ o u t / * r e t u r n i f a c c e s s n o t p e r m i t t e d * /
# else
bnelr-
# endif
or r5 ,r0 ,r6 / * s e t a c c e s s e d / d i r t y b i t s * /
2008-09-24 11:01:24 -05:00
# ifdef C O N F I G _ P T E _ 6 4 B I T
# ifdef C O N F I G _ S M P
subf r10 ,r6 ,r8 / * c r e a t e f a l s e d a t a d e p e n d e n c y * /
subi r10 ,r10 ,P T E _ F L A G S _ O F F S E T
lwzx r10 ,r6 ,r10 / * G e t u p p e r P T E w o r d * /
# else
lwz r10 ,- P T E _ F L A G S _ O F F S E T ( r8 )
# endif / * C O N F I G _ S M P * /
# endif / * C O N F I G _ P T E _ 6 4 B I T * /
2005-09-26 16:04:21 +10:00
stwcx. r5 ,0 ,r8 / * a t t e m p t t o u p d a t e P T E * /
bne- r e t r y / * r e t r y i f s o m e o n e g o t t h e r e f i r s t * /
mfsrin r3 ,r4 / * g e t s e g m e n t r e g f o r s e g m e n t * /
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
# ifndef C O N F I G _ V M A P _ S T A C K
2005-09-26 16:04:21 +10:00
mfctr r0
stw r0 ,_ C T R ( r11 )
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
# endif
2005-09-26 16:04:21 +10:00
bl c r e a t e _ h p t e / * a d d t h e h a s h t a b l e e n t r y * /
# ifdef C O N F I G _ S M P
eieio
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
lis r8 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @ha
2005-09-26 16:04:21 +10:00
li r0 ,0
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
stw r0 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @l(r8)
2005-09-26 16:04:21 +10:00
# endif
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
# ifdef C O N F I G _ V M A P _ S T A C K
b f a s t _ h a s h _ p a g e _ r e t u r n
# else
2005-09-26 16:04:21 +10:00
/* Return from the exception */
lwz r5 ,_ C T R ( r11 )
mtctr r5
lwz r0 ,G P R 0 ( r11 )
lwz r8 ,G P R 8 ( r11 )
b f a s t _ e x c e p t i o n _ r e t u r n
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
# endif
2005-09-26 16:04:21 +10:00
# ifdef C O N F I G _ S M P
hash_page_out :
eieio
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
lis r8 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @ha
2005-09-26 16:04:21 +10:00
li r0 ,0
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
stw r0 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @l(r8)
2005-09-26 16:04:21 +10:00
blr
# endif / * C O N F I G _ S M P * /
/ *
* Add a n e n t r y f o r a p a r t i c u l a r p a g e t o t h e h a s h t a b l e .
*
* add_ h a s h _ p a g e ( u n s i g n e d c o n t e x t , u n s i g n e d l o n g v a , u n s i g n e d l o n g p m d v a l )
*
* We a s s u m e a n y n e c e s s a r y m o d i f i c a t i o n s t o t h e p t e ( e . g . s e t t i n g
* the a c c e s s e d b i t ) h a v e a l r e a d y b e e n d o n e a n d t h a t t h e r e i s a c t u a l l y
* a h a s h t a b l e i n u s e ( i . e . w e ' r e n o t o n a 6 0 3 ) .
* /
_ GLOBAL( a d d _ h a s h _ p a g e )
mflr r0
stw r0 ,4 ( r1 )
/* Convert context and va to VSID */
mulli r3 ,r3 ,8 9 7 * 1 6 / * m u l t i p l y c o n t e x t b y c o n t e x t s k e w * /
rlwinm r0 ,r4 ,4 ,2 8 ,3 1 / * g e t E S I D ( t o p 4 b i t s o f v a ) * /
mulli r0 ,r0 ,0 x11 1 / * m u l t i p l y b y E S I D s k e w * /
add r3 ,r3 ,r0 / * n o t e c r e a t e _ h p t e t r i m s t o 2 4 b i t s * /
# ifdef C O N F I G _ S M P
2019-01-31 10:09:04 +00:00
lwz r8 ,T A S K _ C P U ( r2 ) / * t o g o i n m m u _ h a s h _ l o c k * /
2005-09-26 16:04:21 +10:00
oris r8 ,r8 ,1 2
# endif / * C O N F I G _ S M P * /
/ *
* We d i s a b l e i n t e r r u p t s h e r e , e v e n o n U P , b e c a u s e w e d o n ' t
* want t o r a c e w i t h h a s h _ p a g e , a n d b e c a u s e w e w a n t t h e
* _ PAGE_ H A S H P T E b i t t o b e a r e l i a b l e i n d i c a t i o n o f w h e t h e r
* the H P T E e x i s t s ( o r a t l e a s t w h e t h e r o n e d i d o n c e ) .
* We a l s o t u r n o f f t h e M M U f o r d a t a a c c e s s e s s o t h a t w e
* we c a n ' t t a k e a h a s h t a b l e m i s s ( a s s u m i n g t h e c o d e i s
* covered b y a B A T ) . - - p a u l u s
* /
2008-09-24 11:01:24 -05:00
mfmsr r9
2005-09-26 16:04:21 +10:00
SYNC
2008-09-24 11:01:24 -05:00
rlwinm r0 ,r9 ,0 ,1 7 ,1 5 / * c l e a r b i t 1 6 ( M S R _ E E ) * /
2005-09-26 16:04:21 +10:00
rlwinm r0 ,r0 ,0 ,2 8 ,2 6 / * c l e a r M S R _ D R * /
mtmsr r0
SYNC_ 6 0 1
isync
# ifdef C O N F I G _ S M P
2019-02-21 10:37:57 +00:00
lis r6 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @ha
addi r6 , r6 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @l
2008-09-24 11:01:24 -05:00
10 : lwarx r0 ,0 ,r6 / * t a k e t h e m m u _ h a s h _ l o c k * /
2005-09-26 16:04:21 +10:00
cmpi 0 ,r0 ,0
bne- 1 1 f
2008-09-24 11:01:24 -05:00
stwcx. r8 ,0 ,r6
2005-09-26 16:04:21 +10:00
beq+ 1 2 f
2008-09-24 11:01:24 -05:00
11 : lwz r0 ,0 ( r6 )
2005-09-26 16:04:21 +10:00
cmpi 0 ,r0 ,0
beq 1 0 b
b 1 1 b
12 : isync
# endif
/ *
* Fetch t h e l i n u x p t e a n d t e s t a n d s e t _ P A G E _ H A S H P T E a t o m i c a l l y .
* If _ P A G E _ H A S H P T E w a s a l r e a d y s e t , w e d o n ' t r e p l a c e t h e e x i s t i n g
* HPTE, s o w e j u s t u n l o c k a n d r e t u r n .
* /
mr r8 ,r5
2008-09-24 11:01:24 -05:00
# ifndef C O N F I G _ P T E _ 6 4 B I T
2005-09-26 16:04:21 +10:00
rlwimi r8 ,r4 ,2 2 ,2 0 ,2 9
2008-09-24 11:01:24 -05:00
# else
rlwimi r8 ,r4 ,2 3 ,2 0 ,2 8
addi r8 ,r8 ,P T E _ F L A G S _ O F F S E T
# endif
2005-09-26 16:04:21 +10:00
1 : lwarx r6 ,0 ,r8
andi. r0 ,r6 ,_ P A G E _ H A S H P T E
bne 9 f / * i f H A S H P T E a l r e a d y s e t , d o n e * /
2008-09-24 11:01:24 -05:00
# ifdef C O N F I G _ P T E _ 6 4 B I T
# ifdef C O N F I G _ S M P
subf r10 ,r6 ,r8 / * c r e a t e f a l s e d a t a d e p e n d e n c y * /
subi r10 ,r10 ,P T E _ F L A G S _ O F F S E T
lwzx r10 ,r6 ,r10 / * G e t u p p e r P T E w o r d * /
# else
lwz r10 ,- P T E _ F L A G S _ O F F S E T ( r8 )
# endif / * C O N F I G _ S M P * /
# endif / * C O N F I G _ P T E _ 6 4 B I T * /
2005-09-26 16:04:21 +10:00
ori r5 ,r6 ,_ P A G E _ H A S H P T E
stwcx. r5 ,0 ,r8
bne- 1 b
bl c r e a t e _ h p t e
9 :
# ifdef C O N F I G _ S M P
2019-02-21 10:37:57 +00:00
lis r6 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @ha
addi r6 , r6 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @l
2005-09-26 16:04:21 +10:00
eieio
li r0 ,0
2008-09-24 11:01:24 -05:00
stw r0 ,0 ( r6 ) / * c l e a r m m u _ h a s h _ l o c k * /
2005-09-26 16:04:21 +10:00
# endif
/* reenable interrupts and DR */
2008-09-24 11:01:24 -05:00
mtmsr r9
2005-09-26 16:04:21 +10:00
SYNC_ 6 0 1
isync
lwz r0 ,4 ( r1 )
mtlr r0
blr
/ *
* This r o u t i n e a d d s a h a r d w a r e P T E t o t h e h a s h t a b l e .
* It i s d e s i g n e d t o b e c a l l e d w i t h t h e M M U e i t h e r o n o r o f f .
* r3 c o n t a i n s t h e V S I D , r4 c o n t a i n s t h e v i r t u a l a d d r e s s ,
* r5 c o n t a i n s t h e l i n u x P T E , r6 c o n t a i n s t h e o l d v a l u e o f t h e
2019-02-21 10:37:57 +00:00
* linux P T E ( b e f o r e s e t t i n g _ P A G E _ H A S H P T E ) . r10 c o n t a i n s t h e
* upper h a l f o f t h e P T E i f C O N F I G _ P T E _ 6 4 B I T .
2005-09-26 16:04:21 +10:00
* On S M P , t h e c a l l e r s h o u l d h a v e t h e m m u _ h a s h _ l o c k h e l d .
* We a s s u m e t h a t t h e c a l l e r h a s ( o r w i l l ) s e t t h e _ P A G E _ H A S H P T E
* bit i n t h e l i n u x P T E i n m e m o r y . T h e v a l u e p a s s e d i n r6 s h o u l d
* be t h e o l d l i n u x P T E v a l u e ; if it doesn't have _PAGE_HASHPTE set
* this r o u t i n e w i l l s k i p t h e s e a r c h f o r a n e x i s t i n g H P T E .
* This p r o c e d u r e m o d i f i e s r0 , r3 - r6 , r8 , c r0 .
* - - paulus.
*
* For s p e e d , 4 o f t h e i n s t r u c t i o n s g e t p a t c h e d o n c e t h e s i z e a n d
* physical a d d r e s s o f t h e h a s h t a b l e a r e k n o w n . T h e s e d e f i n i t i o n s
* of H a s h _ b a s e a n d H a s h _ b i t s b e l o w a r e j u s t a n e x a m p l e .
* /
Hash_ b a s e = 0 x c01 8 0 0 0 0
Hash_ b i t s = 1 2 / * e . g . 2 5 6 k B h a s h t a b l e * /
Hash_ m s k = ( ( ( 1 < < H a s h _ b i t s ) - 1 ) * 6 4 )
/* defines for the PTE format for 32-bit PPCs */
2008-09-04 01:37:53 +10:00
# define H P T E _ S I Z E 8
2005-09-26 16:04:21 +10:00
# define P T E G _ S I Z E 6 4
# define L G _ P T E G _ S I Z E 6
# define L D P T E u l w z u
2007-04-12 15:30:22 +10:00
# define L D P T E l w z
2005-09-26 16:04:21 +10:00
# define S T P T E s t w
# define C M P P T E c m p w
# define P T E _ H 0 x40
# define P T E _ V 0 x80 0 0 0 0 0 0
# define T S T _ V ( r ) r l w i n m . r ,r ,0 ,0 ,0
# define S E T _ V ( r ) o r i s r ,r ,P T E _ V @h
# define C L R _ V ( r ,t ) r l w i n m r ,r ,0 ,1 ,3 1
# define H A S H _ L E F T 3 1 - ( L G _ P T E G _ S I Z E + H a s h _ b i t s - 1 )
# define H A S H _ R I G H T 3 1 - L G _ P T E G _ S I Z E
_ GLOBAL( c r e a t e _ h p t e )
/* Convert linux-style PTE (r5) to low word of PPC-style PTE (r8) */
2019-03-11 08:30:36 +00:00
rlwinm r0 ,r5 ,3 2 - 6 ,3 0 ,3 0 / * _ P A G E _ D I R T Y - > P P m s b * /
2020-03-10 17:29:12 +00:00
and r8 ,r5 ,r0 / * w r i t a b l e i f _ R W & _ D I R T Y * /
rlwimi r5 ,r5 ,1 ,3 0 ,3 0 / * _ P A G E _ U S E R - > P P m s b * /
2009-02-10 10:57:46 +00:00
ori r8 ,r8 ,0 x e 0 4 / * c l e a r o u t r e s e r v e d b i t s * /
2019-03-11 08:30:36 +00:00
andc r8 ,r5 ,r8 / * P P = u s e r ? ( r w & d i r t y ? 1 : 3 ) : 0 * /
2005-09-26 16:04:21 +10:00
BEGIN_ F T R _ S E C T I O N
2008-12-18 19:13:51 +00:00
rlwinm r8 ,r8 ,0 ,~ _ P A G E _ C O H E R E N T / * c l e a r M ( c o h e r e n c e n o t r e q u i r e d ) * /
END_ F T R _ S E C T I O N _ I F C L R ( C P U _ F T R _ N E E D _ C O H E R E N T )
2008-09-24 11:01:24 -05:00
# ifdef C O N F I G _ P T E _ 6 4 B I T
/* Put the XPN bits into the PTE */
rlwimi r8 ,r10 ,8 ,2 0 ,2 2
rlwimi r8 ,r10 ,2 ,2 9 ,2 9
# endif
2005-09-26 16:04:21 +10:00
/* Construct the high word of the PPC-style PTE (r5) */
rlwinm r5 ,r3 ,7 ,1 ,2 4 / * p u t V S I D i n 0 x7 f f f f f80 b i t s * /
rlwimi r5 ,r4 ,1 0 ,2 6 ,3 1 / * p u t i n A P I ( a b b r e v p a g e i n d e x ) * /
SET_ V ( r5 ) / * s e t V ( v a l i d ) b i t * /
2018-11-09 17:33:24 +00:00
patch_ s i t e 0 f , p a t c h _ _ h a s h _ p a g e _ A 0
patch_ s i t e 1 f , p a t c h _ _ h a s h _ p a g e _ A 1
patch_ s i t e 2 f , p a t c h _ _ h a s h _ p a g e _ A 2
2005-09-26 16:04:21 +10:00
/* Get the address of the primary PTE group in the hash table (r3) */
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
0 : lis r0 , ( H a s h _ b a s e - P A G E _ O F F S E T ) @h /* base address of hash table */
2018-11-09 17:33:24 +00:00
1 : rlwimi r0 ,r3 ,L G _ P T E G _ S I Z E ,H A S H _ L E F T ,H A S H _ R I G H T / * V S I D - > h a s h * /
2 : rlwinm r3 ,r4 ,2 0 + L G _ P T E G _ S I Z E ,H A S H _ L E F T ,H A S H _ R I G H T / * P I - > h a s h * /
2005-09-26 16:04:21 +10:00
xor r3 ,r3 ,r0 / * m a k e p r i m a r y h a s h * /
li r0 ,8 / * P T E s / g r o u p * /
/ *
* Test t h e _ P A G E _ H A S H P T E b i t i n t h e o l d l i n u x P T E , a n d s k i p t h e s e a r c h
* if i t i s c l e a r , m e a n i n g t h a t t h e H P T E i s n ' t t h e r e a l r e a d y . . .
* /
andi. r6 ,r6 ,_ P A G E _ H A S H P T E
beq+ 1 0 f / * n o P T E : g o l o o k f o r a n e m p t y s l o t * /
tlbie r4
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
lis r4 , ( h t a b _ h a s h _ s e a r c h e s - P A G E _ O F F S E T ) @ha
lwz r6 , ( h t a b _ h a s h _ s e a r c h e s - P A G E _ O F F S E T ) @l(r4)
2005-09-26 16:04:21 +10:00
addi r6 ,r6 ,1 / * c o u n t h o w m a n y s e a r c h e s w e d o * /
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
stw r6 , ( h t a b _ h a s h _ s e a r c h e s - P A G E _ O F F S E T ) @l(r4)
2005-09-26 16:04:21 +10:00
/* Search the primary PTEG for a PTE whose 1st (d)word matches r5 */
mtctr r0
2008-09-04 01:37:53 +10:00
addi r4 ,r3 ,- H P T E _ S I Z E
1 : LDPTEu r6 ,H P T E _ S I Z E ( r4 ) / * g e t n e x t P T E * /
2005-09-26 16:04:21 +10:00
CMPPTE 0 ,r6 ,r5
bdnzf 2 ,1 b / * l o o p w h i l e c t r ! = 0 & & ! c r0 . e q * /
beq+ f o u n d _ s l o t
2018-11-09 17:33:24 +00:00
patch_ s i t e 0 f , p a t c h _ _ h a s h _ p a g e _ B
2005-09-26 16:04:21 +10:00
/* Search the secondary PTEG for a matching PTE */
ori r5 ,r5 ,P T E _ H / * s e t H ( s e c o n d a r y h a s h ) b i t * /
2018-11-09 17:33:24 +00:00
0 : xoris r4 ,r3 ,H a s h _ m s k > > 1 6 / * c o m p u t e s e c o n d a r y h a s h * /
2005-09-26 16:04:21 +10:00
xori r4 ,r4 ,( - P T E G _ S I Z E & 0 x f f f f )
2008-09-04 01:37:53 +10:00
addi r4 ,r4 ,- H P T E _ S I Z E
2005-09-26 16:04:21 +10:00
mtctr r0
2008-09-04 01:37:53 +10:00
2 : LDPTEu r6 ,H P T E _ S I Z E ( r4 )
2005-09-26 16:04:21 +10:00
CMPPTE 0 ,r6 ,r5
bdnzf 2 ,2 b
beq+ f o u n d _ s l o t
xori r5 ,r5 ,P T E _ H / * c l e a r H b i t a g a i n * /
/* Search the primary PTEG for an empty slot */
10 : mtctr r0
2008-09-04 01:37:53 +10:00
addi r4 ,r3 ,- H P T E _ S I Z E / * s e a r c h p r i m a r y P T E G * /
1 : LDPTEu r6 ,H P T E _ S I Z E ( r4 ) / * g e t n e x t P T E * /
2005-09-26 16:04:21 +10:00
TST_ V ( r6 ) / * t e s t v a l i d b i t * /
bdnzf 2 ,1 b / * l o o p w h i l e c t r ! = 0 & & ! c r0 . e q * /
beq+ f o u n d _ e m p t y
/* update counter of times that the primary PTEG is full */
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
lis r4 , ( p r i m a r y _ p t e g _ f u l l - P A G E _ O F F S E T ) @ha
lwz r6 , ( p r i m a r y _ p t e g _ f u l l - P A G E _ O F F S E T ) @l(r4)
2005-09-26 16:04:21 +10:00
addi r6 ,r6 ,1
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
stw r6 , ( p r i m a r y _ p t e g _ f u l l - P A G E _ O F F S E T ) @l(r4)
2005-09-26 16:04:21 +10:00
2018-11-09 17:33:24 +00:00
patch_ s i t e 0 f , p a t c h _ _ h a s h _ p a g e _ C
2005-09-26 16:04:21 +10:00
/* Search the secondary PTEG for an empty slot */
ori r5 ,r5 ,P T E _ H / * s e t H ( s e c o n d a r y h a s h ) b i t * /
2018-11-09 17:33:24 +00:00
0 : xoris r4 ,r3 ,H a s h _ m s k > > 1 6 / * c o m p u t e s e c o n d a r y h a s h * /
2005-09-26 16:04:21 +10:00
xori r4 ,r4 ,( - P T E G _ S I Z E & 0 x f f f f )
2008-09-04 01:37:53 +10:00
addi r4 ,r4 ,- H P T E _ S I Z E
2005-09-26 16:04:21 +10:00
mtctr r0
2008-09-04 01:37:53 +10:00
2 : LDPTEu r6 ,H P T E _ S I Z E ( r4 )
2005-09-26 16:04:21 +10:00
TST_ V ( r6 )
bdnzf 2 ,2 b
beq+ f o u n d _ e m p t y
xori r5 ,r5 ,P T E _ H / * c l e a r H b i t a g a i n * /
/ *
* Choose a n a r b i t r a r y s l o t i n t h e p r i m a r y P T E G t o o v e r w r i t e .
* Since b o t h t h e p r i m a r y a n d s e c o n d a r y P T E G s a r e f u l l , a n d w e
* have n o i n f o r m a t i o n t h a t t h e P T E s i n t h e p r i m a r y P T E G a r e
* more i m p o r t a n t o r u s e f u l t h a n t h o s e i n t h e s e c o n d a r y P T E G ,
* and w e k n o w t h e r e i s a d e f i n i t e ( a l t h o u g h s m a l l ) s p e e d
* advantage t o p u t t i n g t h e P T E i n t h e p r i m a r y P T E G , w e a l w a y s
* put t h e P T E i n t h e p r i m a r y P T E G .
2007-04-12 15:30:22 +10:00
*
* In a d d i t i o n , w e s k i p a n y s l o t t h a t i s m a p p i n g k e r n e l t e x t i n
* order t o a v o i d a d e a d l o c k w h e n n o t u s i n g B A T m a p p i n g s i f
* trying t o h a s h i n t h e k e r n e l h a s h c o d e i t s e l f a f t e r i t h a s
* already t a k e n t h e h a s h t a b l e l o c k . T h i s w o r k s i n c o n j u n c t i o n
* with p r e - f a u l t i n g o f t h e k e r n e l t e x t .
*
* If t h e h a s h t a b l e b u c k e t i s f u l l o f k e r n e l t e x t e n t r i e s , w e ' l l
* lockup h e r e b u t t h a t s h o u l d n ' t h a p p e n
2005-09-26 16:04:21 +10:00
* /
2007-04-12 15:30:22 +10:00
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 10:14:25 +00:00
1 : lis r4 , ( n e x t _ s l o t - P A G E _ O F F S E T ) @ha /* get next evict slot */
lwz r6 , ( n e x t _ s l o t - P A G E _ O F F S E T ) @l(r4)
2008-09-04 01:37:53 +10:00
addi r6 ,r6 ,H P T E _ S I Z E / * s e a r c h f o r c a n d i d a t e * /
andi. r6 ,r6 ,7 * H P T E _ S I Z E
2005-09-26 16:04:21 +10:00
stw r6 ,n e x t _ s l o t @l(r4)
add r4 ,r3 ,r6
2008-09-04 01:37:53 +10:00
LDPTE r0 ,H P T E _ S I Z E / 2 ( r4 ) / * g e t P T E s e c o n d w o r d * /
2007-04-12 15:30:22 +10:00
clrrwi r0 ,r0 ,1 2
lis r6 ,e t e x t @h
ori r6 ,r6 ,e t e x t @l /* get etext */
tophys( r6 ,r6 )
cmpl c r0 ,r0 ,r6 / * c o m p a r e a n d t r y a g a i n * /
blt 1 b
2005-09-26 16:04:21 +10:00
# ifndef C O N F I G _ S M P
/* Store PTE in PTEG */
found_empty :
STPTE r5 ,0 ( r4 )
found_slot :
2008-09-04 01:37:53 +10:00
STPTE r8 ,H P T E _ S I Z E / 2 ( r4 )
2005-09-26 16:04:21 +10:00
# else / * C O N F I G _ S M P * /
/ *
* Between t h e t l b i e a b o v e a n d u p d a t i n g t h e h a s h t a b l e e n t r y b e l o w ,
* another C P U c o u l d r e a d t h e h a s h t a b l e e n t r y a n d p u t i t i n i t s T L B .
* There a r e 3 c a s e s :
* 1 . using a n e m p t y s l o t
* 2 . updating a n e a r l i e r e n t r y t o c h a n g e p e r m i s s i o n s ( i . e . e n a b l e w r i t e )
* 3 . taking o v e r t h e P T E f o r a n u n r e l a t e d a d d r e s s
*
* In e a c h c a s e i t d o e s n ' t r e a l l y m a t t e r i f t h e o t h e r C P U s h a v e t h e o l d
* PTE i n t h e i r T L B . S o w e d o n ' t n e e d t o b o t h e r w i t h a n o t h e r t l b i e h e r e ,
* which i s c o n v e n i e n t a s w e ' v e o v e r w r i t t e n t h e r e g i s t e r t h a t h a d t h e
* address. : - ) T h e t l b i e a b o v e i s m a i n l y t o m a k e s u r e t h a t t h i s C P U c o m e s
* and g e t s t h e n e w P T E f r o m t h e h a s h t a b l e .
*
* We d o h o w e v e r h a v e t o m a k e s u r e t h a t t h e P T E i s n e v e r i n a n i n v a l i d
* state w i t h t h e V b i t s e t .
* /
found_empty :
found_slot :
CLR_ V ( r5 ,r0 ) / * c l e a r V ( v a l i d ) b i t i n P T E * /
STPTE r5 ,0 ( r4 )
sync
TLBSYNC
2008-09-04 01:37:53 +10:00
STPTE r8 ,H P T E _ S I Z E / 2 ( r4 ) / * p u t i n c o r r e c t R P N , W I M G , P P b i t s * /
2005-09-26 16:04:21 +10:00
sync
SET_ V ( r5 )
STPTE r5 ,0 ( r4 ) / * f i n a l l y s e t V b i t i n P T E * /
# endif / * C O N F I G _ S M P * /
sync / * m a k e s u r e p t e u p d a t e s g e t t o m e m o r y * /
blr
2007-05-14 17:11:58 -05:00
.section .bss
.align 2
next_slot :
.space 4
primary_pteg_full :
.space 4
htab_hash_searches :
.space 4
.previous
2005-09-26 16:04:21 +10:00
/ *
* Flush t h e e n t r y f o r a p a r t i c u l a r p a g e f r o m t h e h a s h t a b l e .
*
* flush_ h a s h _ p a g e s ( u n s i g n e d c o n t e x t , u n s i g n e d l o n g v a , u n s i g n e d l o n g p m d v a l ,
* int c o u n t )
*
* We a s s u m e t h a t t h e r e i s a h a s h t a b l e i n u s e ( H a s h ! = 0 ) .
* /
_ GLOBAL( f l u s h _ h a s h _ p a g e s )
/ *
* We d i s a b l e i n t e r r u p t s h e r e , e v e n o n U P , b e c a u s e w e w a n t
* the _ P A G E _ H A S H P T E b i t t o b e a r e l i a b l e i n d i c a t i o n o f
* whether t h e H P T E e x i s t s ( o r a t l e a s t w h e t h e r o n e d i d o n c e ) .
* We a l s o t u r n o f f t h e M M U f o r d a t a a c c e s s e s s o t h a t w e
* we c a n ' t t a k e a h a s h t a b l e m i s s ( a s s u m i n g t h e c o d e i s
* covered b y a B A T ) . - - p a u l u s
* /
mfmsr r10
SYNC
rlwinm r0 ,r10 ,0 ,1 7 ,1 5 / * c l e a r b i t 1 6 ( M S R _ E E ) * /
rlwinm r0 ,r0 ,0 ,2 8 ,2 6 / * c l e a r M S R _ D R * /
mtmsr r0
SYNC_ 6 0 1
isync
/* First find a PTE in the range that has _PAGE_HASHPTE set */
2008-09-24 11:01:24 -05:00
# ifndef C O N F I G _ P T E _ 6 4 B I T
2005-09-26 16:04:21 +10:00
rlwimi r5 ,r4 ,2 2 ,2 0 ,2 9
2008-09-24 11:01:24 -05:00
# else
rlwimi r5 ,r4 ,2 3 ,2 0 ,2 8
# endif
1 : lwz r0 ,P T E _ F L A G S _ O F F S E T ( r5 )
2005-09-26 16:04:21 +10:00
cmpwi c r1 ,r6 ,1
andi. r0 ,r0 ,_ P A G E _ H A S H P T E
bne 2 f
ble c r1 ,1 9 f
addi r4 ,r4 ,0 x10 0 0
2008-09-24 11:01:24 -05:00
addi r5 ,r5 ,P T E _ S I Z E
2005-09-26 16:04:21 +10:00
addi r6 ,r6 ,- 1
b 1 b
/* Convert context and va to VSID */
2 : mulli r3 ,r3 ,8 9 7 * 1 6 / * m u l t i p l y c o n t e x t b y c o n t e x t s k e w * /
rlwinm r0 ,r4 ,4 ,2 8 ,3 1 / * g e t E S I D ( t o p 4 b i t s o f v a ) * /
mulli r0 ,r0 ,0 x11 1 / * m u l t i p l y b y E S I D s k e w * /
add r3 ,r3 ,r0 / * n o t e c o d e b e l o w t r i m s t o 2 4 b i t s * /
/* Construct the high word of the PPC-style PTE (r11) */
rlwinm r11 ,r3 ,7 ,1 ,2 4 / * p u t V S I D i n 0 x7 f f f f f80 b i t s * /
rlwimi r11 ,r4 ,1 0 ,2 6 ,3 1 / * p u t i n A P I ( a b b r e v p a g e i n d e x ) * /
SET_ V ( r11 ) / * s e t V ( v a l i d ) b i t * /
# ifdef C O N F I G _ S M P
2019-02-21 10:37:57 +00:00
lis r9 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @ha
addi r9 , r9 , ( m m u _ h a s h _ l o c k - P A G E _ O F F S E T ) @l
2019-05-09 12:59:38 +00:00
tophys ( r8 , r2 )
lwz r8 , T A S K _ C P U ( r8 )
2005-09-26 16:04:21 +10:00
oris r8 ,r8 ,9
10 : lwarx r0 ,0 ,r9
cmpi 0 ,r0 ,0
bne- 1 1 f
stwcx. r8 ,0 ,r9
beq+ 1 2 f
11 : lwz r0 ,0 ( r9 )
cmpi 0 ,r0 ,0
beq 1 0 b
b 1 1 b
12 : isync
# endif
/ *
* Check t h e _ P A G E _ H A S H P T E b i t i n t h e l i n u x P T E . I f i t i s
* already c l e a r , w e ' r e d o n e ( f o r t h i s p t e ) . I f n o t ,
* clear i t ( a t o m i c a l l y ) a n d p r o c e e d . - - p a u l u s .
* /
2008-09-24 11:01:24 -05:00
# if ( P T E _ F L A G S _ O F F S E T ! = 0 )
addi r5 ,r5 ,P T E _ F L A G S _ O F F S E T
# endif
33 : lwarx r8 ,0 ,r5 / * f e t c h t h e p t e f l a g s w o r d * /
2005-09-26 16:04:21 +10:00
andi. r0 ,r8 ,_ P A G E _ H A S H P T E
beq 8 f / * d o n e i f H A S H P T E i s a l r e a d y c l e a r * /
2020-03-10 17:29:12 +00:00
rlwinm r8 ,r8 ,0 ,~ _ P A G E _ H A S H P T E / * c l e a r H A S H P T E b i t * /
2005-09-26 16:04:21 +10:00
stwcx. r8 ,0 ,r5 / * u p d a t e t h e p t e * /
bne- 3 3 b
2018-11-09 17:33:24 +00:00
patch_ s i t e 0 f , p a t c h _ _ f l u s h _ h a s h _ A 0
patch_ s i t e 1 f , p a t c h _ _ f l u s h _ h a s h _ A 1
patch_ s i t e 2 f , p a t c h _ _ f l u s h _ h a s h _ A 2
2005-09-26 16:04:21 +10:00
/* Get the address of the primary PTE group in the hash table (r3) */
2019-02-21 10:37:57 +00:00
0 : lis r8 , ( H a s h _ b a s e - P A G E _ O F F S E T ) @h /* base address of hash table */
2018-11-09 17:33:24 +00:00
1 : rlwimi r8 ,r3 ,L G _ P T E G _ S I Z E ,H A S H _ L E F T ,H A S H _ R I G H T / * V S I D - > h a s h * /
2 : rlwinm r0 ,r4 ,2 0 + L G _ P T E G _ S I Z E ,H A S H _ L E F T ,H A S H _ R I G H T / * P I - > h a s h * /
2005-09-26 16:04:21 +10:00
xor r8 ,r0 ,r8 / * m a k e p r i m a r y h a s h * /
/* Search the primary PTEG for a PTE whose 1st (d)word matches r5 */
li r0 ,8 / * P T E s / g r o u p * /
mtctr r0
2008-09-04 01:37:53 +10:00
addi r12 ,r8 ,- H P T E _ S I Z E
1 : LDPTEu r0 ,H P T E _ S I Z E ( r12 ) / * g e t n e x t P T E * /
2005-09-26 16:04:21 +10:00
CMPPTE 0 ,r0 ,r11
bdnzf 2 ,1 b / * l o o p w h i l e c t r ! = 0 & & ! c r0 . e q * /
beq+ 3 f
2018-11-09 17:33:24 +00:00
patch_ s i t e 0 f , p a t c h _ _ f l u s h _ h a s h _ B
2005-09-26 16:04:21 +10:00
/* Search the secondary PTEG for a matching PTE */
ori r11 ,r11 ,P T E _ H / * s e t H ( s e c o n d a r y h a s h ) b i t * /
li r0 ,8 / * P T E s / g r o u p * /
2018-11-09 17:33:24 +00:00
0 : xoris r12 ,r8 ,H a s h _ m s k > > 1 6 / * c o m p u t e s e c o n d a r y h a s h * /
2005-09-26 16:04:21 +10:00
xori r12 ,r12 ,( - P T E G _ S I Z E & 0 x f f f f )
2008-09-04 01:37:53 +10:00
addi r12 ,r12 ,- H P T E _ S I Z E
2005-09-26 16:04:21 +10:00
mtctr r0
2008-09-04 01:37:53 +10:00
2 : LDPTEu r0 ,H P T E _ S I Z E ( r12 )
2005-09-26 16:04:21 +10:00
CMPPTE 0 ,r0 ,r11
bdnzf 2 ,2 b
xori r11 ,r11 ,P T E _ H / * c l e a r H a g a i n * /
bne- 4 f / * s h o u l d r a r e l y f a i l t o f i n d i t * /
3 : li r0 ,0
STPTE r0 ,0 ( r12 ) / * i n v a l i d a t e e n t r y * /
4 : sync
tlbie r4 / * i n h w t l b t o o * /
sync
8 : ble c r1 ,9 f / * i f a l l p t e s c h e c k e d * /
81 : addi r6 ,r6 ,- 1
2008-09-24 11:01:24 -05:00
addi r5 ,r5 ,P T E _ S I Z E
2005-09-26 16:04:21 +10:00
addi r4 ,r4 ,0 x10 0 0
lwz r0 ,0 ( r5 ) / * c h e c k n e x t p t e * /
cmpwi c r1 ,r6 ,1
andi. r0 ,r0 ,_ P A G E _ H A S H P T E
bne 3 3 b
bgt c r1 ,8 1 b
9 :
# ifdef C O N F I G _ S M P
TLBSYNC
li r0 ,0
stw r0 ,0 ( r9 ) / * c l e a r m m u _ h a s h _ l o c k * /
# endif
19 : mtmsr r10
SYNC_ 6 0 1
isync
blr
2017-08-23 16:54:34 +02:00
EXPORT_ S Y M B O L ( f l u s h _ h a s h _ p a g e s )
2008-12-18 19:13:42 +00:00
/ *
* Flush a n e n t r y f r o m t h e T L B
* /
_ GLOBAL( _ t l b i e )
# ifdef C O N F I G _ S M P
2019-01-31 10:09:04 +00:00
lwz r8 ,T A S K _ C P U ( r2 )
2008-12-18 19:13:42 +00:00
oris r8 ,r8 ,1 1
mfmsr r10
SYNC
rlwinm r0 ,r10 ,0 ,1 7 ,1 5 / * c l e a r b i t 1 6 ( M S R _ E E ) * /
rlwinm r0 ,r0 ,0 ,2 8 ,2 6 / * c l e a r D R * /
mtmsr r0
SYNC_ 6 0 1
isync
lis r9 ,m m u _ h a s h _ l o c k @h
ori r9 ,r9 ,m m u _ h a s h _ l o c k @l
tophys( r9 ,r9 )
10 : lwarx r7 ,0 ,r9
cmpwi 0 ,r7 ,0
bne- 1 0 b
stwcx. r8 ,0 ,r9
bne- 1 0 b
eieio
tlbie r3
sync
TLBSYNC
li r0 ,0
stw r0 ,0 ( r9 ) / * c l e a r m m u _ h a s h _ l o c k * /
mtmsr r10
SYNC_ 6 0 1
isync
# else / * C O N F I G _ S M P * /
tlbie r3
sync
# endif / * C O N F I G _ S M P * /
blr
/ *
* Flush t h e e n t i r e T L B . 6 0 3 / 6 0 3 e o n l y
* /
_ GLOBAL( _ t l b i a )
# if d e f i n e d ( C O N F I G _ S M P )
2019-01-31 10:09:04 +00:00
lwz r8 ,T A S K _ C P U ( r2 )
2008-12-18 19:13:42 +00:00
oris r8 ,r8 ,1 0
mfmsr r10
SYNC
rlwinm r0 ,r10 ,0 ,1 7 ,1 5 / * c l e a r b i t 1 6 ( M S R _ E E ) * /
rlwinm r0 ,r0 ,0 ,2 8 ,2 6 / * c l e a r D R * /
mtmsr r0
SYNC_ 6 0 1
isync
lis r9 ,m m u _ h a s h _ l o c k @h
ori r9 ,r9 ,m m u _ h a s h _ l o c k @l
tophys( r9 ,r9 )
10 : lwarx r7 ,0 ,r9
cmpwi 0 ,r7 ,0
bne- 1 0 b
stwcx. r8 ,0 ,r9
bne- 1 0 b
2020-02-03 16:47:37 +00:00
# endif / * C O N F I G _ S M P * /
li r5 , 3 2
lis r4 , K E R N E L B A S E @h
mtctr r5
2008-12-18 19:13:42 +00:00
sync
2020-02-03 16:47:37 +00:00
0 : tlbie r4
addi r4 , r4 , 0 x10 0 0
bdnz 0 b
2008-12-18 19:13:42 +00:00
sync
2020-02-03 16:47:37 +00:00
# ifdef C O N F I G _ S M P
2008-12-18 19:13:42 +00:00
TLBSYNC
li r0 ,0
stw r0 ,0 ( r9 ) / * c l e a r m m u _ h a s h _ l o c k * /
mtmsr r10
SYNC_ 6 0 1
isync
# endif / * C O N F I G _ S M P * /
2008-12-21 02:54:25 -07:00
blr
