2019-05-27 09:55:01 +03:00
/* SPDX-License-Identifier: GPL-2.0-or-later */
2005-09-26 10:04:21 +04:00
/ *
* PowerPC v e r s i o n
* Copyright ( C ) 1 9 9 5 - 1 9 9 6 G a r y T h o m a s ( g d t @linuxppc.org)
*
* Rewritten b y C o r t D o u g a n ( c o r t @cs.nmt.edu) for PReP
* Copyright ( C ) 1 9 9 6 C o r t D o u g a n < c o r t @cs.nmt.edu>
* Adapted f o r P o w e r M a c i n t o s h b y P a u l M a c k e r r a s .
* Low- l e v e l e x c e p t i o n h a n d l e r s a n d M M U s u p p o r t
* rewritten b y P a u l M a c k e r r a s .
* Copyright ( C ) 1 9 9 6 P a u l M a c k e r r a s .
* MPC8 x x m o d i f i c a t i o n s C o p y r i g h t ( C ) 1 9 9 7 D a n M a l e k ( d m a l e k @jlc.net).
*
* This f i l e c o n t a i n s t h e l o w - l e v e l s u p p o r t a n d s e t u p f o r t h e
* PowerPC p l a t f o r m , i n c l u d i n g t r a p a n d i n t e r r u p t d i s p a t c h .
* ( The P P C 8 x x e m b e d d e d C P U s u s e h e a d _ 8 x x . S i n s t e a d . )
* /
2009-04-26 06:11:05 +04:00
# include < l i n u x / i n i t . h >
2020-06-09 07:32:42 +03:00
# include < l i n u x / p g t a b l e . h >
2005-10-10 16:20:10 +04:00
# include < a s m / r e g . h >
2005-09-26 10:04:21 +04:00
# include < a s m / p a g e . h >
# include < a s m / m m u . h >
# include < a s m / c p u t a b l e . h >
# include < a s m / c a c h e . h >
# include < a s m / t h r e a d _ i n f o . h >
# include < a s m / p p c _ a s m . h >
# include < a s m / a s m - o f f s e t s . h >
2008-04-17 08:34:59 +04:00
# include < a s m / p t r a c e . h >
2008-12-18 22:13:24 +03:00
# include < a s m / b u g . h >
2010-04-16 02:11:57 +04:00
# include < a s m / k v m _ b o o k 3 s _ a s m . h >
2016-01-14 07:33:46 +03:00
# include < a s m / e x p o r t . h >
2018-07-05 19:25:01 +03:00
# include < a s m / f e a t u r e - f i x u p s . h >
2005-09-26 10:04:21 +04:00
2019-04-30 15:38:50 +03:00
# include " h e a d _ 3 2 . h "
2005-09-26 10:04:21 +04:00
# define L O A D _ B A T ( n , r e g , R A , R B ) \
/* see the comment for clear_bats() -- Cort */ \
li R A ,0 ; \
mtspr S P R N _ I B A T ## n # # U , R A ; \
mtspr S P R N _ D B A T ## n # # U , R A ; \
lwz R A ,( n * 1 6 ) + 0 ( r e g ) ; \
lwz R B ,( n * 1 6 ) + 4 ( r e g ) ; \
mtspr S P R N _ I B A T ## n # # U , R A ; \
mtspr S P R N _ I B A T ## n # # L , R B ; \
lwz R A ,( n * 1 6 ) + 8 ( r e g ) ; \
lwz R B ,( n * 1 6 ) + 1 2 ( r e g ) ; \
mtspr S P R N _ D B A T ## n # # U , R A ; \
2019-08-26 18:52:17 +03:00
mtspr S P R N _ D B A T ## n # # L , R B
2005-09-26 10:04:21 +04:00
2009-04-26 06:11:05 +04:00
_ _ HEAD
2005-10-10 16:20:10 +04:00
.stabs " arch/ p o w e r p c / k e r n e l / " ,N _ S O ,0 ,0 ,0 f
2020-10-06 12:05:26 +03:00
.stabs " head_ b o o k 3 s _ 3 2 . S " ,N _ S O ,0 ,0 ,0 f
2005-09-26 10:04:21 +04:00
0 :
2007-09-14 00:42:35 +04:00
_ ENTRY( _ s t e x t ) ;
2005-09-26 10:04:21 +04:00
/ *
* _ start i s d e f i n e d t h i s w a y b e c a u s e t h e X C O F F l o a d e r i n t h e O p e n F i r m w a r e
* on t h e p o w e r m a c e x p e c t s t h e e n t r y p o i n t t o b e a p r o c e d u r e d e s c r i p t o r .
* /
2007-09-14 00:42:35 +04:00
_ ENTRY( _ s t a r t ) ;
2005-09-26 10:04:21 +04:00
/ *
* These a r e h e r e f o r l e g a c y r e a s o n s , t h e k e r n e l u s e d t o
* need t o l o o k l i k e a c o f f f u n c t i o n e n t r y f o r t h e p m a c
* but w e ' r e a l w a y s s t a r t e d b y s o m e k i n d o f b o o t l o a d e r n o w .
* - - Cort
* /
nop / * u s e d b y _ _ s e c o n d a r y _ h o l d o n p r e p ( m t x ) a n d c h r p s m p * /
nop / * u s e d b y _ _ s e c o n d a r y _ h o l d o n p r e p ( m t x ) a n d c h r p s m p * /
nop
/ * PMAC
* Enter h e r e w i t h t h e k e r n e l t e x t , d a t a a n d b s s l o a d e d s t a r t i n g a t
* 0 , running w i t h v i r t u a l = = p h y s i c a l m a p p i n g .
* r5 p o i n t s t o t h e p r o m e n t r y p o i n t ( t h e c l i e n t i n t e r f a c e h a n d l e r
* address) . A d d r e s s t r a n s l a t i o n i s t u r n e d o n , w i t h t h e p r o m
* managing t h e h a s h t a b l e . I n t e r r u p t s a r e d i s a b l e d . T h e s t a c k
* pointer ( r1 ) p o i n t s t o j u s t b e l o w t h e e n d o f t h e h a l f - m e g r e g i o n
* from 0 x38 0 0 0 0 - 0 x40 0 0 0 0 , w h i c h i s m a p p e d i n a l r e a d y .
*
* If w e a r e b o o t e d f r o m M a c O S v i a B o o t X , w e e n t e r w i t h t h e k e r n e l
* image l o a d e d s o m e w h e r e , a n d t h e f o l l o w i n g v a l u e s i n r e g i s t e r s :
* r3 : ' BooX' ( 0 x42 6 f6 f58 )
* r4 : virtual a d d r e s s o f b o o t _ i n f o s _ t
* r5 : 0
*
* PREP
* This i s j u m p e d t o o n p r e p s y s t e m s r i g h t a f t e r t h e k e r n e l i s r e l o c a t e d
* to i t s p r o p e r p l a c e i n m e m o r y b y t h e b o o t l o a d e r . T h e e x p e c t e d l a y o u t
* of t h e r e g s i s :
* r3 : ptr t o r e s i d u a l d a t a
* r4 : initrd_ s t a r t o r i f n o i n i t r d t h e n 0
* r5 : initrd_ e n d - u n u s e d i f r4 i s 0
* r6 : Start o f c o m m a n d l i n e s t r i n g
* r7 : End o f c o m m a n d l i n e s t r i n g
*
* This j u s t g e t s a m i n i m a l m m u e n v i r o n m e n t s e t u p s o w e c a n c a l l
* start_ h e r e ( ) t o d o t h e r e a l w o r k .
* - - Cort
* /
.globl __start
__start :
/ *
* We h a v e t o d o a n y O F c a l l s b e f o r e w e m a p o u r s e l v e s t o K E R N E L B A S E ,
* because O F m a y h a v e I / O d e v i c e s m a p p e d i n t o t h a t a r e a
* ( particularly o n C H R P ) .
* /
2005-10-06 06:06:20 +04:00
cmpwi 0 ,r5 ,0
beq 1 f
2008-10-13 22:38:48 +04:00
2009-03-10 20:53:27 +03:00
# ifdef C O N F I G _ P P C _ O F _ B O O T _ T R A M P O L I N E
2008-10-13 22:38:48 +04:00
/* find out where we are now */
bcl 2 0 ,3 1 ,$ + 4
0 : mflr r8 / * r8 = r u n t i m e a d d r h e r e * /
addis r8 ,r8 ,( _ s t e x t - 0 b ) @ha
addi r8 ,r8 ,( _ s t e x t - 0 b ) @l /* current runtime base addr */
2005-10-06 06:06:20 +04:00
bl p r o m _ i n i t
2009-03-10 20:53:27 +03:00
# endif / * C O N F I G _ P P C _ O F _ B O O T _ T R A M P O L I N E * /
/ * We n e v e r r e t u r n . W e a l s o h i t t h a t t r a p i f t r y i n g t o b o o t
* from O F w h i l e C O N F I G _ P P C _ O F _ B O O T _ T R A M P O L I N E i s n ' t s e l e c t e d * /
2005-10-06 06:06:20 +04:00
trap
2005-11-23 09:58:13 +03:00
/ *
* Check f o r B o o t X s i g n a t u r e w h e n s u p p o r t i n g P o w e r M a c a n d b r a n c h t o
* appropriate t r a m p o l i n e i f i t ' s p r e s e n t
* /
# ifdef C O N F I G _ P P C _ P M A C
1 : lis r31 ,0 x42 6 f
ori r31 ,r31 ,0 x6 f58
cmpw 0 ,r3 ,r31
bne 1 f
bl b o o t x _ i n i t
trap
# endif / * C O N F I G _ P P C _ P M A C * /
2011-07-25 15:29:33 +04:00
1 : mr r31 ,r3 / * s a v e d e v i c e t r e e p t r * /
2005-09-26 10:04:21 +04:00
li r24 ,0 / * c p u # * /
/ *
* early_ i n i t ( ) d o e s t h e e a r l y m a c h i n e i d e n t i f i c a t i o n a n d d o e s
* the n e c e s s a r y l o w - l e v e l s e t u p a n d c l e a r s t h e B S S
* - - Cort < c o r t @fsmlabs.com>
* /
bl e a r l y _ i n i t
/ * Switch M M U o f f , c l e a r B A T s a n d f l u s h T L B . A t t h i s p o i n t , r3 c o n t a i n s
* the p h y s i c a l a d d r e s s w e a r e r u n n i n g a t , r e t u r n e d b y e a r l y _ i n i t ( )
* /
bl m m u _ o f f
__after_mmu_off :
bl c l e a r _ b a t s
bl f l u s h _ t l b s
bl i n i t i a l _ b a t s
2019-04-26 19:23:36 +03:00
bl l o a d _ s e g m e n t _ r e g i s t e r s
2020-11-07 12:07:40 +03:00
bl r e l o c _ o f f s e t
2019-04-26 19:23:36 +03:00
bl e a r l y _ h a s h _ t a b l e
2007-06-13 08:52:54 +04:00
# if d e f i n e d ( C O N F I G _ B O O T X _ T E X T )
2005-11-23 09:57:25 +03:00
bl s e t u p _ d i s p _ b a t
# endif
2007-07-16 20:43:43 +04:00
# ifdef C O N F I G _ P P C _ E A R L Y _ D E B U G _ C P M
bl s e t u p _ c p m _ b a t
# endif
2009-12-12 09:31:51 +03:00
# ifdef C O N F I G _ P P C _ E A R L Y _ D E B U G _ U S B G E C K O
bl s e t u p _ u s b g e c k o _ b a t
# endif
2005-09-26 10:04:21 +04:00
/ *
* Call s e t u p _ c p u f o r C P U 0 a n d i n i t i a l i z e 6 x x I d l e
* /
bl r e l o c _ o f f s e t
li r24 ,0 / * c p u # * /
bl c a l l _ s e t u p _ c p u / * C a l l s e t u p _ c p u f o r t h i s C P U * /
bl r e l o c _ o f f s e t
bl i n i t _ i d l e _ 6 x x
/ *
* We n e e d t o r u n w i t h _ s t a r t a t p h y s i c a l a d d r e s s 0 .
* On C H R P , w e a r e l o a d e d a t 0 x10 0 0 0 s i n c e O F o n C H R P u s e s
* the e x c e p t i o n v e c t o r s a t 0 ( a n d t h e r e f o r e t h i s c o p y
* overwrites O F ' s e x c e p t i o n v e c t o r s w i t h o u r o w n ) .
2005-10-06 06:06:20 +04:00
* The M M U i s o f f a t t h i s p o i n t .
2005-09-26 10:04:21 +04:00
* /
bl r e l o c _ o f f s e t
mr r26 ,r3
addis r4 ,r3 ,K E R N E L B A S E @h /* current address of _start */
2008-12-17 13:09:13 +03:00
lis r5 ,P H Y S I C A L _ S T A R T @h
cmplw 0 ,r4 ,r5 / * a l r e a d y r u n n i n g a t P H Y S I C A L _ S T A R T ? * /
2005-09-26 10:04:21 +04:00
bne r e l o c a t e _ k e r n e l
/ *
* we n o w h a v e t h e 1 s t 1 6 M o f r a m m a p p e d w i t h t h e b a t s .
* prep n e e d s t h e m m u t o b e t u r n e d o n h e r e , b u t p m a c a l r e a d y h a s i t o n .
* this s h o u l d n ' t b o t h e r t h e p m a c s i n c e i t j u s t g e t s t u r n e d o n a g a i n
* as w e j u m p t o o u r c o d e a t K E R N E L B A S E . - - C o r t
* Actually n o , p m a c d o e s n ' t h a v e i t o n a n y m o r e . B o o t X e n t e r s w i t h M M U
* off, a n d i n o t h e r c a s e s , w e n o w t u r n i t o f f b e f o r e c h a n g i n g B A T s a b o v e .
* /
turn_on_mmu :
mfmsr r0
2019-04-26 19:23:36 +03:00
ori r0 ,r0 ,M S R _ D R | M S R _ I R | M S R _ R I
2005-09-26 10:04:21 +04:00
mtspr S P R N _ S R R 1 ,r0
lis r0 ,s t a r t _ h e r e @h
ori r0 ,r0 ,s t a r t _ h e r e @l
mtspr S P R N _ S R R 0 ,r0
2020-11-08 19:57:36 +03:00
rfi / * e n a b l e s M M U * /
2005-09-26 10:04:21 +04:00
/ *
* We n e e d _ _ s e c o n d a r y _ h o l d a s a p l a c e t o h o l d t h e o t h e r c p u s o n
* an S M P m a c h i n e , e v e n w h e n w e a r e r u n n i n g a U P k e r n e l .
* /
. = 0 xc0 / * f o r p r e p b o o t l o a d e r * /
li r3 ,1 / * M T X o n l y h a s 1 c p u * /
.globl __secondary_hold
__secondary_hold :
/* tell the master we're here */
2005-10-26 15:45:56 +04:00
stw r3 ,_ _ s e c o n d a r y _ h o l d _ a c k n o w l e d g e @l(0)
2005-09-26 10:04:21 +04:00
# ifdef C O N F I G _ S M P
100 : lwz r4 ,0 ( 0 )
/* wait until we're told to start */
cmpw 0 ,r4 ,r3
bne 1 0 0 b
/* our cpu # was at addr 0 - go */
mr r24 ,r3 / * c p u # * /
b _ _ s e c o n d a r y _ s t a r t
# else
b .
# endif / * C O N F I G _ S M P * /
2005-10-26 15:45:56 +04:00
.globl __secondary_hold_spinloop
__secondary_hold_spinloop :
.long 0
.globl __secondary_hold_acknowledge
__secondary_hold_acknowledge :
.long - 1
2005-09-26 10:04:21 +04:00
/* System reset */
/ * core9 9 p m a c s t a r t s t h e s e c o n a r y h e r e b y c h a n g i n g t h e v e c t o r , a n d
2021-01-30 16:08:31 +03:00
putting i t b a c k t o w h a t i t w a s ( u n k n o w n _ a s y n c _ e x c e p t i o n ) w h e n d o n e . * /
EXCEPTION( 0 x10 0 , R e s e t , u n k n o w n _ a s y n c _ e x c e p t i o n , E X C _ X F E R _ S T D )
2005-09-26 10:04:21 +04:00
/* Machine check */
/ *
* On C H R P , t h i s i s c o m p l i c a t e d b y t h e f a c t t h a t w e c o u l d g e t a
* machine c h e c k i n s i d e R T A S , a n d w e h a v e n o g u a r a n t e e t h a t c e r t a i n
* critical r e g i s t e r s w i l l h a v e t h e v a l u e s w e e x p e c t . T h e s e t o f
* registers t h a t m i g h t h a v e b a d v a l u e s i n c l u d e s a l l t h e G P R s
* and a l l t h e B A T s . W e i n d i c a t e t h a t w e a r e i n R T A S b y p u t t i n g
* a n o n - z e r o v a l u e , t h e a d d r e s s o f t h e e x c e p t i o n f r a m e t o u s e ,
2019-02-21 13:37:54 +03:00
* in t h r e a d . r t a s _ s p . T h e m a c h i n e c h e c k h a n d l e r c h e c k s t h r e a d . r t a s _ s p
* and u s e s i t s v a l u e i f i t i s n o n - z e r o .
2005-09-26 10:04:21 +04:00
* ( Other e x c e p t i o n h a n d l e r s a s s u m e t h a t r1 i s a v a l i d k e r n e l s t a c k
* pointer w h e n w e t a k e a n e x c e p t i o n f r o m s u p e r v i s o r m o d e . )
* - - paulus.
* /
. = 0 x2 0 0
2010-04-16 02:11:57 +04:00
DO_ K V M 0 x20 0
2019-12-21 11:32:38 +03:00
MachineCheck :
2019-12-21 11:32:23 +03:00
EXCEPTION_ P R O L O G _ 0
2005-09-26 10:04:21 +04:00
# ifdef C O N F I G _ P P C _ C H R P
2020-12-22 10:11:18 +03:00
# ifdef C O N F I G _ V M A P _ S T A C K
mtspr S P R N _ S P R G _ S C R A T C H 2 ,r1
mfspr r1 , S P R N _ S P R G _ T H R E A D
lwz r1 , R T A S _ S P ( r1 )
cmpwi c r1 , r1 , 0
bne c r1 , 7 f
mfspr r1 , S P R N _ S P R G _ S C R A T C H 2
# else
2019-02-21 13:37:54 +03:00
mfspr r11 , S P R N _ S P R G _ T H R E A D
lwz r11 , R T A S _ S P ( r11 )
cmpwi c r1 , r11 , 0
bne c r1 , 7 f
2020-12-22 10:11:18 +03:00
# endif
2005-09-26 10:04:21 +04:00
# endif / * C O N F I G _ P P C _ C H R P * /
2019-12-21 11:32:38 +03:00
EXCEPTION_ P R O L O G _ 1 f o r _ r t a s =1
2005-09-26 10:04:21 +04:00
7 : EXCEPTION_ P R O L O G _ 2
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
# ifdef C O N F I G _ P P C _ C H R P
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
beq c r1 , m a c h i n e _ c h e c k _ t r a m p
2020-03-31 19:03:43 +03:00
twi 3 1 , 0 , 0
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
# else
b m a c h i n e _ c h e c k _ t r a m p
2005-09-26 10:04:21 +04:00
# endif
/* Data access exception. */
. = 0 x3 0 0
2010-04-16 02:11:57 +04:00
DO_ K V M 0 x30 0
2005-09-26 10:04:21 +04:00
DataAccess :
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
# ifdef C O N F I G _ V M A P _ S T A C K
2020-12-18 09:56:05 +03:00
# ifdef C O N F I G _ P P C _ B O O K 3 S _ 6 0 4
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
BEGIN_ M M U _ F T R _ S E C T I O N
2020-11-25 10:10:52 +03:00
mtspr S P R N _ S P R G _ S C R A T C H 2 ,r10
mfspr r10 , S P R N _ S P R G _ T H R E A D
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
stw r11 , T H R 1 1 ( r10 )
mfspr r10 , S P R N _ D S I S R
mfcr r11
andis. r10 , r10 , ( D S I S R _ B A D _ F A U L T _ 3 2 S | D S I S R _ D A B R M A T C H ) @h
mfspr r10 , S P R N _ S P R G _ T H R E A D
beq h a s h _ p a g e _ d s i
.Lhash_page_dsi_cont :
mtcr r11
lwz r11 , T H R 1 1 ( r10 )
2020-11-25 10:10:52 +03:00
mfspr r10 , S P R N _ S P R G _ S C R A T C H 2
MMU_ F T R _ S E C T I O N _ E L S E
b 1 f
ALT_ M M U _ F T R _ S E C T I O N _ E N D _ I F S E T ( M M U _ F T R _ H P T E _ T A B L E )
2020-12-18 09:56:05 +03:00
# endif
2020-11-25 10:10:52 +03:00
1 : EXCEPTION_ P R O L O G _ 0 h a n d l e _ d a r _ d s i s r =1
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
EXCEPTION_ P R O L O G _ 1
b h a n d l e _ p a g e _ f a u l t _ t r a m p _ 1
# else / * C O N F I G _ V M A P _ S T A C K * /
2019-12-21 11:32:38 +03:00
EXCEPTION_ P R O L O G h a n d l e _ d a r _ d s i s r =1
2019-12-21 11:32:36 +03:00
get_ a n d _ s a v e _ d a r _ d s i s r _ o n _ s t a c k r4 , r5 , r11
2020-12-18 09:56:05 +03:00
# ifdef C O N F I G _ P P C _ B O O K 3 S _ 6 0 4
2019-12-21 11:32:36 +03:00
BEGIN_ M M U _ F T R _ S E C T I O N
andis. r0 , r5 , ( D S I S R _ B A D _ F A U L T _ 3 2 S | D S I S R _ D A B R M A T C H ) @h
bne h a n d l e _ p a g e _ f a u l t _ t r a m p _ 2 / * i f n o t , t r y t o p u t a P T E * /
2020-05-20 13:23:45 +03:00
rlwinm r3 , r5 , 3 2 - 1 5 , 2 1 , 2 1 / * D S I S R _ S T O R E - > _ P A G E _ R W * /
2005-09-26 10:04:21 +04:00
bl h a s h _ p a g e
2019-12-21 11:32:36 +03:00
b h a n d l e _ p a g e _ f a u l t _ t r a m p _ 1
2020-11-25 10:10:48 +03:00
MMU_ F T R _ S E C T I O N _ E L S E
2020-12-18 09:56:05 +03:00
# endif
2019-12-21 11:32:36 +03:00
b h a n d l e _ p a g e _ f a u l t _ t r a m p _ 2
2020-12-18 09:56:05 +03:00
# ifdef C O N F I G _ P P C _ B O O K 3 S _ 6 0 4
2019-12-21 11:32:36 +03:00
ALT_ M M U _ F T R _ S E C T I O N _ E N D _ I F S E T ( M M U _ F T R _ H P T E _ T A B L E )
2020-12-18 09:56:05 +03:00
# endif
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
# endif / * C O N F I G _ V M A P _ S T A C K * /
2005-09-26 10:04:21 +04:00
/* Instruction access exception. */
. = 0 x4 0 0
2010-04-16 02:11:57 +04:00
DO_ K V M 0 x40 0
2005-09-26 10:04:21 +04:00
InstructionAccess :
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
# ifdef C O N F I G _ V M A P _ S T A C K
mtspr S P R N _ S P R G _ S C R A T C H 0 ,r10
mtspr S P R N _ S P R G _ S C R A T C H 1 ,r11
mfspr r10 , S P R N _ S P R G _ T H R E A D
mfspr r11 , S P R N _ S R R 0
stw r11 , S R R 0 ( r10 )
mfspr r11 , S P R N _ S R R 1 / * c h e c k w h e t h e r u s e r o r k e r n e l * /
stw r11 , S R R 1 ( r10 )
mfcr r10
2020-12-18 09:56:05 +03:00
# ifdef C O N F I G _ P P C _ B O O K 3 S _ 6 0 4
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
BEGIN_ M M U _ F T R _ S E C T I O N
andis. r11 , r11 , S R R 1 _ I S I _ N O P T @h /* no pte found? */
bne h a s h _ p a g e _ i s i
.Lhash_page_isi_cont :
mfspr r11 , S P R N _ S R R 1 / * c h e c k w h e t h e r u s e r o r k e r n e l * /
END_ M M U _ F T R _ S E C T I O N _ I F S E T ( M M U _ F T R _ H P T E _ T A B L E )
2020-12-18 09:56:05 +03:00
# endif
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
andi. r11 , r11 , M S R _ P R
EXCEPTION_ P R O L O G _ 1
EXCEPTION_ P R O L O G _ 2
# else / * C O N F I G _ V M A P _ S T A C K * /
2005-09-26 10:04:21 +04:00
EXCEPTION_ P R O L O G
2017-07-19 07:49:28 +03:00
andis. r0 ,r9 ,S R R 1 _ I S I _ N O P T @h /* no pte found? */
2005-09-26 10:04:21 +04:00
beq 1 f / * i f s o , t r y t o p u t a P T E * /
li r3 ,0 / * i n t o t h e h a s h t a b l e * /
mr r4 ,r12 / * S R R 0 i s f a u l t a d d r e s s * /
2020-12-18 09:56:05 +03:00
# ifdef C O N F I G _ P P C _ B O O K 3 S _ 6 0 4
2018-11-09 20:33:22 +03:00
BEGIN_ M M U _ F T R _ S E C T I O N
2005-09-26 10:04:21 +04:00
bl h a s h _ p a g e
2018-11-09 20:33:22 +03:00
END_ M M U _ F T R _ S E C T I O N _ I F S E T ( M M U _ F T R _ H P T E _ T A B L E )
2020-12-18 09:56:05 +03:00
# endif
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
# endif / * C O N F I G _ V M A P _ S T A C K * /
2017-07-19 07:49:28 +03:00
andis. r5 ,r9 ,D S I S R _ S R R 1 _ M A T C H _ 3 2 S @h /* Filter relevant SRR1 bits */
2021-01-30 16:08:16 +03:00
stw r5 , _ D S I S R ( r11 )
stw r12 , _ D A R ( r11 )
2012-03-07 09:48:45 +04:00
EXC_ X F E R _ L I T E ( 0 x40 0 , h a n d l e _ p a g e _ f a u l t )
2005-09-26 10:04:21 +04:00
/* External interrupt */
EXCEPTION( 0 x50 0 , H a r d w a r e I n t e r r u p t , d o _ I R Q , E X C _ X F E R _ L I T E )
/* Alignment exception */
. = 0 x6 0 0
2010-04-16 02:11:57 +04:00
DO_ K V M 0 x60 0
2005-09-26 10:04:21 +04:00
Alignment :
2019-12-21 11:32:38 +03:00
EXCEPTION_ P R O L O G h a n d l e _ d a r _ d s i s r =1
2019-12-21 11:32:26 +03:00
save_ d a r _ d s i s r _ o n _ s t a c k r4 , r5 , r11
2005-09-26 10:04:21 +04:00
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
b a l i g n m e n t _ e x c e p t i o n _ t r a m p
2005-09-26 10:04:21 +04:00
/* Program check exception */
2005-10-01 12:43:42 +04:00
EXCEPTION( 0 x70 0 , P r o g r a m C h e c k , p r o g r a m _ c h e c k _ e x c e p t i o n , E X C _ X F E R _ S T D )
2005-09-26 10:04:21 +04:00
/* Floating-point unavailable */
. = 0 x8 0 0
2010-04-16 02:11:57 +04:00
DO_ K V M 0 x80 0
2005-09-26 10:04:21 +04:00
FPUnavailable :
2020-08-18 20:19:18 +03:00
# ifdef C O N F I G _ P P C _ F P U
2006-12-08 11:43:30 +03:00
BEGIN_ F T R _ S E C T I O N
/ *
* Certain F r e e s c a l e c o r e s d o n ' t h a v e a F P U a n d t r e a t f p i n s t r u c t i o n s
* as a F P U n a v a i l a b l e e x c e p t i o n . R e d i r e c t t o i l l e g a l / e m u l a t i o n h a n d l i n g .
* /
b P r o g r a m C h e c k
END_ F T R _ S E C T I O N _ I F S E T ( C P U _ F T R _ F P U _ U N A V A I L A B L E )
2005-09-26 10:04:21 +04:00
EXCEPTION_ P R O L O G
2008-06-25 08:07:18 +04:00
beq 1 f
bl l o a d _ u p _ f p u / * i f f r o m u s e r , j u s t l o a d i t u p * /
b f a s t _ e x c e p t i o n _ r e t u r n
1 : addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
2019-04-30 15:38:59 +03:00
EXC_ X F E R _ L I T E ( 0 x80 0 , k e r n e l _ f p _ u n a v a i l a b l e _ e x c e p t i o n )
2020-08-18 20:19:18 +03:00
# else
b P r o g r a m C h e c k
# endif
2005-09-26 10:04:21 +04:00
/* Decrementer */
EXCEPTION( 0 x90 0 , D e c r e m e n t e r , t i m e r _ i n t e r r u p t , E X C _ X F E R _ L I T E )
2019-04-30 15:38:59 +03:00
EXCEPTION( 0 x a00 , T r a p _ 0 a , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x b00 , T r a p _ 0 b , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
2005-09-26 10:04:21 +04:00
/* System call */
. = 0 xc0 0
2010-04-16 02:11:57 +04:00
DO_ K V M 0 x c00
2005-09-26 10:04:21 +04:00
SystemCall :
2019-04-30 15:39:02 +03:00
SYSCALL_ E N T R Y 0 x c00
2005-09-26 10:04:21 +04:00
2005-10-01 12:43:42 +04:00
EXCEPTION( 0 x d00 , S i n g l e S t e p , s i n g l e _ s t e p _ e x c e p t i o n , E X C _ X F E R _ S T D )
2019-04-30 15:38:59 +03:00
EXCEPTION( 0 x e 0 0 , T r a p _ 0 e , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
2005-09-26 10:04:21 +04:00
/ *
* The A l t i v e c u n a v a i l a b l e t r a p i s a t 0 x0 f20 . F o o .
* We e f f e c t i v e l y r e m a p i t t o 0 x30 0 0 .
* We i n c l u d e a n a l t i v e c u n a v a i l a b l e e x c e p t i o n v e c t o r e v e n i f
* not c o n f i g u r e d f o r A l t i v e c , s o t h a t y o u c a n ' t p a n i c a
* non- a l t i v e c k e r n e l r u n n i n g o n a m a c h i n e w i t h a l t i v e c j u s t
* by e x e c u t i n g a n a l t i v e c i n s t r u c t i o n .
* /
. = 0 xf0 0
2010-04-16 02:11:57 +04:00
DO_ K V M 0 x f00
2005-12-16 05:02:04 +03:00
b P e r f o r m a n c e M o n i t o r
2005-09-26 10:04:21 +04:00
. = 0 xf2 0
2010-04-16 02:11:57 +04:00
DO_ K V M 0 x f20
2005-09-26 10:04:21 +04:00
b A l t i V e c U n a v a i l a b l e
/ *
* Handle T L B m i s s f o r i n s t r u c t i o n o n 6 0 3 / 6 0 3 e .
* Note : we g e t a n a l t e r n a t e s e t o f r0 - r3 t o u s e a u t o m a t i c a l l y .
* /
. = 0 x1 0 0 0
InstructionTLBMiss :
/ *
2009-03-19 06:55:39 +03:00
* r0 : scratch
2005-09-26 10:04:21 +04:00
* r1 : linux s t y l e p t e ( l a t e r b e c o m e s p p c h a r d w a r e p t e )
* r2 : ptr t o l i n u x - s t y l e p t e
* r3 : scratch
* /
/* Get PTE (linux-style) and check access */
mfspr r3 ,S P R N _ I M I S S
2020-11-25 10:10:46 +03:00
# ifdef C O N F I G _ M O D U L E S
2020-06-29 14:15:24 +03:00
lis r1 , T A S K _ S I Z E @h /* check if kernel address */
2007-10-11 22:36:52 +04:00
cmplw 0 ,r1 ,r3
2019-02-21 13:37:58 +03:00
# endif
2020-11-25 10:10:50 +03:00
mfspr r2 , S P R N _ S D R 1
2019-02-21 13:38:02 +03:00
li r1 ,_ P A G E _ P R E S E N T | _ P A G E _ A C C E S S E D | _ P A G E _ E X E C
2020-11-25 10:10:50 +03:00
rlwinm r2 , r2 , 2 8 , 0 x f f f f f00 0
2020-11-25 10:10:46 +03:00
# ifdef C O N F I G _ M O D U L E S
2020-06-29 14:15:24 +03:00
bgt- 1 1 2 f
2019-02-21 13:37:56 +03:00
lis r2 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @ha /* if kernel address, use */
addi r2 , r2 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @l /* kernel page table */
2019-02-21 13:37:58 +03:00
# endif
2019-02-21 13:37:55 +03:00
112 : rlwimi r2 ,r3 ,1 2 ,2 0 ,2 9 / * i n s e r t t o p 1 0 b i t s o f a d d r e s s * /
2005-09-26 10:04:21 +04:00
lwz r2 ,0 ( r2 ) / * g e t p m d e n t r y * /
rlwinm. r2 ,r2 ,0 ,0 ,1 9 / * e x t r a c t a d d r e s s o f p t e p a g e * /
beq- I n s t r u c t i o n A d d r e s s I n v a l i d / * r e t u r n i f n o m a p p i n g * /
rlwimi r2 ,r3 ,2 2 ,2 0 ,2 9 / * i n s e r t n e x t 1 0 b i t s o f a d d r e s s * /
2009-03-19 06:55:40 +03:00
lwz r0 ,0 ( r2 ) / * g e t l i n u x - s t y l e p t e * /
andc. r1 ,r1 ,r0 / * c h e c k a c c e s s & ~ p e r m i s s i o n * /
2005-09-26 10:04:21 +04:00
bne- I n s t r u c t i o n A d d r e s s I n v a l i d / * r e t u r n i f a c c e s s n o t p e r m i t t e d * /
/* Convert linux-style PTE to low word of PPC-style PTE */
2020-05-20 13:23:45 +03:00
rlwimi r0 ,r0 ,3 2 - 2 ,3 1 ,3 1 / * _ P A G E _ U S E R - > P P l s b * /
2019-03-11 11:30:36 +03:00
ori r1 , r1 , 0 x e 0 6 / * c l e a r o u t r e s e r v e d b i t s * /
andc r1 , r0 , r1 / * P P = u s e r ? 1 : 0 * /
2009-03-14 17:23:03 +03:00
BEGIN_ F T R _ S E C T I O N
rlwinm r1 ,r1 ,0 ,~ _ P A G E _ C O H E R E N T / * c l e a r M ( c o h e r e n c e n o t r e q u i r e d ) * /
END_ F T R _ S E C T I O N _ I F C L R ( C P U _ F T R _ N E E D _ C O H E R E N T )
2005-09-26 10:04:21 +04:00
mtspr S P R N _ R P A ,r1
tlbli r3
mfspr r3 ,S P R N _ S R R 1 / * N e e d t o r e s t o r e C R 0 * /
mtcrf 0 x80 ,r3
rfi
InstructionAddressInvalid :
mfspr r3 ,S P R N _ S R R 1
rlwinm r1 ,r3 ,9 ,6 ,6 / * G e t l o a d / s t o r e b i t * /
addis r1 ,r1 ,0 x20 0 0
mtspr S P R N _ D S I S R ,r1 / * ( s h o u l d n ' t b e n e e d e d ) * /
andi. r2 ,r3 ,0 x F F F F / * C l e a r u p p e r b i t s o f S R R 1 * /
or r2 ,r2 ,r1
mtspr S P R N _ S R R 1 ,r2
mfspr r1 ,S P R N _ I M I S S / * G e t f a i l i n g a d d r e s s * /
rlwinm. r2 ,r2 ,0 ,3 1 ,3 1 / * C h e c k f o r l i t t l e e n d i a n a c c e s s * /
rlwimi r2 ,r2 ,1 ,3 0 ,3 0 / * c h a n g e 1 - > 3 * /
xor r1 ,r1 ,r2
mtspr S P R N _ D A R ,r1 / * S e t f a u l t a d d r e s s * /
mfmsr r0 / * R e s t o r e " n o r m a l " r e g i s t e r s * /
xoris r0 ,r0 ,M S R _ T G P R > > 1 6
mtcrf 0 x80 ,r3 / * R e s t o r e C R 0 * /
mtmsr r0
b I n s t r u c t i o n A c c e s s
/ *
* Handle T L B m i s s f o r D A T A L o a d o p e r a t i o n o n 6 0 3 / 6 0 3 e
* /
. = 0 x1 1 0 0
DataLoadTLBMiss :
/ *
2009-03-19 06:55:39 +03:00
* r0 : scratch
2005-09-26 10:04:21 +04:00
* r1 : linux s t y l e p t e ( l a t e r b e c o m e s p p c h a r d w a r e p t e )
* r2 : ptr t o l i n u x - s t y l e p t e
* r3 : scratch
* /
/* Get PTE (linux-style) and check access */
mfspr r3 ,S P R N _ D M I S S
2020-06-29 14:15:24 +03:00
lis r1 , T A S K _ S I Z E @h /* check if kernel address */
2007-10-11 22:36:52 +04:00
cmplw 0 ,r1 ,r3
2020-11-25 10:10:50 +03:00
mfspr r2 , S P R N _ S D R 1
2019-02-21 13:38:02 +03:00
li r1 , _ P A G E _ P R E S E N T | _ P A G E _ A C C E S S E D
2020-11-25 10:10:50 +03:00
rlwinm r2 , r2 , 2 8 , 0 x f f f f f00 0
2020-06-29 14:15:24 +03:00
bgt- 1 1 2 f
2019-02-21 13:37:56 +03:00
lis r2 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @ha /* if kernel address, use */
addi r2 , r2 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @l /* kernel page table */
2019-02-21 13:37:55 +03:00
112 : rlwimi r2 ,r3 ,1 2 ,2 0 ,2 9 / * i n s e r t t o p 1 0 b i t s o f a d d r e s s * /
2005-09-26 10:04:21 +04:00
lwz r2 ,0 ( r2 ) / * g e t p m d e n t r y * /
rlwinm. r2 ,r2 ,0 ,0 ,1 9 / * e x t r a c t a d d r e s s o f p t e p a g e * /
beq- D a t a A d d r e s s I n v a l i d / * r e t u r n i f n o m a p p i n g * /
rlwimi r2 ,r3 ,2 2 ,2 0 ,2 9 / * i n s e r t n e x t 1 0 b i t s o f a d d r e s s * /
2009-03-19 06:55:40 +03:00
lwz r0 ,0 ( r2 ) / * g e t l i n u x - s t y l e p t e * /
andc. r1 ,r1 ,r0 / * c h e c k a c c e s s & ~ p e r m i s s i o n * /
2005-09-26 10:04:21 +04:00
bne- D a t a A d d r e s s I n v a l i d / * r e t u r n i f a c c e s s n o t p e r m i t t e d * /
/ *
* NOTE! W e a r e a s s u m i n g t h i s i s n o t a n S M P s y s t e m , o t h e r w i s e
* we w o u l d n e e d t o u p d a t e t h e p t e a t o m i c a l l y w i t h l w a r x / s t w c x .
* /
/* Convert linux-style PTE to low word of PPC-style PTE */
2020-05-20 13:23:45 +03:00
rlwinm r1 ,r0 ,3 2 - 9 ,3 0 ,3 0 / * _ P A G E _ R W - > P P m s b * /
rlwimi r0 ,r0 ,3 2 - 1 ,3 0 ,3 0 / * _ P A G E _ U S E R - > P P m s b * /
rlwimi r0 ,r0 ,3 2 - 1 ,3 1 ,3 1 / * _ P A G E _ U S E R - > P P l s b * /
2009-03-17 18:17:50 +03:00
ori r1 ,r1 ,0 x e 0 4 / * c l e a r o u t r e s e r v e d b i t s * /
2019-03-11 11:30:36 +03:00
andc r1 ,r0 ,r1 / * P P = u s e r ? r w ? 1 : 3 : 0 * /
2009-03-14 17:23:03 +03:00
BEGIN_ F T R _ S E C T I O N
rlwinm r1 ,r1 ,0 ,~ _ P A G E _ C O H E R E N T / * c l e a r M ( c o h e r e n c e n o t r e q u i r e d ) * /
END_ F T R _ S E C T I O N _ I F C L R ( C P U _ F T R _ N E E D _ C O H E R E N T )
2005-09-26 10:04:21 +04:00
mtspr S P R N _ R P A ,r1
2009-03-19 06:55:41 +03:00
mfspr r2 ,S P R N _ S R R 1 / * N e e d t o r e s t o r e C R 0 * /
mtcrf 0 x80 ,r2
BEGIN_ M M U _ F T R _ S E C T I O N
li r0 ,1
2009-07-15 00:52:54 +04:00
mfspr r1 ,S P R N _ S P R G _ 6 0 3 _ L R U
2009-03-19 06:55:41 +03:00
rlwinm r2 ,r3 ,2 0 ,2 7 ,3 1 / * G e t A d d r e s s b i t s 1 5 : 1 9 * /
slw r0 ,r0 ,r2
xor r1 ,r0 ,r1
srw r0 ,r1 ,r2
2009-07-15 00:52:54 +04:00
mtspr S P R N _ S P R G _ 6 0 3 _ L R U ,r1
2009-03-19 06:55:41 +03:00
mfspr r2 ,S P R N _ S R R 1
rlwimi r2 ,r0 ,3 1 - 1 4 ,1 4 ,1 4
mtspr S P R N _ S R R 1 ,r2
END_ M M U _ F T R _ S E C T I O N _ I F S E T ( M M U _ F T R _ N E E D _ D T L B _ S W _ L R U )
2005-09-26 10:04:21 +04:00
tlbld r3
rfi
DataAddressInvalid :
mfspr r3 ,S P R N _ S R R 1
rlwinm r1 ,r3 ,9 ,6 ,6 / * G e t l o a d / s t o r e b i t * /
addis r1 ,r1 ,0 x20 0 0
mtspr S P R N _ D S I S R ,r1
andi. r2 ,r3 ,0 x F F F F / * C l e a r u p p e r b i t s o f S R R 1 * /
mtspr S P R N _ S R R 1 ,r2
mfspr r1 ,S P R N _ D M I S S / * G e t f a i l i n g a d d r e s s * /
rlwinm. r2 ,r2 ,0 ,3 1 ,3 1 / * C h e c k f o r l i t t l e e n d i a n a c c e s s * /
beq 2 0 f / * J u m p i f b i g e n d i a n * /
xori r1 ,r1 ,3
20 : mtspr S P R N _ D A R ,r1 / * S e t f a u l t a d d r e s s * /
mfmsr r0 / * R e s t o r e " n o r m a l " r e g i s t e r s * /
xoris r0 ,r0 ,M S R _ T G P R > > 1 6
mtcrf 0 x80 ,r3 / * R e s t o r e C R 0 * /
mtmsr r0
b D a t a A c c e s s
/ *
* Handle T L B m i s s f o r D A T A S t o r e o n 6 0 3 / 6 0 3 e
* /
. = 0 x1 2 0 0
DataStoreTLBMiss :
/ *
2009-03-19 06:55:39 +03:00
* r0 : scratch
2005-09-26 10:04:21 +04:00
* r1 : linux s t y l e p t e ( l a t e r b e c o m e s p p c h a r d w a r e p t e )
* r2 : ptr t o l i n u x - s t y l e p t e
* r3 : scratch
* /
/* Get PTE (linux-style) and check access */
mfspr r3 ,S P R N _ D M I S S
2020-06-29 14:15:24 +03:00
lis r1 , T A S K _ S I Z E @h /* check if kernel address */
2007-10-11 22:36:52 +04:00
cmplw 0 ,r1 ,r3
2020-11-25 10:10:50 +03:00
mfspr r2 , S P R N _ S D R 1
2019-08-19 09:40:25 +03:00
li r1 , _ P A G E _ R W | _ P A G E _ D I R T Y | _ P A G E _ P R E S E N T | _ P A G E _ A C C E S S E D
2020-11-25 10:10:50 +03:00
rlwinm r2 , r2 , 2 8 , 0 x f f f f f00 0
2020-06-29 14:15:24 +03:00
bgt- 1 1 2 f
2019-02-21 13:37:56 +03:00
lis r2 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @ha /* if kernel address, use */
addi r2 , r2 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @l /* kernel page table */
2019-02-21 13:37:55 +03:00
112 : rlwimi r2 ,r3 ,1 2 ,2 0 ,2 9 / * i n s e r t t o p 1 0 b i t s o f a d d r e s s * /
2005-09-26 10:04:21 +04:00
lwz r2 ,0 ( r2 ) / * g e t p m d e n t r y * /
rlwinm. r2 ,r2 ,0 ,0 ,1 9 / * e x t r a c t a d d r e s s o f p t e p a g e * /
beq- D a t a A d d r e s s I n v a l i d / * r e t u r n i f n o m a p p i n g * /
rlwimi r2 ,r3 ,2 2 ,2 0 ,2 9 / * i n s e r t n e x t 1 0 b i t s o f a d d r e s s * /
2009-03-19 06:55:40 +03:00
lwz r0 ,0 ( r2 ) / * g e t l i n u x - s t y l e p t e * /
andc. r1 ,r1 ,r0 / * c h e c k a c c e s s & ~ p e r m i s s i o n * /
2005-09-26 10:04:21 +04:00
bne- D a t a A d d r e s s I n v a l i d / * r e t u r n i f a c c e s s n o t p e r m i t t e d * /
/ *
* NOTE! W e a r e a s s u m i n g t h i s i s n o t a n S M P s y s t e m , o t h e r w i s e
* we w o u l d n e e d t o u p d a t e t h e p t e a t o m i c a l l y w i t h l w a r x / s t w c x .
* /
/* Convert linux-style PTE to low word of PPC-style PTE */
2020-05-20 13:23:45 +03:00
rlwimi r0 ,r0 ,3 2 - 2 ,3 1 ,3 1 / * _ P A G E _ U S E R - > P P l s b * /
2019-03-11 11:30:36 +03:00
li r1 ,0 x e 0 6 / * c l e a r o u t r e s e r v e d b i t s & P P m s b * /
andc r1 ,r0 ,r1 / * P P = u s e r ? 1 : 0 * /
2009-03-14 17:23:03 +03:00
BEGIN_ F T R _ S E C T I O N
rlwinm r1 ,r1 ,0 ,~ _ P A G E _ C O H E R E N T / * c l e a r M ( c o h e r e n c e n o t r e q u i r e d ) * /
END_ F T R _ S E C T I O N _ I F C L R ( C P U _ F T R _ N E E D _ C O H E R E N T )
2005-09-26 10:04:21 +04:00
mtspr S P R N _ R P A ,r1
2009-03-19 06:55:41 +03:00
mfspr r2 ,S P R N _ S R R 1 / * N e e d t o r e s t o r e C R 0 * /
mtcrf 0 x80 ,r2
BEGIN_ M M U _ F T R _ S E C T I O N
li r0 ,1
2009-07-15 00:52:54 +04:00
mfspr r1 ,S P R N _ S P R G _ 6 0 3 _ L R U
2009-03-19 06:55:41 +03:00
rlwinm r2 ,r3 ,2 0 ,2 7 ,3 1 / * G e t A d d r e s s b i t s 1 5 : 1 9 * /
slw r0 ,r0 ,r2
xor r1 ,r0 ,r1
srw r0 ,r1 ,r2
2009-07-15 00:52:54 +04:00
mtspr S P R N _ S P R G _ 6 0 3 _ L R U ,r1
2009-03-19 06:55:41 +03:00
mfspr r2 ,S P R N _ S R R 1
rlwimi r2 ,r0 ,3 1 - 1 4 ,1 4 ,1 4
mtspr S P R N _ S R R 1 ,r2
END_ M M U _ F T R _ S E C T I O N _ I F S E T ( M M U _ F T R _ N E E D _ D T L B _ S W _ L R U )
2005-09-26 10:04:21 +04:00
tlbld r3
rfi
# ifndef C O N F I G _ A L T I V E C
2005-10-01 12:43:42 +04:00
# define a l t i v e c _ a s s i s t _ e x c e p t i o n u n k n o w n _ e x c e p t i o n
2020-07-24 16:17:25 +03:00
# endif
# ifndef C O N F I G _ T A U _ I N T
2021-01-30 16:08:31 +03:00
# define T A U E x c e p t i o n u n k n o w n _ a s y n c _ e x c e p t i o n
2005-09-26 10:04:21 +04:00
# endif
2019-04-30 15:38:59 +03:00
EXCEPTION( 0 x13 0 0 , T r a p _ 1 3 , i n s t r u c t i o n _ b r e a k p o i n t _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x14 0 0 , S M I , S M I E x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x15 0 0 , T r a p _ 1 5 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x16 0 0 , T r a p _ 1 6 , a l t i v e c _ a s s i s t _ e x c e p t i o n , E X C _ X F E R _ S T D )
2005-09-26 10:04:21 +04:00
EXCEPTION( 0 x17 0 0 , T r a p _ 1 7 , T A U E x c e p t i o n , E X C _ X F E R _ S T D )
2019-04-30 15:38:59 +03:00
EXCEPTION( 0 x18 0 0 , T r a p _ 1 8 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x19 0 0 , T r a p _ 1 9 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x1 a00 , T r a p _ 1 a , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x1 b00 , T r a p _ 1 b , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x1 c00 , T r a p _ 1 c , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x1 d00 , T r a p _ 1 d , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x1 e 0 0 , T r a p _ 1 e , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x1 f00 , T r a p _ 1 f , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x20 0 0 , R u n M o d e , R u n M o d e E x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x21 0 0 , T r a p _ 2 1 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x22 0 0 , T r a p _ 2 2 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x23 0 0 , T r a p _ 2 3 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x24 0 0 , T r a p _ 2 4 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x25 0 0 , T r a p _ 2 5 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x26 0 0 , T r a p _ 2 6 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x27 0 0 , T r a p _ 2 7 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x28 0 0 , T r a p _ 2 8 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x29 0 0 , T r a p _ 2 9 , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x2 a00 , T r a p _ 2 a , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x2 b00 , T r a p _ 2 b , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x2 c00 , T r a p _ 2 c , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x2 d00 , T r a p _ 2 d , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x2 e 0 0 , T r a p _ 2 e , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
EXCEPTION( 0 x2 f00 , T r a p _ 2 f , u n k n o w n _ e x c e p t i o n , E X C _ X F E R _ S T D )
2005-09-26 10:04:21 +04:00
. = 0 x3 0 0 0
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
machine_check_tramp :
EXC_ X F E R _ S T D ( 0 x20 0 , m a c h i n e _ c h e c k _ e x c e p t i o n )
alignment_exception_tramp :
EXC_ X F E R _ S T D ( 0 x60 0 , a l i g n m e n t _ e x c e p t i o n )
2019-12-21 11:32:36 +03:00
handle_page_fault_tramp_1 :
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
# ifdef C O N F I G _ V M A P _ S T A C K
EXCEPTION_ P R O L O G _ 2 h a n d l e _ d a r _ d s i s r =1
# endif
2019-12-21 11:32:36 +03:00
lwz r5 , _ D S I S R ( r11 )
/* fall through */
handle_page_fault_tramp_2 :
2021-01-30 16:08:13 +03:00
andis. r0 , r5 , D S I S R _ D A B R M A T C H @h
bne- 1 f
2019-12-21 11:32:36 +03:00
EXC_ X F E R _ L I T E ( 0 x30 0 , h a n d l e _ p a g e _ f a u l t )
2021-01-30 16:08:13 +03:00
1 : EXC_ X F E R _ S T D ( 0 x30 0 , d o _ b r e a k )
2019-12-21 11:32:36 +03:00
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
# ifdef C O N F I G _ V M A P _ S T A C K
2020-12-18 09:56:05 +03:00
# ifdef C O N F I G _ P P C _ B O O K 3 S _ 6 0 4
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
.macro save_regs_thread thread
stw r0 , T H R 0 ( \ t h r e a d )
stw r3 , T H R 3 ( \ t h r e a d )
stw r4 , T H R 4 ( \ t h r e a d )
stw r5 , T H R 5 ( \ t h r e a d )
stw r6 , T H R 6 ( \ t h r e a d )
stw r8 , T H R 8 ( \ t h r e a d )
stw r9 , T H R 9 ( \ t h r e a d )
mflr r0
stw r0 , T H L R ( \ t h r e a d )
mfctr r0
stw r0 , T H C T R ( \ t h r e a d )
.endm
.macro restore_regs_thread thread
lwz r0 , T H L R ( \ t h r e a d )
mtlr r0
lwz r0 , T H C T R ( \ t h r e a d )
mtctr r0
lwz r0 , T H R 0 ( \ t h r e a d )
lwz r3 , T H R 3 ( \ t h r e a d )
lwz r4 , T H R 4 ( \ t h r e a d )
lwz r5 , T H R 5 ( \ t h r e a d )
lwz r6 , T H R 6 ( \ t h r e a d )
lwz r8 , T H R 8 ( \ t h r e a d )
lwz r9 , T H R 9 ( \ t h r e a d )
.endm
hash_page_dsi :
save_ r e g s _ t h r e a d r10
mfdsisr r3
mfdar r4
mfsrr0 r5
mfsrr1 r9
rlwinm r3 , r3 , 3 2 - 1 5 , _ P A G E _ R W / * D S I S R _ S T O R E - > _ P A G E _ R W * /
bl h a s h _ p a g e
mfspr r10 , S P R N _ S P R G _ T H R E A D
restore_ r e g s _ t h r e a d r10
b . L h a s h _ p a g e _ d s i _ c o n t
hash_page_isi :
mr r11 , r10
mfspr r10 , S P R N _ S P R G _ T H R E A D
save_ r e g s _ t h r e a d r10
li r3 , 0
lwz r4 , S R R 0 ( r10 )
lwz r9 , S R R 1 ( r10 )
bl h a s h _ p a g e
mfspr r10 , S P R N _ S P R G _ T H R E A D
restore_ r e g s _ t h r e a d r10
mr r10 , r11
b . L h a s h _ p a g e _ i s i _ c o n t
.globl fast_hash_page_return
fast_hash_page_return :
andis. r10 , r9 , S R R 1 _ I S I _ N O P T @h /* Set on ISI, cleared on DSI */
mfspr r10 , S P R N _ S P R G _ T H R E A D
restore_ r e g s _ t h r e a d r10
bne 1 f
/* DSI */
mtcr r11
lwz r11 , T H R 1 1 ( r10 )
2020-11-25 10:10:52 +03:00
mfspr r10 , S P R N _ S P R G _ S C R A T C H 2
2020-11-08 19:57:36 +03:00
rfi
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
1 : /* ISI */
mtcr r11
mfspr r11 , S P R N _ S P R G _ S C R A T C H 1
mfspr r10 , S P R N _ S P R G _ S C R A T C H 0
2020-11-08 19:57:36 +03:00
rfi
2020-12-18 09:56:05 +03:00
# endif / * C O N F I G _ P P C _ B O O K 3 S _ 6 0 4 * /
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
2019-12-21 11:32:38 +03:00
stack_overflow :
vmap_ s t a c k _ o v e r f l o w _ e x c e p t i o n
powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Link: https://lore.kernel.org/r/64a4aa44686e9fd4b01333401367029771d9b231.1581761633.git.christophe.leroy@c-s.fr
2020-02-15 13:14:25 +03:00
# endif
2019-12-21 11:32:38 +03:00
2005-09-26 10:04:21 +04:00
AltiVecUnavailable :
EXCEPTION_ P R O L O G
# ifdef C O N F I G _ A L T I V E C
2009-06-12 10:46:47 +04:00
beq 1 f
bl l o a d _ u p _ a l t i v e c / * i f f r o m u s e r , j u s t l o a d i t u p * /
b f a s t _ e x c e p t i o n _ r e t u r n
2005-09-26 10:04:21 +04:00
# endif / * C O N F I G _ A L T I V E C * /
2009-06-12 10:46:47 +04:00
1 : addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
2019-04-30 15:38:59 +03:00
EXC_ X F E R _ L I T E ( 0 x f20 , a l t i v e c _ u n a v a i l a b l e _ e x c e p t i o n )
2005-09-26 10:04:21 +04:00
2005-12-16 05:02:04 +03:00
PerformanceMonitor :
EXCEPTION_ P R O L O G
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
EXC_ X F E R _ S T D ( 0 x f00 , p e r f o r m a n c e _ m o n i t o r _ e x c e p t i o n )
2005-09-26 10:04:21 +04:00
/ *
* This c o d e i s j u m p e d t o f r o m t h e s t a r t u p c o d e t o c o p y
2008-12-17 13:09:13 +03:00
* the k e r n e l i m a g e t o p h y s i c a l a d d r e s s P H Y S I C A L _ S T A R T .
2005-09-26 10:04:21 +04:00
* /
relocate_kernel :
addis r9 ,r26 ,k l i m i t @ha /* fetch klimit */
lwz r25 ,k l i m i t @l(r9)
addis r25 ,r25 ,- K E R N E L B A S E @h
2008-12-17 13:09:13 +03:00
lis r3 ,P H Y S I C A L _ S T A R T @h /* Destination base address */
2005-09-26 10:04:21 +04:00
li r6 ,0 / * D e s t i n a t i o n o f f s e t * /
li r5 ,0 x40 0 0 / * # b y t e s o f m e m o r y t o c o p y * /
bl c o p y _ a n d _ f l u s h / * c o p y t h e f i r s t 0 x40 0 0 b y t e s * /
addi r0 ,r3 ,4 f @l /* jump to the address of 4f */
mtctr r0 / * i n c o p y a n d d o t h e r e s t . * /
bctr / * j u m p t o t h e c o p y * /
4 : mr r5 ,r25
bl c o p y _ a n d _ f l u s h / * c o p y t h e r e s t * /
b t u r n _ o n _ m m u
/ *
* Copy r o u t i n e u s e d t o c o p y t h e k e r n e l t o s t a r t a t p h y s i c a l a d d r e s s 0
* and f l u s h a n d i n v a l i d a t e t h e c a c h e s a s n e e d e d .
* r3 = d e s t a d d r , r4 = s o u r c e a d d r , r5 = c o p y l i m i t , r6 = s t a r t o f f s e t
* on e x i t , r3 , r4 , r5 a r e u n c h a n g e d , r6 i s u p d a t e d t o b e > = r5 .
* /
2007-09-14 00:42:35 +04:00
_ ENTRY( c o p y _ a n d _ f l u s h )
2005-09-26 10:04:21 +04:00
addi r5 ,r5 ,- 4
addi r6 ,r6 ,- 4
2005-10-17 05:50:32 +04:00
4 : li r0 ,L 1 _ C A C H E _ B Y T E S / 4
2005-09-26 10:04:21 +04:00
mtctr r0
3 : addi r6 ,r6 ,4 / * c o p y a c a c h e l i n e * /
lwzx r0 ,r6 ,r4
stwx r0 ,r6 ,r3
bdnz 3 b
dcbst r6 ,r3 / * w r i t e i t t o m e m o r y * /
sync
icbi r6 ,r3 / * f l u s h t h e i c a c h e l i n e * /
cmplw 0 ,r6 ,r5
blt 4 b
sync / * a d d i t i o n a l s y n c n e e d e d o n g 4 * /
isync
addi r5 ,r5 ,4
addi r6 ,r6 ,4
blr
# ifdef C O N F I G _ S M P
2006-06-18 02:52:44 +04:00
.globl __secondary_start_mpc86xx
__secondary_start_mpc86xx :
mfspr r3 , S P R N _ P I R
stw r3 , _ _ s e c o n d a r y _ h o l d _ a c k n o w l e d g e @l(0)
mr r24 , r3 / * c p u # * /
b _ _ s e c o n d a r y _ s t a r t
2005-09-26 10:04:21 +04:00
.globl __secondary_start_pmac_0
__secondary_start_pmac_0 :
/* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
li r24 ,0
b 1 f
li r24 ,1
b 1 f
li r24 ,2
b 1 f
li r24 ,3
1 :
/ * on p o w e r s u r g e , w e c o m e i n h e r e w i t h I R =0 a n d D R =1 , a n d D B A T 0
set t o m a p t h e 0 x f00 0 0 0 0 0 - 0 x f f f f f f f f r e g i o n * /
mfmsr r0
rlwinm r0 ,r0 ,0 ,2 8 ,2 6 / * c l e a r D R ( 0 x10 ) * /
mtmsr r0
isync
.globl __secondary_start
__secondary_start :
/* Copy some CPU settings from CPU 0 */
bl _ _ r e s t o r e _ c p u _ s e t u p
lis r3 ,- K E R N E L B A S E @h
mr r4 ,r24
bl c a l l _ s e t u p _ c p u / * C a l l s e t u p _ c p u f o r t h i s C P U * /
lis r3 ,- K E R N E L B A S E @h
bl i n i t _ i d l e _ 6 x x
2019-01-17 15:25:53 +03:00
/* get current's stack and current */
2019-01-31 13:09:02 +03:00
lis r2 ,s e c o n d a r y _ c u r r e n t @ha
tophys( r2 ,r2 )
lwz r2 ,s e c o n d a r y _ c u r r e n t @l(r2)
2019-01-31 13:08:58 +03:00
tophys( r1 ,r2 )
lwz r1 ,T A S K _ S T A C K ( r1 )
2005-09-26 10:04:21 +04:00
/* stack */
addi r1 ,r1 ,T H R E A D _ S I Z E - S T A C K _ F R A M E _ O V E R H E A D
li r0 ,0
tophys( r3 ,r1 )
stw r0 ,0 ( r3 )
/* load up the MMU */
2019-06-11 18:47:20 +03:00
bl l o a d _ s e g m e n t _ r e g i s t e r s
2005-09-26 10:04:21 +04:00
bl l o a d _ u p _ m m u
/* ptr to phys current thread */
tophys( r4 ,r2 )
addi r4 ,r4 ,T H R E A D / * p h y s a d d r e s s o f o u r t h r e a d _ s t r u c t * /
2009-07-15 00:52:54 +04:00
mtspr S P R N _ S P R G _ T H R E A D ,r4
2020-11-25 10:10:50 +03:00
BEGIN_ M M U _ F T R _ S E C T I O N
2019-03-11 11:30:27 +03:00
lis r4 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @h
ori r4 , r4 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @l
2020-11-25 10:10:50 +03:00
rlwinm r4 , r4 , 4 , 0 x f f f f01 f f
mtspr S P R N _ S D R 1 , r4
END_ M M U _ F T R _ S E C T I O N _ I F C L R ( M M U _ F T R _ H P T E _ T A B L E )
2005-09-26 10:04:21 +04:00
/* enable MMU and jump to start_secondary */
li r4 ,M S R _ K E R N E L
lis r3 ,s t a r t _ s e c o n d a r y @h
ori r3 ,r3 ,s t a r t _ s e c o n d a r y @l
mtspr S P R N _ S R R 0 ,r3
mtspr S P R N _ S R R 1 ,r4
2020-11-08 19:57:36 +03:00
rfi
2005-09-26 10:04:21 +04:00
# endif / * C O N F I G _ S M P * /
2010-04-16 02:11:57 +04:00
# ifdef C O N F I G _ K V M _ B O O K 3 S _ H A N D L E R
# include " . . / k v m / b o o k 3 s _ r m h a n d l e r s . S "
# endif
2005-09-26 10:04:21 +04:00
/ *
* Load s t u f f i n t o t h e M M U . I n t e n d e d t o b e c a l l e d w i t h
* IR=0 a n d D R =0 .
* /
2019-04-26 19:23:36 +03:00
early_hash_table :
sync / * F o r c e a l l P T E u p d a t e s t o f i n i s h * /
isync
tlbia / * C l e a r a l l T L B e n t r i e s * /
sync / * w a i t f o r t l b i a / t l b i e t o f i n i s h * /
TLBSYNC / * . . . o n a l l C P U s * /
/* Load the SDR1 register (hash table base & size) */
lis r6 , e a r l y _ h a s h - P A G E _ O F F S E T @h
ori r6 , r6 , 3 / * 2 5 6 k B t a b l e * /
mtspr S P R N _ S D R 1 , r6
blr
2005-09-26 10:04:21 +04:00
load_up_mmu :
sync / * F o r c e a l l P T E u p d a t e s t o f i n i s h * /
isync
tlbia / * C l e a r a l l T L B e n t r i e s * /
sync / * w a i t f o r t l b i a / t l b i e t o f i n i s h * /
TLBSYNC / * . . . o n a l l C P U s * /
2020-11-25 10:10:50 +03:00
BEGIN_ M M U _ F T R _ S E C T I O N
2005-09-26 10:04:21 +04:00
/* Load the SDR1 register (hash table base & size) */
lis r6 ,_ S D R 1 @ha
tophys( r6 ,r6 )
lwz r6 ,_ S D R 1 @l(r6)
mtspr S P R N _ S D R 1 ,r6
2020-11-25 10:10:50 +03:00
END_ M M U _ F T R _ S E C T I O N _ I F S E T ( M M U _ F T R _ H P T E _ T A B L E )
2005-10-06 06:49:05 +04:00
2020-09-29 09:48:36 +03:00
/* Load the BAT registers with the values set up by MMU_init. */
2005-09-26 10:04:21 +04:00
lis r3 ,B A T S @ha
addi r3 ,r3 ,B A T S @l
tophys( r3 ,r3 )
LOAD_ B A T ( 0 ,r3 ,r4 ,r5 )
LOAD_ B A T ( 1 ,r3 ,r4 ,r5 )
LOAD_ B A T ( 2 ,r3 ,r4 ,r5 )
LOAD_ B A T ( 3 ,r3 ,r4 ,r5 )
2008-12-18 22:13:32 +03:00
BEGIN_ M M U _ F T R _ S E C T I O N
2006-06-18 02:52:44 +04:00
LOAD_ B A T ( 4 ,r3 ,r4 ,r5 )
LOAD_ B A T ( 5 ,r3 ,r4 ,r5 )
LOAD_ B A T ( 6 ,r3 ,r4 ,r5 )
LOAD_ B A T ( 7 ,r3 ,r4 ,r5 )
2008-12-18 22:13:32 +03:00
END_ M M U _ F T R _ S E C T I O N _ I F S E T ( M M U _ F T R _ U S E _ H I G H _ B A T S )
2005-09-26 10:04:21 +04:00
blr
2020-09-11 13:29:15 +03:00
_ GLOBAL( l o a d _ s e g m e n t _ r e g i s t e r s )
2019-04-26 19:23:36 +03:00
li r0 , N U M _ U S E R _ S E G M E N T S / * l o a d u p u s e r s e g m e n t r e g i s t e r v a l u e s * /
mtctr r0 / * f o r c o n t e x t 0 * /
li r3 , 0 / * K p = 0 , K s = 0 , V S I D = 0 * /
# ifdef C O N F I G _ P P C _ K U E P
oris r3 , r3 , S R _ N X @h /* Set Nx */
# endif
# ifdef C O N F I G _ P P C _ K U A P
oris r3 , r3 , S R _ K S @h /* Set Ks */
# endif
li r4 , 0
3 : mtsrin r3 , r4
addi r3 , r3 , 0 x11 1 / * i n c r e m e n t V S I D * /
addis r4 , r4 , 0 x10 0 0 / * a d d r e s s o f n e x t s e g m e n t * /
bdnz 3 b
li r0 , 1 6 - N U M _ U S E R _ S E G M E N T S / * l o a d u p k e r n e l s e g m e n t r e g i s t e r s * /
mtctr r0 / * f o r c o n t e x t 0 * /
rlwinm r3 , r3 , 0 , ~ S R _ N X / * N x = 0 * /
rlwinm r3 , r3 , 0 , ~ S R _ K S / * K s = 0 * /
oris r3 , r3 , S R _ K P @h /* Kp = 1 */
3 : mtsrin r3 , r4
addi r3 , r3 , 0 x11 1 / * i n c r e m e n t V S I D * /
addis r4 , r4 , 0 x10 0 0 / * a d d r e s s o f n e x t s e g m e n t * /
bdnz 3 b
blr
2005-09-26 10:04:21 +04:00
/ *
* This i s w h e r e t h e m a i n k e r n e l c o d e s t a r t s .
* /
start_here :
/* ptr to current */
lis r2 ,i n i t _ t a s k @h
ori r2 ,r2 ,i n i t _ t a s k @l
/* Set up for using our exception vectors */
/* ptr to phys current thread */
tophys( r4 ,r2 )
addi r4 ,r4 ,T H R E A D / * i n i t t a s k ' s T H R E A D * /
2009-07-15 00:52:54 +04:00
mtspr S P R N _ S P R G _ T H R E A D ,r4
2020-11-25 10:10:50 +03:00
BEGIN_ M M U _ F T R _ S E C T I O N
2019-03-11 11:30:27 +03:00
lis r4 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @h
ori r4 , r4 , ( s w a p p e r _ p g _ d i r - P A G E _ O F F S E T ) @l
2020-11-25 10:10:50 +03:00
rlwinm r4 , r4 , 4 , 0 x f f f f01 f f
mtspr S P R N _ S D R 1 , r4
END_ M M U _ F T R _ S E C T I O N _ I F C L R ( M M U _ F T R _ H P T E _ T A B L E )
2005-09-26 10:04:21 +04:00
/* stack */
lis r1 ,i n i t _ t h r e a d _ u n i o n @ha
addi r1 ,r1 ,i n i t _ t h r e a d _ u n i o n @l
li r0 ,0
stwu r0 ,T H R E A D _ S I Z E - S T A C K _ F R A M E _ O V E R H E A D ( r1 )
/ *
2005-10-06 06:49:05 +04:00
* Do e a r l y p l a t f o r m - s p e c i f i c i n i t i a l i z a t i o n ,
2005-09-26 10:04:21 +04:00
* and s e t u p t h e M M U .
* /
2019-04-26 19:23:34 +03:00
# ifdef C O N F I G _ K A S A N
bl k a s a n _ e a r l y _ i n i t
# endif
2011-07-25 15:29:33 +04:00
li r3 ,0
mr r4 ,r31
2005-09-26 10:04:21 +04:00
bl m a c h i n e _ i n i t
2005-11-11 14:34:43 +03:00
bl _ _ s a v e _ c p u _ s e t u p
2005-09-26 10:04:21 +04:00
bl M M U _ i n i t
2019-04-26 19:23:35 +03:00
bl M M U _ i n i t _ h w _ p a t c h
2005-09-26 10:04:21 +04:00
/ *
* Go b a c k t o r u n n i n g u n m a p p e d s o w e c a n l o a d u p n e w v a l u e s
* for S D R 1 ( h a s h t a b l e p o i n t e r ) a n d t h e s e g m e n t r e g i s t e r s
* and c h a n g e t o u s i n g o u r e x c e p t i o n v e c t o r s .
* /
lis r4 ,2 f @h
ori r4 ,r4 ,2 f @l
tophys( r4 ,r4 )
li r3 ,M S R _ K E R N E L & ~ ( M S R _ I R | M S R _ D R )
2019-12-21 11:32:37 +03:00
.align 4
2005-09-26 10:04:21 +04:00
mtspr S P R N _ S R R 0 ,r4
mtspr S P R N _ S R R 1 ,r3
2020-11-08 19:57:36 +03:00
rfi
2005-09-26 10:04:21 +04:00
/* Load up the kernel context */
2 : bl l o a d _ u p _ m m u
# ifdef C O N F I G _ B D I _ S W I T C H
/ * Add h e l p e r i n f o r m a t i o n f o r t h e A b a t r o n b d i G D B d e b u g g e r .
* We d o t h i s h e r e b e c a u s e w e k n o w t h e m m u i s d i s a b l e d , a n d
* will b e e n a b l e d f o r r e a l i n j u s t a f e w i n s t r u c t i o n s .
* /
lis r5 , a b a t r o n _ p t e p t r s @h
ori r5 , r5 , a b a t r o n _ p t e p t r s @l
2020-08-06 09:01:42 +03:00
stw r5 , 0 x f0 ( 0 ) / * T h i s m u c h m a t c h y o u r A b a t r o n c o n f i g * /
2005-09-26 10:04:21 +04:00
lis r6 , s w a p p e r _ p g _ d i r @h
ori r6 , r6 , s w a p p e r _ p g _ d i r @l
tophys( r5 , r5 )
stw r6 , 0 ( r5 )
# endif / * C O N F I G _ B D I _ S W I T C H * /
/* Now turn on the MMU for real! */
li r4 ,M S R _ K E R N E L
lis r3 ,s t a r t _ k e r n e l @h
ori r3 ,r3 ,s t a r t _ k e r n e l @l
mtspr S P R N _ S R R 0 ,r3
mtspr S P R N _ S R R 1 ,r4
2020-11-08 19:57:36 +03:00
rfi
2005-09-26 10:04:21 +04:00
/ *
2008-12-18 22:13:24 +03:00
* void s w i t c h _ m m u _ c o n t e x t ( s t r u c t m m _ s t r u c t * p r e v , s t r u c t m m _ s t r u c t * n e x t ) ;
*
2005-09-26 10:04:21 +04:00
* Set u p t h e s e g m e n t r e g i s t e r s f o r a n e w c o n t e x t .
* /
2008-12-18 22:13:24 +03:00
_ ENTRY( s w i t c h _ m m u _ c o n t e x t )
lwz r3 ,M M C O N T E X T I D ( r4 )
cmpwi c r0 ,r3 ,0
blt- 4 f
2005-09-26 10:04:21 +04:00
mulli r3 ,r3 ,8 9 7 / * m u l t i p l y c o n t e x t b y s k e w f a c t o r * /
rlwinm r3 ,r3 ,4 ,8 ,2 7 / * V S I D = ( c o n t e x t & 0 x f f f f f ) < < 4 * /
2019-03-11 11:30:35 +03:00
# ifdef C O N F I G _ P P C _ K U E P
oris r3 , r3 , S R _ N X @h /* Set Nx */
2019-03-11 11:30:38 +03:00
# endif
# ifdef C O N F I G _ P P C _ K U A P
oris r3 , r3 , S R _ K S @h /* Set Ks */
2019-03-11 11:30:35 +03:00
# endif
2005-09-26 10:04:21 +04:00
li r0 ,N U M _ U S E R _ S E G M E N T S
mtctr r0
# ifdef C O N F I G _ B D I _ S W I T C H
/ * Context s w i t c h t h e P T E p o i n t e r f o r t h e A b a t r o n B D I 2 0 0 0 .
* The P G D I R i s p a s s e d a s s e c o n d a r g u m e n t .
* /
2020-11-25 10:10:50 +03:00
lwz r4 , M M _ P G D ( r4 )
2019-02-21 13:37:53 +03:00
lis r5 , a b a t r o n _ p t e p t r s @ha
stw r4 , a b a t r o n _ p t e p t r s @l + 0x4(r5)
2020-11-25 10:10:50 +03:00
# endif
BEGIN_ M M U _ F T R _ S E C T I O N
# ifndef C O N F I G _ B D I _ S W I T C H
lwz r4 , M M _ P G D ( r4 )
2005-09-26 10:04:21 +04:00
# endif
2019-02-21 13:37:55 +03:00
tophys( r4 , r4 )
2020-11-25 10:10:50 +03:00
rlwinm r4 , r4 , 4 , 0 x f f f f01 f f
mtspr S P R N _ S D R 1 , r4
END_ M M U _ F T R _ S E C T I O N _ I F C L R ( M M U _ F T R _ H P T E _ T A B L E )
2005-09-26 10:04:21 +04:00
li r4 ,0
isync
3 :
mtsrin r3 ,r4
addi r3 ,r3 ,0 x11 1 / * n e x t V S I D * /
rlwinm r3 ,r3 ,0 ,8 ,3 / * c l e a r o u t a n y o v e r f l o w f r o m V S I D f i e l d * /
addis r4 ,r4 ,0 x10 0 0 / * a d d r e s s o f n e x t s e g m e n t * /
bdnz 3 b
sync
isync
blr
2008-12-18 22:13:24 +03:00
4 : trap
EMIT_ B U G _ E N T R Y 4 b ,_ _ F I L E _ _ ,_ _ L I N E _ _ ,0
blr
2016-01-14 07:33:46 +03:00
EXPORT_ S Y M B O L ( s w i t c h _ m m u _ c o n t e x t )
2005-09-26 10:04:21 +04:00
/ *
* An u n d o c u m e n t e d " f e a t u r e " o f 6 0 4 e r e q u i r e s t h a t t h e v b i t
* be c l e a r e d b e f o r e c h a n g i n g B A T v a l u e s .
*
* Also, n e w e r I B M f i r m w a r e d o e s n o t c l e a r b a t 3 a n d 4 s o
* this m a k e s s u r e i t ' s d o n e .
* - - Cort
* /
clear_bats :
li r10 ,0
mtspr S P R N _ D B A T 0 U ,r10
mtspr S P R N _ D B A T 0 L ,r10
mtspr S P R N _ D B A T 1 U ,r10
mtspr S P R N _ D B A T 1 L ,r10
mtspr S P R N _ D B A T 2 U ,r10
mtspr S P R N _ D B A T 2 L ,r10
mtspr S P R N _ D B A T 3 U ,r10
mtspr S P R N _ D B A T 3 L ,r10
mtspr S P R N _ I B A T 0 U ,r10
mtspr S P R N _ I B A T 0 L ,r10
mtspr S P R N _ I B A T 1 U ,r10
mtspr S P R N _ I B A T 1 L ,r10
mtspr S P R N _ I B A T 2 U ,r10
mtspr S P R N _ I B A T 2 L ,r10
mtspr S P R N _ I B A T 3 U ,r10
mtspr S P R N _ I B A T 3 L ,r10
2008-12-18 22:13:32 +03:00
BEGIN_ M M U _ F T R _ S E C T I O N
2005-09-26 10:04:21 +04:00
/ * Here' s a t w e a k : a t t h i s p o i n t , C P U s e t u p h a v e
* not b e e n c a l l e d y e t , s o H I G H _ B A T _ E N m a y n o t b e
* set i n H I D 0 f o r t h e 7 4 5 x p r o c e s s o r s . H o w e v e r , i t
* seems t h a t d o e s n ' t a f f e c t o u r a b i l i t y t o a c t u a l l y
* write t o t h e s e S P R s .
* /
mtspr S P R N _ D B A T 4 U ,r10
mtspr S P R N _ D B A T 4 L ,r10
mtspr S P R N _ D B A T 5 U ,r10
mtspr S P R N _ D B A T 5 L ,r10
mtspr S P R N _ D B A T 6 U ,r10
mtspr S P R N _ D B A T 6 L ,r10
mtspr S P R N _ D B A T 7 U ,r10
mtspr S P R N _ D B A T 7 L ,r10
mtspr S P R N _ I B A T 4 U ,r10
mtspr S P R N _ I B A T 4 L ,r10
mtspr S P R N _ I B A T 5 U ,r10
mtspr S P R N _ I B A T 5 L ,r10
mtspr S P R N _ I B A T 6 U ,r10
mtspr S P R N _ I B A T 6 L ,r10
mtspr S P R N _ I B A T 7 U ,r10
mtspr S P R N _ I B A T 7 L ,r10
2008-12-18 22:13:32 +03:00
END_ M M U _ F T R _ S E C T I O N _ I F S E T ( M M U _ F T R _ U S E _ H I G H _ B A T S )
2005-09-26 10:04:21 +04:00
blr
2019-02-21 22:08:48 +03:00
_ ENTRY( u p d a t e _ b a t s )
lis r4 , 1 f @h
ori r4 , r4 , 1 f @l
tophys( r4 , r4 )
mfmsr r6
mflr r7
li r3 , M S R _ K E R N E L & ~ ( M S R _ I R | M S R _ D R )
rlwinm r0 , r6 , 0 , ~ M S R _ R I
rlwinm r0 , r0 , 0 , ~ M S R _ E E
mtmsr r0
2019-12-21 11:32:37 +03:00
.align 4
2019-02-21 22:08:48 +03:00
mtspr S P R N _ S R R 0 , r4
mtspr S P R N _ S R R 1 , r3
2020-11-08 19:57:36 +03:00
rfi
2019-02-21 22:08:48 +03:00
1 : bl c l e a r _ b a t s
lis r3 , B A T S @ha
addi r3 , r3 , B A T S @l
tophys( r3 , r3 )
LOAD_ B A T ( 0 , r3 , r4 , r5 )
LOAD_ B A T ( 1 , r3 , r4 , r5 )
LOAD_ B A T ( 2 , r3 , r4 , r5 )
LOAD_ B A T ( 3 , r3 , r4 , r5 )
BEGIN_ M M U _ F T R _ S E C T I O N
LOAD_ B A T ( 4 , r3 , r4 , r5 )
LOAD_ B A T ( 5 , r3 , r4 , r5 )
LOAD_ B A T ( 6 , r3 , r4 , r5 )
LOAD_ B A T ( 7 , r3 , r4 , r5 )
END_ M M U _ F T R _ S E C T I O N _ I F S E T ( M M U _ F T R _ U S E _ H I G H _ B A T S )
li r3 , M S R _ K E R N E L & ~ ( M S R _ I R | M S R _ D R | M S R _ R I )
mtmsr r3
mtspr S P R N _ S R R 0 , r7
mtspr S P R N _ S R R 1 , r6
2020-11-08 19:57:36 +03:00
rfi
2019-02-21 22:08:48 +03:00
2005-09-26 10:04:21 +04:00
flush_tlbs :
lis r10 , 0 x40
1 : addic. r10 , r10 , - 0 x10 0 0
tlbie r10
2008-08-14 17:11:54 +04:00
bgt 1 b
2005-09-26 10:04:21 +04:00
sync
blr
mmu_off :
addi r4 , r3 , _ _ a f t e r _ m m u _ o f f - _ s t a r t
mfmsr r3
andi. r0 ,r3 ,M S R _ D R | M S R _ I R / * M M U e n a b l e d ? * /
beqlr
andc r3 ,r3 ,r0
2019-12-21 11:32:37 +03:00
.align 4
2005-09-26 10:04:21 +04:00
mtspr S P R N _ S R R 0 ,r4
mtspr S P R N _ S R R 1 ,r3
sync
2020-11-08 19:57:36 +03:00
rfi
2005-09-26 10:04:21 +04:00
2020-09-29 09:48:36 +03:00
/* We use one BAT to map up to 256M of RAM at _PAGE_OFFSET */
2005-09-26 10:04:21 +04:00
initial_bats :
2008-12-17 13:09:13 +03:00
lis r11 ,P A G E _ O F F S E T @h
2019-08-26 18:52:17 +03:00
tophys( r8 ,r11 )
2005-09-26 10:04:21 +04:00
# ifdef C O N F I G _ S M P
ori r8 ,r8 ,0 x12 / * R / W a c c e s s , M =1 * /
# else
ori r8 ,r8 ,2 / * R / W a c c e s s * /
# endif / * C O N F I G _ S M P * /
ori r11 ,r11 ,B L _ 2 5 6 M < < 2 | 0 x2 / * s e t u p B A T r e g i s t e r s f o r 6 0 4 * /
2020-09-29 09:48:36 +03:00
mtspr S P R N _ D B A T 0 L ,r8 / * N . B . 6 x x h a v e v a l i d * /
2005-09-26 10:04:21 +04:00
mtspr S P R N _ D B A T 0 U ,r11 / * b i t i n u p p e r B A T r e g i s t e r * /
mtspr S P R N _ I B A T 0 L ,r8
mtspr S P R N _ I B A T 0 U ,r11
isync
blr
2007-06-13 08:52:54 +04:00
# ifdef C O N F I G _ B O O T X _ T E X T
2005-11-23 09:57:25 +03:00
setup_disp_bat :
/ *
* setup t h e d i s p l a y b a t p r e p a r e d f o r u s i n p r o m . c
* /
mflr r8
bl r e l o c _ o f f s e t
mtlr r8
addis r8 ,r3 ,d i s p _ B A T @ha
addi r8 ,r8 ,d i s p _ B A T @l
cmpwi c r0 ,r8 ,0
beqlr
lwz r11 ,0 ( r8 )
lwz r8 ,4 ( r8 )
mtspr S P R N _ D B A T 3 L ,r8
mtspr S P R N _ D B A T 3 U ,r11
blr
2007-06-13 08:52:54 +04:00
# endif / * C O N F I G _ B O O T X _ T E X T * /
2005-11-23 09:57:25 +03:00
2007-07-16 20:43:43 +04:00
# ifdef C O N F I G _ P P C _ E A R L Y _ D E B U G _ C P M
setup_cpm_bat :
lis r8 , 0 x f00 0
ori r8 , r8 , 0 x00 2 a
mtspr S P R N _ D B A T 1 L , r8
lis r11 , 0 x f00 0
ori r11 , r11 , ( B L _ 1 M < < 2 ) | 2
mtspr S P R N _ D B A T 1 U , r11
blr
# endif
2009-12-12 09:31:51 +03:00
# ifdef C O N F I G _ P P C _ E A R L Y _ D E B U G _ U S B G E C K O
setup_usbgecko_bat :
/* prepare a BAT for early io */
# if d e f i n e d ( C O N F I G _ G A M E C U B E )
lis r8 , 0 x0 c00
# elif d e f i n e d ( C O N F I G _ W I I )
lis r8 , 0 x0 d00
# else
# error I n v a l i d p l a t f o r m f o r U S B G e c k o b a s e d e a r l y d e b u g g i n g .
# endif
/ *
* The v i r t u a l a d d r e s s u s e d m u s t m a t c h t h e v i r t u a l a d d r e s s
* associated t o t h e f i x m a p e n t r y F I X _ E A R L Y _ D E B U G _ B A S E .
* /
lis r11 , 0 x f f f e / * t o p 1 2 8 K * /
ori r8 , r8 , 0 x00 2 a / * u n c a c h e d , g u a r d e d ,r w * /
ori r11 , r11 , 0 x2 / * 1 2 8 K , V s =1 , V p =0 * /
mtspr S P R N _ D B A T 1 L , r8
mtspr S P R N _ D B A T 1 U , r11
blr
# endif
2005-09-26 10:04:21 +04:00
# ifdef C O N F I G _ 8 2 6 0
/ * Jump i n t o t h e s y s t e m r e s e t f o r t h e r o m .
* We f i r s t d i s a b l e t h e M M U , a n d t h e n j u m p t o t h e R O M r e s e t a d d r e s s .
*
* r3 i s t h e b o a r d i n f o s t r u c t u r e , r4 i s t h e l o c a t i o n f o r s t a r t i n g .
* I u s e t h i s f o r b u i l d i n g a s m a l l k e r n e l t h a t c a n l o a d o t h e r k e r n e l s ,
* rather t h a n t r y i n g t o w r i t e o r r e l y o n a r o m m o n i t o r t h a t c a n t f t p l o a d .
* /
.globl m8260_gorom
m8260_gorom :
mfmsr r0
rlwinm r0 ,r0 ,0 ,1 7 ,1 5 / * c l e a r M S R _ E E i n r0 * /
sync
mtmsr r0
sync
mfspr r11 , S P R N _ H I D 0
lis r10 , 0
ori r10 ,r10 ,H I D 0 _ I C E | H I D 0 _ D C E
andc r11 , r11 , r10
mtspr S P R N _ H I D 0 , r11
isync
li r5 , M S R _ M E | M S R _ R I
lis r6 ,2 f @h
addis r6 ,r6 ,- K E R N E L B A S E @h
ori r6 ,r6 ,2 f @l
mtspr S P R N _ S R R 0 ,r6
mtspr S P R N _ S R R 1 ,r5
isync
sync
rfi
2 :
mtlr r4
blr
# endif
/ *
* We p u t a f e w t h i n g s h e r e t h a t h a v e t o b e p a g e - a l i g n e d .
* This s t u f f g o e s a t t h e b e g i n n i n g o f t h e d a t a s e g m e n t ,
* which i s p a g e - a l i g n e d .
* /
.data
.globl sdata
sdata :
.globl empty_zero_page
empty_zero_page :
.space 4096
2016-01-14 07:33:46 +03:00
EXPORT_ S Y M B O L ( e m p t y _ z e r o _ p a g e )
2005-09-26 10:04:21 +04:00
.globl swapper_pg_dir
swapper_pg_dir :
2007-12-06 22:11:04 +03:00
.space PGD_TABLE_SIZE
2005-09-26 10:04:21 +04:00
/ * Room f o r t w o P T E p o i n t e r s , u s u a l l y t h e k e r n e l a n d c u r r e n t u s e r p o i n t e r s
* to t h e i r r e s p e c t i v e r o o t p a g e t a b l e .
* /
abatron_pteptrs :
.space 8
