2009-06-03 01:17:38 +04:00
/ *
* This f i l e c o n t a i n s t h e 6 4 - b i t " s e r v e r " P o w e r P C v a r i a n t
* of t h e l o w l e v e l e x c e p t i o n h a n d l i n g i n c l u d i n g e x c e p t i o n
* vectors, e x c e p t i o n r e t u r n , p a r t o f t h e s l b a n d s t a b
* handling a n d o t h e r f i x e d o f f s e t s p e c i f i c t h i n g s .
*
* This f i l e i s m e a n t t o b e #i n c l u d e d f r o m h e a d _ 64 . S d u e t o
* position d e p e n d a n t a s s e m b l y .
*
* Most o f t h i s o r i g i n a t e s f r o m h e a d _ 6 4 . S a n d t h u s h a s t h e s a m e
* copyright h i s t o r y .
*
* /
2009-07-15 00:52:52 +04:00
# include < a s m / e x c e p t i o n - 6 4 s . h >
2009-06-03 01:17:38 +04:00
/ *
* We l a y o u t p h y s i c a l m e m o r y a s f o l l o w s :
* 0 x0 0 0 0 - 0 x00 f f : S e c o n d a r y p r o c e s s o r s p i n c o d e
* 0 x0 1 0 0 - 0 x2 f f f : p S e r i e s I n t e r r u p t p r o l o g s
* 0 x3 0 0 0 - 0 x5 f f f : i n t e r r u p t s u p p o r t , i S e r i e s a n d c o m m o n i n t e r r u p t p r o l o g s
* 0 x6 0 0 0 - 0 x6 f f f : I n i t i a l ( C P U 0 ) s e g m e n t t a b l e
* 0 x7 0 0 0 - 0 x7 f f f : F W N M I d a t a a r e a
* 0 x8 0 0 0 - : E a r l y i n i t a n d s u p p o r t c o d e
* /
/ *
* This i s t h e s t a r t o f t h e i n t e r r u p t h a n d l e r s f o r p S e r i e s
* This c o d e r u n s w i t h r e l o c a t i o n o f f .
* Code f r o m h e r e t o _ _ e n d _ i n t e r r u p t s g e t s c o p i e d d o w n t o r e a l
* address 0 x10 0 w h e n w e a r e r u n n i n g a r e l o c a t a b l e k e r n e l .
* Therefore a n y r e l a t i v e b r a n c h e s i n t h i s s e c t i o n m u s t o n l y
* branch t o l a b e l s i n t h i s s e c t i o n .
* /
. = 0 x1 0 0
.globl __start_interrupts
__start_interrupts :
STD_ E X C E P T I O N _ P S E R I E S ( 0 x10 0 , s y s t e m _ r e s e t )
. = 0 x2 0 0
_machine_check_pSeries :
HMT_ M E D I U M
2009-10-30 08:47:17 +03:00
DO_ K V M 0 x20 0
2009-07-15 00:52:54 +04:00
mtspr S P R N _ S P R G _ S C R A T C H 0 ,r13 / * s a v e r13 * /
2009-06-03 01:17:38 +04:00
EXCEPTION_ P R O L O G _ P S E R I E S ( P A C A _ E X M C , m a c h i n e _ c h e c k _ c o m m o n )
. = 0 x3 0 0
.globl data_access_pSeries
data_access_pSeries :
HMT_ M E D I U M
2009-10-30 08:47:17 +03:00
DO_ K V M 0 x30 0
2009-07-15 00:52:54 +04:00
mtspr S P R N _ S P R G _ S C R A T C H 0 ,r13
2009-06-03 01:17:38 +04:00
BEGIN_ F T R _ S E C T I O N
2009-07-16 23:36:57 +04:00
mfspr r13 ,S P R N _ S P R G _ P A C A
std r9 ,P A C A _ E X S L B + E X _ R 9 ( r13 )
std r10 ,P A C A _ E X S L B + E X _ R 1 0 ( r13 )
mfspr r10 ,S P R N _ D A R
mfspr r9 ,S P R N _ D S I S R
srdi r10 ,r10 ,6 0
rlwimi r10 ,r9 ,1 6 ,0 x20
mfcr r9
cmpwi r10 ,0 x2 c
2009-06-03 01:17:38 +04:00
beq d o _ s t a b _ b o l t e d _ p S e r i e s
2009-07-16 23:36:57 +04:00
ld r10 ,P A C A _ E X S L B + E X _ R 1 0 ( r13 )
std r11 ,P A C A _ E X G E N + E X _ R 1 1 ( r13 )
ld r11 ,P A C A _ E X S L B + E X _ R 9 ( r13 )
std r12 ,P A C A _ E X G E N + E X _ R 1 2 ( r13 )
mfspr r12 ,S P R N _ S P R G _ S C R A T C H 0
std r10 ,P A C A _ E X G E N + E X _ R 1 0 ( r13 )
std r11 ,P A C A _ E X G E N + E X _ R 9 ( r13 )
std r12 ,P A C A _ E X G E N + E X _ R 1 3 ( r13 )
EXCEPTION_ P R O L O G _ P S E R I E S _ 1 ( d a t a _ a c c e s s _ c o m m o n )
FTR_ S E C T I O N _ E L S E
2009-06-03 01:17:38 +04:00
EXCEPTION_ P R O L O G _ P S E R I E S ( P A C A _ E X G E N , d a t a _ a c c e s s _ c o m m o n )
2009-07-16 23:36:57 +04:00
ALT_ F T R _ S E C T I O N _ E N D _ I F C L R ( C P U _ F T R _ S L B )
2009-06-03 01:17:38 +04:00
. = 0 x3 8 0
.globl data_access_slb_pSeries
data_access_slb_pSeries :
HMT_ M E D I U M
2009-10-30 08:47:17 +03:00
DO_ K V M 0 x38 0
2009-07-15 00:52:54 +04:00
mtspr S P R N _ S P R G _ S C R A T C H 0 ,r13
mfspr r13 ,S P R N _ S P R G _ P A C A / * g e t p a c a a d d r e s s i n t o r13 * /
2009-06-03 01:17:38 +04:00
std r3 ,P A C A _ E X S L B + E X _ R 3 ( r13 )
mfspr r3 ,S P R N _ D A R
std r9 ,P A C A _ E X S L B + E X _ R 9 ( r13 ) / * s a v e r9 - r12 * /
mfcr r9
# ifdef _ _ D I S A B L E D _ _
/* Keep that around for when we re-implement dynamic VSIDs */
cmpdi r3 ,0
bge s l b _ m i s s _ u s e r _ p s e r i e s
# endif / * _ _ D I S A B L E D _ _ * /
std r10 ,P A C A _ E X S L B + E X _ R 1 0 ( r13 )
std r11 ,P A C A _ E X S L B + E X _ R 1 1 ( r13 )
std r12 ,P A C A _ E X S L B + E X _ R 1 2 ( r13 )
2009-07-15 00:52:54 +04:00
mfspr r10 ,S P R N _ S P R G _ S C R A T C H 0
2009-06-03 01:17:38 +04:00
std r10 ,P A C A _ E X S L B + E X _ R 1 3 ( r13 )
mfspr r12 ,S P R N _ S R R 1 / * a n d S R R 1 * /
# ifndef C O N F I G _ R E L O C A T A B L E
b . s l b _ m i s s _ r e a l m o d e
# else
/ *
* We c a n ' t j u s t u s e a d i r e c t b r a n c h t o . s l b _ m i s s _ r e a l m o d e
* because t h e d i s t a n c e f r o m h e r e t o t h e r e d e p e n d s o n w h e r e
* the k e r n e l e n d s u p b e i n g p u t .
* /
mfctr r11
ld r10 ,P A C A K B A S E ( r13 )
LOAD_ H A N D L E R ( r10 , . s l b _ m i s s _ r e a l m o d e )
mtctr r10
bctr
# endif
STD_ E X C E P T I O N _ P S E R I E S ( 0 x40 0 , i n s t r u c t i o n _ a c c e s s )
. = 0 x4 8 0
.globl instruction_access_slb_pSeries
instruction_access_slb_pSeries :
HMT_ M E D I U M
2009-10-30 08:47:17 +03:00
DO_ K V M 0 x48 0
2009-07-15 00:52:54 +04:00
mtspr S P R N _ S P R G _ S C R A T C H 0 ,r13
mfspr r13 ,S P R N _ S P R G _ P A C A / * g e t p a c a a d d r e s s i n t o r13 * /
2009-06-03 01:17:38 +04:00
std r3 ,P A C A _ E X S L B + E X _ R 3 ( r13 )
mfspr r3 ,S P R N _ S R R 0 / * S R R 0 i s f a u l t i n g a d d r e s s * /
std r9 ,P A C A _ E X S L B + E X _ R 9 ( r13 ) / * s a v e r9 - r12 * /
mfcr r9
# ifdef _ _ D I S A B L E D _ _
/* Keep that around for when we re-implement dynamic VSIDs */
cmpdi r3 ,0
bge s l b _ m i s s _ u s e r _ p s e r i e s
# endif / * _ _ D I S A B L E D _ _ * /
std r10 ,P A C A _ E X S L B + E X _ R 1 0 ( r13 )
std r11 ,P A C A _ E X S L B + E X _ R 1 1 ( r13 )
std r12 ,P A C A _ E X S L B + E X _ R 1 2 ( r13 )
2009-07-15 00:52:54 +04:00
mfspr r10 ,S P R N _ S P R G _ S C R A T C H 0
2009-06-03 01:17:38 +04:00
std r10 ,P A C A _ E X S L B + E X _ R 1 3 ( r13 )
mfspr r12 ,S P R N _ S R R 1 / * a n d S R R 1 * /
# ifndef C O N F I G _ R E L O C A T A B L E
b . s l b _ m i s s _ r e a l m o d e
# else
mfctr r11
ld r10 ,P A C A K B A S E ( r13 )
LOAD_ H A N D L E R ( r10 , . s l b _ m i s s _ r e a l m o d e )
mtctr r10
bctr
# endif
MASKABLE_ E X C E P T I O N _ P S E R I E S ( 0 x50 0 , h a r d w a r e _ i n t e r r u p t )
STD_ E X C E P T I O N _ P S E R I E S ( 0 x60 0 , a l i g n m e n t )
STD_ E X C E P T I O N _ P S E R I E S ( 0 x70 0 , p r o g r a m _ c h e c k )
STD_ E X C E P T I O N _ P S E R I E S ( 0 x80 0 , f p _ u n a v a i l a b l e )
MASKABLE_ E X C E P T I O N _ P S E R I E S ( 0 x90 0 , d e c r e m e n t e r )
STD_ E X C E P T I O N _ P S E R I E S ( 0 x a00 , t r a p _ 0 a )
STD_ E X C E P T I O N _ P S E R I E S ( 0 x b00 , t r a p _ 0 b )
. = 0 xc0 0
.globl system_call_pSeries
system_call_pSeries :
HMT_ M E D I U M
2009-10-30 08:47:17 +03:00
DO_ K V M 0 x c00
2009-06-03 01:17:38 +04:00
BEGIN_ F T R _ S E C T I O N
cmpdi r0 ,0 x1 e b e
beq- 1 f
END_ F T R _ S E C T I O N _ I F S E T ( C P U _ F T R _ R E A L _ L E )
mr r9 ,r13
2009-07-15 00:52:54 +04:00
mfspr r13 ,S P R N _ S P R G _ P A C A
2009-06-03 01:17:38 +04:00
mfspr r11 ,S P R N _ S R R 0
ld r12 ,P A C A K B A S E ( r13 )
ld r10 ,P A C A K M S R ( r13 )
LOAD_ H A N D L E R ( r12 , s y s t e m _ c a l l _ e n t r y )
mtspr S P R N _ S R R 0 ,r12
mfspr r12 ,S P R N _ S R R 1
mtspr S P R N _ S R R 1 ,r10
rfid
b . / * p r e v e n t s p e c u l a t i v e e x e c u t i o n * /
/* Fast LE/BE switch system call */
1 : mfspr r12 ,S P R N _ S R R 1
xori r12 ,r12 ,M S R _ L E
mtspr S P R N _ S R R 1 ,r12
rfid / * r e t u r n t o u s e r s p a c e * /
b .
STD_ E X C E P T I O N _ P S E R I E S ( 0 x d00 , s i n g l e _ s t e p )
STD_ E X C E P T I O N _ P S E R I E S ( 0 x e 0 0 , t r a p _ 0 e )
/ * We n e e d t o d e a l w i t h t h e A l t i v e c u n a v a i l a b l e e x c e p t i o n
* here w h i c h i s a t 0 x f20 , t h u s i n t h e m i d d l e o f t h e
* prolog c o d e o f t h e P e r f o r m a n c e M o n i t o r o n e . A l i t t l e
* trickery i s t h u s n e c e s s a r y
* /
2009-10-18 05:24:06 +04:00
performance_monitor_pSeries_1 :
2009-06-03 01:17:38 +04:00
. = 0 xf0 0
2009-10-30 08:47:17 +03:00
DO_ K V M 0 x f00
2009-06-03 01:17:38 +04:00
b p e r f o r m a n c e _ m o n i t o r _ p S e r i e s
2009-10-18 05:24:06 +04:00
altivec_unavailable_pSeries_1 :
2009-06-03 01:17:38 +04:00
. = 0 xf2 0
2009-10-30 08:47:17 +03:00
DO_ K V M 0 x f20
2009-06-03 01:17:38 +04:00
b a l t i v e c _ u n a v a i l a b l e _ p S e r i e s
2009-10-18 05:24:06 +04:00
vsx_unavailable_pSeries_1 :
2009-06-03 01:17:38 +04:00
. = 0 xf4 0
2009-10-30 08:47:17 +03:00
DO_ K V M 0 x f40
2009-06-03 01:17:38 +04:00
b v s x _ u n a v a i l a b l e _ p S e r i e s
# ifdef C O N F I G _ C B E _ R A S
HSTD_ E X C E P T I O N _ P S E R I E S ( 0 x12 0 0 , c b e _ s y s t e m _ e r r o r )
# endif / * C O N F I G _ C B E _ R A S * /
STD_ E X C E P T I O N _ P S E R I E S ( 0 x13 0 0 , i n s t r u c t i o n _ b r e a k p o i n t )
# ifdef C O N F I G _ C B E _ R A S
HSTD_ E X C E P T I O N _ P S E R I E S ( 0 x16 0 0 , c b e _ m a i n t e n a n c e )
# endif / * C O N F I G _ C B E _ R A S * /
STD_ E X C E P T I O N _ P S E R I E S ( 0 x17 0 0 , a l t i v e c _ a s s i s t )
# ifdef C O N F I G _ C B E _ R A S
HSTD_ E X C E P T I O N _ P S E R I E S ( 0 x18 0 0 , c b e _ t h e r m a l )
# endif / * C O N F I G _ C B E _ R A S * /
. = 0 x3 0 0 0
/*** pSeries interrupt support ***/
/* moved from 0xf00 */
STD_ E X C E P T I O N _ P S E R I E S ( . , p e r f o r m a n c e _ m o n i t o r )
STD_ E X C E P T I O N _ P S E R I E S ( . , a l t i v e c _ u n a v a i l a b l e )
STD_ E X C E P T I O N _ P S E R I E S ( . , v s x _ u n a v a i l a b l e )
/ *
* An i n t e r r u p t c a m e i n w h i l e s o f t - d i s a b l e d ; clear EE in SRR1,
* clear p a c a - > h a r d _ e n a b l e d a n d r e t u r n .
* /
masked_interrupt :
stb r10 ,P A C A H A R D I R Q E N ( r13 )
mtcrf 0 x80 ,r9
ld r9 ,P A C A _ E X G E N + E X _ R 9 ( r13 )
mfspr r10 ,S P R N _ S R R 1
rldicl r10 ,r10 ,4 8 ,1 / * c l e a r M S R _ E E * /
rotldi r10 ,r10 ,1 6
mtspr S P R N _ S R R 1 ,r10
ld r10 ,P A C A _ E X G E N + E X _ R 1 0 ( r13 )
2009-07-15 00:52:54 +04:00
mfspr r13 ,S P R N _ S P R G _ S C R A T C H 0
2009-06-03 01:17:38 +04:00
rfid
b .
.align 7
do_stab_bolted_pSeries :
2009-07-16 23:36:57 +04:00
std r11 ,P A C A _ E X S L B + E X _ R 1 1 ( r13 )
std r12 ,P A C A _ E X S L B + E X _ R 1 2 ( r13 )
mfspr r10 ,S P R N _ S P R G _ S C R A T C H 0
std r10 ,P A C A _ E X S L B + E X _ R 1 3 ( r13 )
EXCEPTION_ P R O L O G _ P S E R I E S _ 1 ( . d o _ s t a b _ b o l t e d )
2009-06-03 01:17:38 +04:00
# ifdef C O N F I G _ P P C _ P S E R I E S
/ *
* Vectors f o r t h e F W N M I o p t i o n . S h a r e c o m m o n c o d e .
* /
.globl system_reset_fwnmi
.align 7
system_reset_fwnmi :
HMT_ M E D I U M
2009-07-15 00:52:54 +04:00
mtspr S P R N _ S P R G _ S C R A T C H 0 ,r13 / * s a v e r13 * /
2009-06-03 01:17:38 +04:00
EXCEPTION_ P R O L O G _ P S E R I E S ( P A C A _ E X G E N , s y s t e m _ r e s e t _ c o m m o n )
.globl machine_check_fwnmi
.align 7
machine_check_fwnmi :
HMT_ M E D I U M
2009-07-15 00:52:54 +04:00
mtspr S P R N _ S P R G _ S C R A T C H 0 ,r13 / * s a v e r13 * /
2009-06-03 01:17:38 +04:00
EXCEPTION_ P R O L O G _ P S E R I E S ( P A C A _ E X M C , m a c h i n e _ c h e c k _ c o m m o n )
# endif / * C O N F I G _ P P C _ P S E R I E S * /
# ifdef _ _ D I S A B L E D _ _
/ *
* This i s u s e d f o r w h e n t h e S L B m i s s h a n d l e r h a s t o g o v i r t u a l ,
* which d o e s n ' t h a p p e n f o r n o w a n y m o r e b u t w i l l o n c e w e r e - i m p l e m e n t
* dynamic V S I D s f o r s h a r e d p a g e t a b l e s
* /
slb_miss_user_pseries :
std r10 ,P A C A _ E X G E N + E X _ R 1 0 ( r13 )
std r11 ,P A C A _ E X G E N + E X _ R 1 1 ( r13 )
std r12 ,P A C A _ E X G E N + E X _ R 1 2 ( r13 )
2009-07-15 00:52:54 +04:00
mfspr r10 ,S P R G _ S C R A T C H 0
2009-06-03 01:17:38 +04:00
ld r11 ,P A C A _ E X S L B + E X _ R 9 ( r13 )
ld r12 ,P A C A _ E X S L B + E X _ R 3 ( r13 )
std r10 ,P A C A _ E X G E N + E X _ R 1 3 ( r13 )
std r11 ,P A C A _ E X G E N + E X _ R 9 ( r13 )
std r12 ,P A C A _ E X G E N + E X _ R 3 ( r13 )
clrrdi r12 ,r13 ,3 2
mfmsr r10
mfspr r11 ,S R R 0 / * s a v e S R R 0 * /
ori r12 ,r12 ,s l b _ m i s s _ u s e r _ c o m m o n @l /* virt addr of handler */
ori r10 ,r10 ,M S R _ I R | M S R _ D R | M S R _ R I
mtspr S R R 0 ,r12
mfspr r12 ,S R R 1 / * a n d S R R 1 * /
mtspr S R R 1 ,r10
rfid
b . / * p r e v e n t s p e c . e x e c u t i o n * /
# endif / * _ _ D I S A B L E D _ _ * /
.align 7
.globl __end_interrupts
__end_interrupts :
/ *
* Code f r o m h e r e d o w n t o _ _ e n d _ h a n d l e r s i s i n v o k e d f r o m t h e
* exception p r o l o g s a b o v e . B e c a u s e t h e p r o l o g s a s s e m b l e t h e
* addresses o f t h e s e h a n d l e r s u s i n g t h e L O A D _ H A N D L E R m a c r o ,
* which u s e s a n a d d i i n s t r u c t i o n , t h e s e h a n d l e r s m u s t b e i n
* the f i r s t 3 2 k o f t h e k e r n e l i m a g e .
* /
/*** Common interrupt handlers ***/
STD_ E X C E P T I O N _ C O M M O N ( 0 x10 0 , s y s t e m _ r e s e t , . s y s t e m _ r e s e t _ e x c e p t i o n )
/ *
* Machine c h e c k i s d i f f e r e n t b e c a u s e w e u s e a d i f f e r e n t
* save a r e a : P A C A _ E X M C i n s t e a d o f P A C A _ E X G E N .
* /
.align 7
.globl machine_check_common
machine_check_common :
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x20 0 , P A C A _ E X M C )
FINISH_ N A P
DISABLE_ I N T S
bl . s a v e _ n v g p r s
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
bl . m a c h i n e _ c h e c k _ e x c e p t i o n
b . r e t _ f r o m _ e x c e p t
STD_ E X C E P T I O N _ C O M M O N _ L I T E ( 0 x90 0 , d e c r e m e n t e r , . t i m e r _ i n t e r r u p t )
STD_ E X C E P T I O N _ C O M M O N ( 0 x a00 , t r a p _ 0 a , . u n k n o w n _ e x c e p t i o n )
STD_ E X C E P T I O N _ C O M M O N ( 0 x b00 , t r a p _ 0 b , . u n k n o w n _ e x c e p t i o n )
STD_ E X C E P T I O N _ C O M M O N ( 0 x d00 , s i n g l e _ s t e p , . s i n g l e _ s t e p _ e x c e p t i o n )
STD_ E X C E P T I O N _ C O M M O N ( 0 x e 0 0 , t r a p _ 0 e , . u n k n o w n _ e x c e p t i o n )
STD_ E X C E P T I O N _ C O M M O N _ I D L E ( 0 x f00 , p e r f o r m a n c e _ m o n i t o r , . p e r f o r m a n c e _ m o n i t o r _ e x c e p t i o n )
STD_ E X C E P T I O N _ C O M M O N ( 0 x13 0 0 , i n s t r u c t i o n _ b r e a k p o i n t , . i n s t r u c t i o n _ b r e a k p o i n t _ e x c e p t i o n )
# ifdef C O N F I G _ A L T I V E C
STD_ E X C E P T I O N _ C O M M O N ( 0 x17 0 0 , a l t i v e c _ a s s i s t , . a l t i v e c _ a s s i s t _ e x c e p t i o n )
# else
STD_ E X C E P T I O N _ C O M M O N ( 0 x17 0 0 , a l t i v e c _ a s s i s t , . u n k n o w n _ e x c e p t i o n )
# endif
# ifdef C O N F I G _ C B E _ R A S
STD_ E X C E P T I O N _ C O M M O N ( 0 x12 0 0 , c b e _ s y s t e m _ e r r o r , . c b e _ s y s t e m _ e r r o r _ e x c e p t i o n )
STD_ E X C E P T I O N _ C O M M O N ( 0 x16 0 0 , c b e _ m a i n t e n a n c e , . c b e _ m a i n t e n a n c e _ e x c e p t i o n )
STD_ E X C E P T I O N _ C O M M O N ( 0 x18 0 0 , c b e _ t h e r m a l , . c b e _ t h e r m a l _ e x c e p t i o n )
# endif / * C O N F I G _ C B E _ R A S * /
.align 7
system_call_entry :
b s y s t e m _ c a l l _ c o m m o n
/ *
* Here w e h a v e d e t e c t e d t h a t t h e k e r n e l s t a c k p o i n t e r i s b a d .
* R9 c o n t a i n s t h e s a v e d C R , r13 p o i n t s t o t h e p a c a ,
* r1 0 c o n t a i n s t h e ( b a d ) k e r n e l s t a c k p o i n t e r ,
* r1 1 a n d r12 c o n t a i n t h e s a v e d S R R 0 a n d S R R 1 .
* We s w i t c h t o u s i n g a n e m e r g e n c y s t a c k , s a v e t h e r e g i s t e r s t h e r e ,
* and c a l l k e r n e l _ b a d _ s t a c k ( ) , w h i c h p a n i c s .
* /
bad_stack :
ld r1 ,P A C A E M E R G S P ( r13 )
subi r1 ,r1 ,6 4 + I N T _ F R A M E _ S I Z E
std r9 ,_ C C R ( r1 )
std r10 ,G P R 1 ( r1 )
std r11 ,_ N I P ( r1 )
std r12 ,_ M S R ( r1 )
mfspr r11 ,S P R N _ D A R
mfspr r12 ,S P R N _ D S I S R
std r11 ,_ D A R ( r1 )
std r12 ,_ D S I S R ( r1 )
mflr r10
mfctr r11
mfxer r12
std r10 ,_ L I N K ( r1 )
std r11 ,_ C T R ( r1 )
std r12 ,_ X E R ( r1 )
SAVE_ G P R ( 0 ,r1 )
SAVE_ G P R ( 2 ,r1 )
SAVE_ 4 G P R S ( 3 ,r1 )
SAVE_ 2 G P R S ( 7 ,r1 )
SAVE_ 1 0 G P R S ( 1 2 ,r1 )
SAVE_ 1 0 G P R S ( 2 2 ,r1 )
lhz r12 ,P A C A _ T R A P _ S A V E ( r13 )
std r12 ,_ T R A P ( r1 )
addi r11 ,r1 ,I N T _ F R A M E _ S I Z E
std r11 ,0 ( r1 )
li r12 ,0
std r12 ,0 ( r11 )
ld r2 ,P A C A T O C ( r13 )
1 : addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
bl . k e r n e l _ b a d _ s t a c k
b 1 b
/ *
* Here r13 p o i n t s t o t h e p a c a , r9 c o n t a i n s t h e s a v e d C R ,
* SRR0 a n d S R R 1 a r e s a v e d i n r11 a n d r12 ,
* r9 - r13 a r e s a v e d i n p a c a - > e x g e n .
* /
.align 7
.globl data_access_common
data_access_common :
mfspr r10 ,S P R N _ D A R
std r10 ,P A C A _ E X G E N + E X _ D A R ( r13 )
mfspr r10 ,S P R N _ D S I S R
stw r10 ,P A C A _ E X G E N + E X _ D S I S R ( r13 )
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x30 0 , P A C A _ E X G E N )
ld r3 ,P A C A _ E X G E N + E X _ D A R ( r13 )
lwz r4 ,P A C A _ E X G E N + E X _ D S I S R ( r13 )
li r5 ,0 x30 0
b . d o _ h a s h _ p a g e / * T r y t o h a n d l e a s h p t e f a u l t * /
.align 7
.globl instruction_access_common
instruction_access_common :
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x40 0 , P A C A _ E X G E N )
ld r3 ,_ N I P ( r1 )
andis. r4 ,r12 ,0 x58 2 0
li r5 ,0 x40 0
b . d o _ h a s h _ p a g e / * T r y t o h a n d l e a s h p t e f a u l t * /
/ *
* Here i s t h e c o m m o n S L B m i s s u s e r t h a t i s u s e d w h e n g o i n g t o v i r t u a l
* mode f o r S L B m i s s e s , t h a t i s c u r r e n t l y n o t u s e d
* /
# ifdef _ _ D I S A B L E D _ _
.align 7
.globl slb_miss_user_common
slb_miss_user_common :
mflr r10
std r3 ,P A C A _ E X G E N + E X _ D A R ( r13 )
stw r9 ,P A C A _ E X G E N + E X _ C C R ( r13 )
std r10 ,P A C A _ E X G E N + E X _ L R ( r13 )
std r11 ,P A C A _ E X G E N + E X _ S R R 0 ( r13 )
bl . s l b _ a l l o c a t e _ u s e r
ld r10 ,P A C A _ E X G E N + E X _ L R ( r13 )
ld r3 ,P A C A _ E X G E N + E X _ R 3 ( r13 )
lwz r9 ,P A C A _ E X G E N + E X _ C C R ( r13 )
ld r11 ,P A C A _ E X G E N + E X _ S R R 0 ( r13 )
mtlr r10
beq- s l b _ m i s s _ f a u l t
andi. r10 ,r12 ,M S R _ R I / * c h e c k f o r u n r e c o v e r a b l e e x c e p t i o n * /
beq- u n r e c o v _ u s e r _ s l b
mfmsr r10
.machine push
.machine " power4 "
mtcrf 0 x80 ,r9
.machine pop
clrrdi r10 ,r10 ,2 / * c l e a r R I b e f o r e s e t t i n g S R R 0 / 1 * /
mtmsrd r10 ,1
mtspr S R R 0 ,r11
mtspr S R R 1 ,r12
ld r9 ,P A C A _ E X G E N + E X _ R 9 ( r13 )
ld r10 ,P A C A _ E X G E N + E X _ R 1 0 ( r13 )
ld r11 ,P A C A _ E X G E N + E X _ R 1 1 ( r13 )
ld r12 ,P A C A _ E X G E N + E X _ R 1 2 ( r13 )
ld r13 ,P A C A _ E X G E N + E X _ R 1 3 ( r13 )
rfid
b .
slb_miss_fault :
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x38 0 , P A C A _ E X G E N )
ld r4 ,P A C A _ E X G E N + E X _ D A R ( r13 )
li r5 ,0
std r4 ,_ D A R ( r1 )
std r5 ,_ D S I S R ( r1 )
b h a n d l e _ p a g e _ f a u l t
unrecov_user_slb :
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x42 0 0 , P A C A _ E X G E N )
DISABLE_ I N T S
bl . s a v e _ n v g p r s
1 : addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
bl . u n r e c o v e r a b l e _ e x c e p t i o n
b 1 b
# endif / * _ _ D I S A B L E D _ _ * /
/ *
* r1 3 p o i n t s t o t h e P A C A , r9 c o n t a i n s t h e s a v e d C R ,
* r1 2 c o n t a i n t h e s a v e d S R R 1 , S R R 0 i s s t i l l r e a d y f o r r e t u r n
* r3 h a s t h e f a u l t i n g a d d r e s s
* r9 - r13 a r e s a v e d i n p a c a - > e x s l b .
* r3 i s s a v e d i n p a c a - > s l b _ r3
* We a s s u m e w e a r e n ' t g o i n g t o t a k e a n y e x c e p t i o n s d u r i n g t h i s p r o c e d u r e .
* /
_ GLOBAL( s l b _ m i s s _ r e a l m o d e )
mflr r10
# ifdef C O N F I G _ R E L O C A T A B L E
mtctr r11
# endif
stw r9 ,P A C A _ E X S L B + E X _ C C R ( r13 ) / * s a v e C R i n e x c . f r a m e * /
std r10 ,P A C A _ E X S L B + E X _ L R ( r13 ) / * s a v e L R * /
bl . s l b _ a l l o c a t e _ r e a l m o d e
/* All done -- return from exception. */
ld r10 ,P A C A _ E X S L B + E X _ L R ( r13 )
ld r3 ,P A C A _ E X S L B + E X _ R 3 ( r13 )
lwz r9 ,P A C A _ E X S L B + E X _ C C R ( r13 ) / * g e t s a v e d C R * /
# ifdef C O N F I G _ P P C _ I S E R I E S
BEGIN_ F W _ F T R _ S E C T I O N
ld r11 ,P A C A L P P A C A P T R ( r13 )
ld r11 ,L P P A C A S R R 0 ( r11 ) / * g e t S R R 0 v a l u e * /
END_ F W _ F T R _ S E C T I O N _ I F S E T ( F W _ F E A T U R E _ I S E R I E S )
# endif / * C O N F I G _ P P C _ I S E R I E S * /
mtlr r10
andi. r10 ,r12 ,M S R _ R I / * c h e c k f o r u n r e c o v e r a b l e e x c e p t i o n * /
beq- 2 f
.machine push
.machine " power4 "
mtcrf 0 x80 ,r9
mtcrf 0 x01 ,r9 / * s l b _ a l l o c a t e u s e s c r0 a n d c r7 * /
.machine pop
# ifdef C O N F I G _ P P C _ I S E R I E S
BEGIN_ F W _ F T R _ S E C T I O N
mtspr S P R N _ S R R 0 ,r11
mtspr S P R N _ S R R 1 ,r12
END_ F W _ F T R _ S E C T I O N _ I F S E T ( F W _ F E A T U R E _ I S E R I E S )
# endif / * C O N F I G _ P P C _ I S E R I E S * /
ld r9 ,P A C A _ E X S L B + E X _ R 9 ( r13 )
ld r10 ,P A C A _ E X S L B + E X _ R 1 0 ( r13 )
ld r11 ,P A C A _ E X S L B + E X _ R 1 1 ( r13 )
ld r12 ,P A C A _ E X S L B + E X _ R 1 2 ( r13 )
ld r13 ,P A C A _ E X S L B + E X _ R 1 3 ( r13 )
rfid
b . / * p r e v e n t s p e c u l a t i v e e x e c u t i o n * /
2 :
# ifdef C O N F I G _ P P C _ I S E R I E S
BEGIN_ F W _ F T R _ S E C T I O N
b u n r e c o v _ s l b
END_ F W _ F T R _ S E C T I O N _ I F S E T ( F W _ F E A T U R E _ I S E R I E S )
# endif / * C O N F I G _ P P C _ I S E R I E S * /
mfspr r11 ,S P R N _ S R R 0
ld r10 ,P A C A K B A S E ( r13 )
LOAD_ H A N D L E R ( r10 ,u n r e c o v _ s l b )
mtspr S P R N _ S R R 0 ,r10
ld r10 ,P A C A K M S R ( r13 )
mtspr S P R N _ S R R 1 ,r10
rfid
b .
unrecov_slb :
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x41 0 0 , P A C A _ E X S L B )
DISABLE_ I N T S
bl . s a v e _ n v g p r s
1 : addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
bl . u n r e c o v e r a b l e _ e x c e p t i o n
b 1 b
.align 7
.globl hardware_interrupt_common
.globl hardware_interrupt_entry
hardware_interrupt_common :
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x50 0 , P A C A _ E X G E N )
FINISH_ N A P
hardware_interrupt_entry :
DISABLE_ I N T S
BEGIN_ F T R _ S E C T I O N
bl . p p c64 _ r u n l a t c h _ o n
END_ F T R _ S E C T I O N _ I F S E T ( C P U _ F T R _ C T R L )
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
bl . d o _ I R Q
b . r e t _ f r o m _ e x c e p t _ l i t e
# ifdef C O N F I G _ P P C _ 9 7 0 _ N A P
power4_fixup_nap :
andc r9 ,r9 ,r10
std r9 ,T I _ L O C A L _ F L A G S ( r11 )
ld r10 ,_ L I N K ( r1 ) / * m a k e i d l e t a s k d o t h e * /
std r10 ,_ N I P ( r1 ) / * e q u i v a l e n t o f a b l r * /
blr
# endif
.align 7
.globl alignment_common
alignment_common :
mfspr r10 ,S P R N _ D A R
std r10 ,P A C A _ E X G E N + E X _ D A R ( r13 )
mfspr r10 ,S P R N _ D S I S R
stw r10 ,P A C A _ E X G E N + E X _ D S I S R ( r13 )
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x60 0 , P A C A _ E X G E N )
ld r3 ,P A C A _ E X G E N + E X _ D A R ( r13 )
lwz r4 ,P A C A _ E X G E N + E X _ D S I S R ( r13 )
std r3 ,_ D A R ( r1 )
std r4 ,_ D S I S R ( r1 )
bl . s a v e _ n v g p r s
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
ENABLE_ I N T S
bl . a l i g n m e n t _ e x c e p t i o n
b . r e t _ f r o m _ e x c e p t
.align 7
.globl program_check_common
program_check_common :
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x70 0 , P A C A _ E X G E N )
bl . s a v e _ n v g p r s
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
ENABLE_ I N T S
bl . p r o g r a m _ c h e c k _ e x c e p t i o n
b . r e t _ f r o m _ e x c e p t
.align 7
.globl fp_unavailable_common
fp_unavailable_common :
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x80 0 , P A C A _ E X G E N )
bne 1 f / * i f f r o m u s e r , j u s t l o a d i t u p * /
bl . s a v e _ n v g p r s
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
ENABLE_ I N T S
bl . k e r n e l _ f p _ u n a v a i l a b l e _ e x c e p t i o n
BUG_ O P C O D E
1 : bl . l o a d _ u p _ f p u
b f a s t _ e x c e p t i o n _ r e t u r n
.align 7
.globl altivec_unavailable_common
altivec_unavailable_common :
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x f20 , P A C A _ E X G E N )
# ifdef C O N F I G _ A L T I V E C
BEGIN_ F T R _ S E C T I O N
beq 1 f
bl . l o a d _ u p _ a l t i v e c
b f a s t _ e x c e p t i o n _ r e t u r n
1 :
END_ F T R _ S E C T I O N _ I F S E T ( C P U _ F T R _ A L T I V E C )
# endif
bl . s a v e _ n v g p r s
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
ENABLE_ I N T S
bl . a l t i v e c _ u n a v a i l a b l e _ e x c e p t i o n
b . r e t _ f r o m _ e x c e p t
.align 7
.globl vsx_unavailable_common
vsx_unavailable_common :
EXCEPTION_ P R O L O G _ C O M M O N ( 0 x f40 , P A C A _ E X G E N )
# ifdef C O N F I G _ V S X
BEGIN_ F T R _ S E C T I O N
bne . l o a d _ u p _ v s x
1 :
END_ F T R _ S E C T I O N _ I F S E T ( C P U _ F T R _ V S X )
# endif
bl . s a v e _ n v g p r s
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
ENABLE_ I N T S
bl . v s x _ u n a v a i l a b l e _ e x c e p t i o n
b . r e t _ f r o m _ e x c e p t
.align 7
.globl __end_handlers
__end_handlers :
/ *
* Return f r o m a n e x c e p t i o n w i t h m i n i m a l c h e c k s .
* The c a l l e r i s a s s u m e d t o h a v e d o n e E X C E P T I O N _ P R O L O G _ C O M M O N .
* If i n t e r r u p t s h a v e b e e n e n a b l e d , o r a n y t h i n g h a s b e e n
* done t h a t m i g h t h a v e c h a n g e d t h e s c h e d u l i n g s t a t u s o f
* any t a s k o r s e n t a n y t a s k a s i g n a l , y o u s h o u l d u s e
* ret_ f r o m _ e x c e p t o r r e t _ f r o m _ e x c e p t _ l i t e i n s t e a d o f t h i s .
* /
fast_exc_return_irq : /* restores irq state too */
ld r3 ,S O F T E ( r1 )
TRACE_ A N D _ R E S T O R E _ I R Q ( r3 ) ;
ld r12 ,_ M S R ( r1 )
rldicl r4 ,r12 ,4 9 ,6 3 / * g e t M S R _ E E t o L S B * /
stb r4 ,P A C A H A R D I R Q E N ( r13 ) / * r e s t o r e p a c a - > h a r d _ e n a b l e d * /
b 1 f
.globl fast_exception_return
fast_exception_return :
ld r12 ,_ M S R ( r1 )
1 : ld r11 ,_ N I P ( r1 )
andi. r3 ,r12 ,M S R _ R I / * c h e c k i f R I i s s e t * /
beq- u n r e c o v _ f e r
# ifdef C O N F I G _ V I R T _ C P U _ A C C O U N T I N G
andi. r3 ,r12 ,M S R _ P R
beq 2 f
ACCOUNT_ C P U _ U S E R _ E X I T ( r3 , r4 )
2 :
# endif
ld r3 ,_ C C R ( r1 )
ld r4 ,_ L I N K ( r1 )
ld r5 ,_ C T R ( r1 )
ld r6 ,_ X E R ( r1 )
mtcr r3
mtlr r4
mtctr r5
mtxer r6
REST_ G P R ( 0 , r1 )
REST_ 8 G P R S ( 2 , r1 )
mfmsr r10
rldicl r10 ,r10 ,4 8 ,1 / * c l e a r E E * /
rldicr r10 ,r10 ,1 6 ,6 1 / * c l e a r R I ( L E i s 0 a l r e a d y ) * /
mtmsrd r10 ,1
mtspr S P R N _ S R R 1 ,r12
mtspr S P R N _ S R R 0 ,r11
REST_ 4 G P R S ( 1 0 , r1 )
ld r1 ,G P R 1 ( r1 )
rfid
b . / * p r e v e n t s p e c u l a t i v e e x e c u t i o n * /
unrecov_fer :
bl . s a v e _ n v g p r s
1 : addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
bl . u n r e c o v e r a b l e _ e x c e p t i o n
b 1 b
/ *
* Hash t a b l e s t u f f
* /
.align 7
_ STATIC( d o _ h a s h _ p a g e )
std r3 ,_ D A R ( r1 )
std r4 ,_ D S I S R ( r1 )
andis. r0 ,r4 ,0 x a45 0 / * w e i r d e r r o r ? * /
bne- h a n d l e _ p a g e _ f a u l t / * i f n o t , t r y t o i n s e r t a H P T E * /
BEGIN_ F T R _ S E C T I O N
andis. r0 ,r4 ,0 x00 2 0 / * I s i t a s e g m e n t t a b l e f a u l t ? * /
bne- d o _ s t e _ a l l o c / * I f s o h a n d l e i t * /
END_ F T R _ S E C T I O N _ I F C L R ( C P U _ F T R _ S L B )
powerpc: Allow perf_counters to access user memory at interrupt time
This provides a mechanism to allow the perf_counters code to access
user memory in a PMU interrupt routine. Such an access can cause
various kinds of interrupt: SLB miss, MMU hash table miss, segment
table miss, or TLB miss, depending on the processor. This commit
only deals with 64-bit classic/server processors, which use an MMU
hash table. 32-bit processors are already able to access user memory
at interrupt time. Since we don't soft-disable on 32-bit, we avoid
the possibility of reentering hash_page or the TLB miss handlers,
since they run with interrupts disabled.
On 64-bit processors, an SLB miss interrupt on a user address will
update the slb_cache and slb_cache_ptr fields in the paca. This is
OK except in the case where a PMU interrupt occurs in switch_slb,
which also accesses those fields. To prevent this, we hard-disable
interrupts in switch_slb. Interrupts are already soft-disabled at
this point, and will get hard-enabled when they get soft-enabled
later.
This also reworks slb_flush_and_rebolt: to avoid hard-disabling twice,
and to make sure that it clears the slb_cache_ptr when called from
other callers than switch_slb, the existing routine is renamed to
__slb_flush_and_rebolt, which is called by switch_slb and the new
version of slb_flush_and_rebolt.
Similarly, switch_stab (used on POWER3 and RS64 processors) gets a
hard_irq_disable() to protect the per-cpu variables used there and
in ste_allocate.
If a MMU hashtable miss interrupt occurs, normally we would call
hash_page to look up the Linux PTE for the address and create a HPTE.
However, hash_page is fairly complex and takes some locks, so to
avoid the possibility of deadlock, we check the preemption count
to see if we are in a (pseudo-)NMI handler, and if so, we don't call
hash_page but instead treat it like a bad access that will get
reported up through the exception table mechanism. An interrupt
whose handler runs even though the interrupt occurred when
soft-disabled (such as the PMU interrupt) is considered a pseudo-NMI
handler, which should use nmi_enter()/nmi_exit() rather than
irq_enter()/irq_exit().
Acked-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Paul Mackerras <paulus@samba.org>
2009-08-17 09:17:54 +04:00
clrrdi r11 ,r1 ,T H R E A D _ S H I F T
lwz r0 ,T I _ P R E E M P T ( r11 ) / * I f w e ' r e i n a n " N M I " * /
andis. r0 ,r0 ,N M I _ M A S K @h /* (i.e. an irq when soft-disabled) */
bne 7 7 f / * t h e n d o n ' t c a l l h a s h _ p a g e n o w * /
2009-06-03 01:17:38 +04:00
/ *
* On i S e r i e s , w e s o f t - d i s a b l e i n t e r r u p t s h e r e , t h e n
* hard- e n a b l e i n t e r r u p t s s o t h a t t h e h a s h _ p a g e c o d e c a n s p i n o n
* the h a s h _ t a b l e _ l o c k w i t h o u t p r o b l e m s o n a s h a r e d p r o c e s s o r .
* /
DISABLE_ I N T S
/ *
* Currently, t r a c e _ h a r d i r q s _ o f f ( ) w i l l b e c a l l e d b y D I S A B L E _ I N T S
* and w i l l c l o b b e r v o l a t i l e r e g i s t e r s w h e n i r q t r a c i n g i s e n a b l e d
* so w e n e e d t o r e l o a d t h e m . I t m a y b e p o s s i b l e t o b e s m a r t e r h e r e
* and m o v e t h e i r q t r a c i n g e l s e w h e r e b u t l e t ' s k e e p i t s i m p l e f o r
* now
* /
# ifdef C O N F I G _ T R A C E _ I R Q F L A G S
ld r3 ,_ D A R ( r1 )
ld r4 ,_ D S I S R ( r1 )
ld r5 ,_ T R A P ( r1 )
ld r12 ,_ M S R ( r1 )
clrrdi r5 ,r5 ,4
# endif / * C O N F I G _ T R A C E _ I R Q F L A G S * /
/ *
* We n e e d t o s e t t h e _ P A G E _ U S E R b i t i f M S R _ P R i s s e t o r i f w e a r e
* accessing a u s e r s p a c e s e g m e n t ( e v e n f r o m t h e k e r n e l ) . W e a s s u m e
* kernel a d d r e s s e s a l w a y s h a v e t h e h i g h b i t s e t .
* /
rlwinm r4 ,r4 ,3 2 - 2 5 + 9 ,3 1 - 9 ,3 1 - 9 / * D S I S R _ S T O R E - > _ P A G E _ R W * /
rotldi r0 ,r3 ,1 5 / * M o v e h i g h b i t i n t o M S R _ P R p o s n * /
orc r0 ,r12 ,r0 / * M S R _ P R | ~ h i g h _ b i t * /
rlwimi r4 ,r0 ,3 2 - 1 3 ,3 0 ,3 0 / * b e c o m e s _ P A G E _ U S E R a c c e s s b i t * /
ori r4 ,r4 ,1 / * a d d _ P A G E _ P R E S E N T * /
rlwimi r4 ,r5 ,2 2 + 2 ,3 1 - 2 ,3 1 - 2 / * S e t _ P A G E _ E X E C i f t r a p i s 0 x40 0 * /
/ *
* r3 c o n t a i n s t h e f a u l t i n g a d d r e s s
* r4 c o n t a i n s t h e r e q u i r e d a c c e s s p e r m i s s i o n s
* r5 c o n t a i n s t h e t r a p n u m b e r
*
* at r e t u r n r3 = 0 f o r s u c c e s s
* /
bl . h a s h _ p a g e / * b u i l d H P T E i f p o s s i b l e * /
cmpdi r3 ,0 / * s e e i f h a s h _ p a g e s u c c e e d e d * /
BEGIN_ F W _ F T R _ S E C T I O N
/ *
* If w e h a d i n t e r r u p t s s o f t - e n a b l e d a t t h e p o i n t w h e r e t h e
* DSI/ I S I o c c u r r e d , a n d a n i n t e r r u p t c a m e i n d u r i n g h a s h _ p a g e ,
* handle i t n o w .
* We j u m p t o r e t _ f r o m _ e x c e p t _ l i t e r a t h e r t h a n f a s t _ e x c e p t i o n _ r e t u r n
* because r e t _ f r o m _ e x c e p t _ l i t e w i l l c h e c k f o r a n d h a n d l e p e n d i n g
* interrupts i f n e c e s s a r y .
* /
beq 1 3 f
END_ F W _ F T R _ S E C T I O N _ I F S E T ( F W _ F E A T U R E _ I S E R I E S )
BEGIN_ F W _ F T R _ S E C T I O N
/ *
* Here w e h a v e i n t e r r u p t s h a r d - d i s a b l e d , s o i t i s s u f f i c i e n t
* to r e s t o r e p a c a - > { s o f t ,h a r d } _ e n a b l e a n d g e t o u t .
* /
beq f a s t _ e x c _ r e t u r n _ i r q / * R e t u r n f r o m e x c e p t i o n o n s u c c e s s * /
END_ F W _ F T R _ S E C T I O N _ I F C L R ( F W _ F E A T U R E _ I S E R I E S )
/* For a hash failure, we don't bother re-enabling interrupts */
ble- 1 2 f
/ *
* hash_ p a g e c o u l d n ' t h a n d l e i t , s e t s o f t i n t e r r u p t e n a b l e b a c k
* to w h a t i t w a s b e f o r e t h e t r a p . N o t e t h a t . r a w _ l o c a l _ i r q _ r e s t o r e
* handles a n y i n t e r r u p t s p e n d i n g a t t h i s p o i n t .
* /
ld r3 ,S O F T E ( r1 )
TRACE_ A N D _ R E S T O R E _ I R Q _ P A R T I A L ( r3 , 1 1 f )
bl . r a w _ l o c a l _ i r q _ r e s t o r e
b 1 1 f
/* Here we have a page fault that hash_page can't handle. */
handle_page_fault :
ENABLE_ I N T S
11 : ld r4 ,_ D A R ( r1 )
ld r5 ,_ D S I S R ( r1 )
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
bl . d o _ p a g e _ f a u l t
cmpdi r3 ,0
beq+ 1 3 f
bl . s a v e _ n v g p r s
mr r5 ,r3
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
lwz r4 ,_ D A R ( r1 )
bl . b a d _ p a g e _ f a u l t
b . r e t _ f r o m _ e x c e p t
13 : b . r e t _ f r o m _ e x c e p t _ l i t e
/ * We h a v e a p a g e f a u l t t h a t h a s h _ p a g e c o u l d h a n d l e b u t H V r e f u s e d
* the P T E i n s e r t i o n
* /
12 : bl . s a v e _ n v g p r s
mr r5 ,r3
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
ld r4 ,_ D A R ( r1 )
bl . l o w _ h a s h _ f a u l t
b . r e t _ f r o m _ e x c e p t
powerpc: Allow perf_counters to access user memory at interrupt time
This provides a mechanism to allow the perf_counters code to access
user memory in a PMU interrupt routine. Such an access can cause
various kinds of interrupt: SLB miss, MMU hash table miss, segment
table miss, or TLB miss, depending on the processor. This commit
only deals with 64-bit classic/server processors, which use an MMU
hash table. 32-bit processors are already able to access user memory
at interrupt time. Since we don't soft-disable on 32-bit, we avoid
the possibility of reentering hash_page or the TLB miss handlers,
since they run with interrupts disabled.
On 64-bit processors, an SLB miss interrupt on a user address will
update the slb_cache and slb_cache_ptr fields in the paca. This is
OK except in the case where a PMU interrupt occurs in switch_slb,
which also accesses those fields. To prevent this, we hard-disable
interrupts in switch_slb. Interrupts are already soft-disabled at
this point, and will get hard-enabled when they get soft-enabled
later.
This also reworks slb_flush_and_rebolt: to avoid hard-disabling twice,
and to make sure that it clears the slb_cache_ptr when called from
other callers than switch_slb, the existing routine is renamed to
__slb_flush_and_rebolt, which is called by switch_slb and the new
version of slb_flush_and_rebolt.
Similarly, switch_stab (used on POWER3 and RS64 processors) gets a
hard_irq_disable() to protect the per-cpu variables used there and
in ste_allocate.
If a MMU hashtable miss interrupt occurs, normally we would call
hash_page to look up the Linux PTE for the address and create a HPTE.
However, hash_page is fairly complex and takes some locks, so to
avoid the possibility of deadlock, we check the preemption count
to see if we are in a (pseudo-)NMI handler, and if so, we don't call
hash_page but instead treat it like a bad access that will get
reported up through the exception table mechanism. An interrupt
whose handler runs even though the interrupt occurred when
soft-disabled (such as the PMU interrupt) is considered a pseudo-NMI
handler, which should use nmi_enter()/nmi_exit() rather than
irq_enter()/irq_exit().
Acked-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Paul Mackerras <paulus@samba.org>
2009-08-17 09:17:54 +04:00
/ *
* We c o m e h e r e a s a r e s u l t o f a D S I a t a p o i n t w h e r e w e d o n ' t w a n t
* to c a l l h a s h _ p a g e , s u c h a s w h e n w e a r e a c c e s s i n g m e m o r y ( p o s s i b l y
* user m e m o r y ) i n s i d e a P M U i n t e r r u p t t h a t o c c u r r e d w h i l e i n t e r r u p t s
* were s o f t - d i s a b l e d . W e w a n t t o i n v o k e t h e e x c e p t i o n h a n d l e r f o r
* the a c c e s s , o r p a n i c i f t h e r e i s n ' t a h a n d l e r .
* /
77 : bl . s a v e _ n v g p r s
mr r4 ,r3
addi r3 ,r1 ,S T A C K _ F R A M E _ O V E R H E A D
li r5 ,S I G S E G V
bl . b a d _ p a g e _ f a u l t
b . r e t _ f r o m _ e x c e p t
2009-06-03 01:17:38 +04:00
/* here we have a segment miss */
do_ste_alloc :
bl . s t e _ a l l o c a t e / * t r y t o i n s e r t s t a b e n t r y * /
cmpdi r3 ,0
bne- h a n d l e _ p a g e _ f a u l t
b f a s t _ e x c e p t i o n _ r e t u r n
/ *
* r1 3 p o i n t s t o t h e P A C A , r9 c o n t a i n s t h e s a v e d C R ,
* r1 1 a n d r12 c o n t a i n t h e s a v e d S R R 0 a n d S R R 1 .
* r9 - r13 a r e s a v e d i n p a c a - > e x s l b .
* We a s s u m e w e a r e n ' t g o i n g t o t a k e a n y e x c e p t i o n s d u r i n g t h i s p r o c e d u r e .
* We a s s u m e ( D A R > > 6 0 ) = = 0 x c .
* /
.align 7
_ GLOBAL( d o _ s t a b _ b o l t e d )
stw r9 ,P A C A _ E X S L B + E X _ C C R ( r13 ) / * s a v e C R i n e x c . f r a m e * /
std r11 ,P A C A _ E X S L B + E X _ S R R 0 ( r13 ) / * s a v e S R R 0 i n e x c . f r a m e * /
/* Hash to the primary group */
ld r10 ,P A C A S T A B V I R T ( r13 )
mfspr r11 ,S P R N _ D A R
srdi r11 ,r11 ,2 8
rldimi r10 ,r11 ,7 ,5 2 / * r10 = f i r s t s t e o f t h e g r o u p * /
/* Calculate VSID */
/* This is a kernel address, so protovsid = ESID */
ASM_ V S I D _ S C R A M B L E ( r11 , r9 , 2 5 6 M )
rldic r9 ,r11 ,1 2 ,1 6 / * r9 = v s i d < < 1 2 * /
/* Search the primary group for a free entry */
1 : ld r11 ,0 ( r10 ) / * T e s t v a l i d b i t o f t h e c u r r e n t s t e * /
andi. r11 ,r11 ,0 x80
beq 2 f
addi r10 ,r10 ,1 6
andi. r11 ,r10 ,0 x70
bne 1 b
/* Stick for only searching the primary group for now. */
/* At least for now, we use a very simple random castout scheme */
/* Use the TB as a random number ; OR in 1 to avoid entry 0 */
mftb r11
rldic r11 ,r11 ,4 ,5 7 / * r11 = ( r11 < < 4 ) & 0 x70 * /
ori r11 ,r11 ,0 x10
/* r10 currently points to an ste one past the group of interest */
/* make it point to the randomly selected entry */
subi r10 ,r10 ,1 2 8
or r10 ,r10 ,r11 / * r10 i s t h e e n t r y t o i n v a l i d a t e * /
isync / * m a r k t h e e n t r y i n v a l i d * /
ld r11 ,0 ( r10 )
rldicl r11 ,r11 ,5 6 ,1 / * c l e a r t h e v a l i d b i t * /
rotldi r11 ,r11 ,8
std r11 ,0 ( r10 )
sync
clrrdi r11 ,r11 ,2 8 / * G e t t h e e s i d p a r t o f t h e s t e * /
slbie r11
2 : std r9 ,8 ( r10 ) / * S t o r e t h e v s i d p a r t o f t h e s t e * /
eieio
mfspr r11 ,S P R N _ D A R / * G e t t h e n e w e s i d * /
clrrdi r11 ,r11 ,2 8 / * P e r m i t s a f u l l 3 2 b o f E S I D * /
ori r11 ,r11 ,0 x90 / * T u r n o n v a l i d a n d k p * /
std r11 ,0 ( r10 ) / * P u t n e w e n t r y b a c k i n t o t h e s t a b * /
sync
/* All done -- return from exception. */
lwz r9 ,P A C A _ E X S L B + E X _ C C R ( r13 ) / * g e t s a v e d C R * /
ld r11 ,P A C A _ E X S L B + E X _ S R R 0 ( r13 ) / * g e t s a v e d S R R 0 * /
andi. r10 ,r12 ,M S R _ R I
beq- u n r e c o v _ s l b
mtcrf 0 x80 ,r9 / * r e s t o r e C R * /
mfmsr r10
clrrdi r10 ,r10 ,2
mtmsrd r10 ,1
mtspr S P R N _ S R R 0 ,r11
mtspr S P R N _ S R R 1 ,r12
ld r9 ,P A C A _ E X S L B + E X _ R 9 ( r13 )
ld r10 ,P A C A _ E X S L B + E X _ R 1 0 ( r13 )
ld r11 ,P A C A _ E X S L B + E X _ R 1 1 ( r13 )
ld r12 ,P A C A _ E X S L B + E X _ R 1 2 ( r13 )
ld r13 ,P A C A _ E X S L B + E X _ R 1 3 ( r13 )
rfid
b . / * p r e v e n t s p e c u l a t i v e e x e c u t i o n * /
/ *
* Space f o r C P U 0 ' s s e g m e n t t a b l e .
*
* On i S e r i e s , t h e h y p e r v i s o r m u s t f i l l i n a t l e a s t o n e e n t r y b e f o r e
* we g e t c o n t r o l ( w i t h r e l o c a t e o n ) . T h e a d d r e s s i s g i v e n t o t h e h v
* as a p a g e n u m b e r ( s e e x L p a r M a p b e l o w ) , s o t h i s m u s t b e a t a
* fixed a d d r e s s ( t h e l i n k e r c a n ' t c o m p u t e ( u 6 4 ) & i n i t i a l _ s t a b > >
* PAGE_ S H I F T ) .
* /
. = STAB0 _ O F F S E T / * 0 x60 0 0 * /
.globl initial_stab
initial_stab :
.space 4096
# ifdef C O N F I G _ P P C _ P S E R I E S
/ *
* Data a r e a r e s e r v e d f o r F W N M I o p t i o n .
* This a d d r e s s ( 0 x70 0 0 ) i s f i x e d b y t h e R P A .
* /
. = 0 x7 0 0 0
.globl fwnmi_data_area
fwnmi_data_area :
# endif / * C O N F I G _ P P C _ P S E R I E S * /
/ * iSeries d o e s n o t u s e t h e F W N M I s t u f f , s o i t i s s a f e t o p u t
* this h e r e , e v e n i f w e l a t e r a l l o w k e r n e l s t h a t w i l l b o o t o n
* both p S e r i e s a n d i S e r i e s * /
# ifdef C O N F I G _ P P C _ I S E R I E S
. = LPARMAP_ P H Y S
.globl xLparMap
xLparMap :
.quad HvEsidsToMap /* xNumberEsids */
.quad HvRangesToMap /* xNumberRanges */
.quad STAB0_PAGE /* xSegmentTableOffs */
.zero 40 /* xRsvd */
/* xEsids (HvEsidsToMap entries of 2 quads) */
.quad PAGE_OFFSET_ESID /* xKernelEsid */
.quad PAGE_OFFSET_VSID /* xKernelVsid */
.quad VMALLOC_START_ESID /* xKernelEsid */
.quad VMALLOC_START_VSID /* xKernelVsid */
/* xRanges (HvRangesToMap entries of 3 quads) */
.quad HvPagesToMap /* xPages */
.quad 0 /* xOffset */
.quad PAGE_OFFSET_VSID < < ( SID_ S H I F T - H W _ P A G E _ S H I F T ) / * x V P N * /
# endif / * C O N F I G _ P P C _ I S E R I E S * /
# ifdef C O N F I G _ P P C _ P S E R I E S
. = 0 x8 0 0 0
# endif / * C O N F I G _ P P C _ P S E R I E S * /
