2013-02-13 20:21:35 +04:00
/ *
* Transactional m e m o r y s u p p o r t r o u t i n e s t o r e c l a i m a n d r e c h e c k p o i n t
* transactional p r o c e s s s t a t e .
*
* Copyright 2 0 1 2 M a t t E v a n s & M i c h a e l N e u l i n g , I B M C o r p o r a t i o n .
* /
# include < a s m / a s m - o f f s e t s . h >
# include < a s m / p p c _ a s m . h >
# include < a s m / p p c - o p c o d e . h >
# include < a s m / p t r a c e . h >
# include < a s m / r e g . h >
# ifdef C O N F I G _ V S X
/* See fpu.S, this is very similar but to save/restore checkpointed FPRs/VSRs */
# define _ _ S A V E _ 3 2 F P R S _ V S R S _ T R A N S A C T ( n ,c ,b a s e ) \
BEGIN_ F T R _ S E C T I O N \
b 2 f ; \
END_ F T R _ S E C T I O N _ I F S E T ( C P U _ F T R _ V S X ) ; \
SAVE_ 3 2 F P R S _ T R A N S A C T ( n ,b a s e ) ; \
b 3 f ; \
2 : SAVE_ 3 2 V S R S _ T R A N S A C T ( n ,c ,b a s e ) ; \
3 :
/* ...and this is just plain borrowed from there. */
# define _ _ R E S T _ 3 2 F P R S _ V S R S ( n ,c ,b a s e ) \
BEGIN_ F T R _ S E C T I O N \
b 2 f ; \
END_ F T R _ S E C T I O N _ I F S E T ( C P U _ F T R _ V S X ) ; \
REST_ 3 2 F P R S ( n ,b a s e ) ; \
b 3 f ; \
2 : REST_ 3 2 V S R S ( n ,c ,b a s e ) ; \
3 :
# else
# define _ _ S A V E _ 3 2 F P R S _ V S R S _ T R A N S A C T ( n ,c ,b a s e ) S A V E _ 3 2 F P R S _ T R A N S A C T ( n , b a s e )
# define _ _ R E S T _ 3 2 F P R S _ V S R S ( n ,c ,b a s e ) R E S T _ 3 2 F P R S ( n , b a s e )
# endif
# define S A V E _ 3 2 F P R S _ V S R S _ T R A N S A C T ( n ,c ,b a s e ) \
_ _ SAVE_ 3 2 F P R S _ V S R S _ T R A N S A C T ( n ,_ _ R E G _ ## c , _ _ R E G _ # # b a s e )
# define R E S T _ 3 2 F P R S _ V S R S ( n ,c ,b a s e ) \
_ _ REST_ 3 2 F P R S _ V S R S ( n ,_ _ R E G _ ## c , _ _ R E G _ # # b a s e )
/* Stack frame offsets for local variables. */
# define T M _ F R A M E _ L 0 T M _ F R A M E _ S I Z E - 1 6
# define T M _ F R A M E _ L 1 T M _ F R A M E _ S I Z E - 8
# define S T A C K _ P A R A M ( x ) ( 4 8 + ( ( x ) * 8 ) )
/* In order to access the TM SPRs, TM must be enabled. So, do so: */
_ GLOBAL( t m _ e n a b l e )
mfmsr r4
li r3 , M S R _ T M > > 3 2
sldi r3 , r3 , 3 2
and. r0 , r4 , r3
bne 1 f
or r4 , r4 , r3
mtmsrd r4
1 : blr
_ GLOBAL( t m _ s a v e _ s p r s )
mfspr r0 , S P R N _ T F H A R
std r0 , T H R E A D _ T M _ T F H A R ( r3 )
mfspr r0 , S P R N _ T E X A S R
std r0 , T H R E A D _ T M _ T E X A S R ( r3 )
mfspr r0 , S P R N _ T F I A R
std r0 , T H R E A D _ T M _ T F I A R ( r3 )
blr
_ GLOBAL( t m _ r e s t o r e _ s p r s )
ld r0 , T H R E A D _ T M _ T F H A R ( r3 )
mtspr S P R N _ T F H A R , r0
ld r0 , T H R E A D _ T M _ T E X A S R ( r3 )
mtspr S P R N _ T E X A S R , r0
ld r0 , T H R E A D _ T M _ T F I A R ( r3 )
mtspr S P R N _ T F I A R , r0
blr
/* Passed an 8-bit failure cause as first argument. */
_ GLOBAL( t m _ a b o r t )
TABORT( R 3 )
blr
/ * void t m _ r e c l a i m ( s t r u c t t h r e a d _ s t r u c t * t h r e a d ,
* unsigned l o n g o r i g _ m s r ,
* uint8 _ t c a u s e )
*
* - Performs a f u l l r e c l a i m . T h i s d e s t r o y s o u t s t a n d i n g
* transactions a n d u p d a t e s t h r e a d - > r e g s . t m _ c k p t _ * w i t h t h e
* original c h e c k p o i n t e d s t a t e . N o t e t h a t t h r e a d - > r e g s i s
* unchanged.
* - FP r e g s a r e w r i t t e n b a c k t o t h r e a d - > t r a n s a c t _ f p r b e f o r e
* reclaiming. T h e s e a r e t h e t r a n s a c t i o n a l ( c u r r e n t ) v e r s i o n s .
*
* Purpose i s t o b o t h a b o r t t r a n s a c t i o n s o f , a n d p r e s e r v e t h e s t a t e o f ,
* a t r a n s a c t i o n s a t a c o n t e x t s w i t c h . W e p r e s e r v e / r e s t o r e b o t h s e t s o f p r o c e s s
* state t o r e s t o r e t h e m w h e n t h e t h r e a d ' s s c h e d u l e d a g a i n . W e c o n t i n u e i n
* userland a s t h o u g h n o t h i n g h a p p e n e d , b u t w h e n t h e t r a n s a c t i o n i s r e s u m e d
* they w i l l a b o r t b a c k t o t h e c h e c k p o i n t e d s t a t e w e s a v e o u t h e r e .
*
* Call w i t h I R Q s o f f , s t a c k s g e t a l l o u t o f s y n c f o r s o m e p e r i o d s i n h e r e !
* /
_ GLOBAL( t m _ r e c l a i m )
mfcr r6
mflr r0
std r6 , 8 ( r1 )
std r0 , 1 6 ( r1 )
std r2 , 4 0 ( r1 )
stdu r1 , - T M _ F R A M E _ S I Z E ( r1 )
/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD]. */
std r3 , S T A C K _ P A R A M ( 0 ) ( r1 )
SAVE_ N V G P R S ( r1 )
2013-06-28 12:17:09 +04:00
/ * We n e e d t o s e t u p M S R f o r V S X r e g i s t e r s a v e i n s t r u c t i o n s . H e r e w e
* also c l e a r t h e M S R R I s i n c e w h e n w e d o t h e t r e c l a i m , w e w o n ' t h a v e a
* valid k e r n e l p o i n t e r f o r a w h i l e . W e c l e a r R I h e r e a s i t a v o i d s
* adding a n o t h e r m t m s r c l o s e r t o t h e t r e c l a i m . T h i s m a k e s t h e r e g i o n
* maked a s n o n - r e c o v e r a b l e w i d e r t h a n i t n e e d s t o b e b u t i t s a v e s o n
* inserting a n o t h e r m t m s r d l a t e r .
* /
2013-02-13 20:21:35 +04:00
mfmsr r14
mr r15 , r14
ori r15 , r15 , M S R _ F P
2013-06-28 12:17:09 +04:00
li r16 , M S R _ R I
2013-10-02 11:15:15 +04:00
ori r16 , r16 , M S R _ E E / * I R Q s h a r d o f f * /
2013-06-28 12:17:09 +04:00
andc r15 , r15 , r16
2013-02-13 20:21:35 +04:00
oris r15 , r15 , M S R _ V E C @h
# ifdef C O N F I G _ V S X
BEGIN_ F T R _ S E C T I O N
oris r15 ,r15 , M S R _ V S X @h
END_ F T R _ S E C T I O N _ I F S E T ( C P U _ F T R _ V S X )
# endif
mtmsrd r15
std r14 , T M _ F R A M E _ L 0 ( r1 )
/* Stash the stack pointer away for use after reclaim */
std r1 , P A C A R 1 ( r13 )
/ * * * * * * * * * * * * * * * * * * * * * FPR/ V R / V S R s * * * * * * * * * * * *
* Before r e c l a i m i n g , c a p t u r e t h e c u r r e n t / t r a n s a c t i o n a l F P R / V R
* versions / i f u s e d / .
*
* ( If V S X u s e d , F P a n d V M X a r e i m p l i e d . O r , w e d o n ' t n e e d t o l o o k
* at M S R . V S X a s c o p y i n g F P r e g s i f . F P , v e c t o r r e g s i f . V M X c o v e r s i t . )
*
* We' r e p a s s e d t h e t h r e a d ' s M S R a s p a r a m e t e r 2 .
*
* We e n a b l e d V E C / F P / V S X i n t h e m s r a b o v e , s o w e c a n e x e c u t e t h e s e
* instructions!
* /
andis. r0 , r4 , M S R _ V E C @h
beq d o n t _ b a c k u p _ v e c
SAVE_ 3 2 V R S _ T R A N S A C T ( 0 , r6 , r3 ) / * r6 s c r a t c h , r3 t h r e a d * /
mfvscr v r0
li r6 , T H R E A D _ T R A N S A C T _ V S C R
stvx v r0 , r3 , r6
2013-08-05 08:13:16 +04:00
dont_backup_vec :
2013-02-13 20:21:35 +04:00
mfspr r0 , S P R N _ V R S A V E
std r0 , T H R E A D _ T R A N S A C T _ V R S A V E ( r3 )
andi. r0 , r4 , M S R _ F P
beq d o n t _ b a c k u p _ f p
SAVE_ 3 2 F P R S _ V S R S _ T R A N S A C T ( 0 , R 6 , R 3 ) / * r6 s c r a t c h , r3 t h r e a d * /
mffs f r0
stfd f r0 ,T H R E A D _ T R A N S A C T _ F P S C R ( r3 )
dont_backup_fp :
/ * The m o m e n t w e t r e c l a i m , A L L o f o u r G P R s w i l l s w i t c h
* to u s e r r e g i s t e r s t a t e . ( F P R s , C C R e t c . a l s o ! )
* Use a n s p r g a n d a t m _ s c r a t c h i n t h e P A C A t o s h u f f l e .
* /
TRECLAIM( R 5 ) / * C a u s e i n r5 * /
/* ******************** GPRs ******************** */
/ * Stash t h e c h e c k p o i n t e d r13 a w a y i n t h e s c r a t c h S P R a n d g e t t h e r e a l
* paca
* /
SET_ S C R A T C H 0 ( r13 )
GET_ P A C A ( r13 )
/ * Stash t h e c h e c k p o i n t e d r1 a w a y i n p a c a t m _ s c r a t c h a n d g e t t h e r e a l
* stack p o i n t e r b a c k
* /
std r1 , P A C A T M S C R A T C H ( r13 )
ld r1 , P A C A R 1 ( r13 )
/* Now get some more GPRS free */
std r7 , G P R 7 ( r1 ) / * T e m p o r a r y s t a s h * /
std r12 , G P R 1 2 ( r1 ) / * ' ' ' ' ' ' * /
ld r12 , S T A C K _ P A R A M ( 0 ) ( r1 ) / * P a r a m 0 , t h r e a d _ s t r u c t * * /
addi r7 , r12 , P T _ C K P T _ R E G S / * T h r e a d ' s c k p t _ r e g s * /
/ * Make r7 l o o k l i k e a n e x c e p t i o n f r a m e s o t h a t w e
* can u s e t h e n e a t G P R x ( n ) m a c r o s . r7 i s N O T a p t _ r e g s p t r !
* /
subi r7 , r7 , S T A C K _ F R A M E _ O V E R H E A D
/* Sync the userland GPRs 2-12, 14-31 to thread->regs: */
SAVE_ G P R ( 0 , r7 ) / * u s e r r0 * /
SAVE_ G P R ( 2 , r7 ) / * u s e r r2 * /
SAVE_ 4 G P R S ( 3 , r7 ) / * u s e r r3 - r6 * /
SAVE_ 4 G P R S ( 8 , r7 ) / * u s e r r8 - r11 * /
ld r3 , P A C A T M S C R A T C H ( r13 ) / * u s e r r1 * /
ld r4 , G P R 7 ( r1 ) / * u s e r r7 * /
ld r5 , G P R 1 2 ( r1 ) / * u s e r r12 * /
GET_ S C R A T C H 0 ( 6 ) / * u s e r r13 * /
std r3 , G P R 1 ( r7 )
std r4 , G P R 7 ( r7 )
std r5 , G P R 1 2 ( r7 )
std r6 , G P R 1 3 ( r7 )
SAVE_ N V G P R S ( r7 ) / * u s e r r14 - r31 * /
/* ******************** NIP ******************** */
mfspr r3 , S P R N _ T F H A R
std r3 , _ N I P ( r7 ) / * R e t u r n s t o f a i l h a n d l e r * /
/ * The c h e c k p o i n t e d N I P i s i g n o r e d w h e n r e s c h e d u l i n g / r e c h k p t i n g ,
* but i s u s e d i n s i g n a l r e t u r n t o ' w i n d b a c k ' t o t h e a b o r t h a n d l e r .
* /
/* ******************** CR,LR,CCR,MSR ********** */
mfctr r3
mflr r4
mfcr r5
mfxer r6
std r3 , _ C T R ( r7 )
std r4 , _ L I N K ( r7 )
std r5 , _ C C R ( r7 )
std r6 , _ X E R ( r7 )
2013-08-09 11:29:31 +04:00
/* ******************** TAR, PPR, DSCR ********** */
mfspr r3 , S P R N _ T A R
mfspr r4 , S P R N _ P P R
mfspr r5 , S P R N _ D S C R
std r3 , T H R E A D _ T M _ T A R ( r12 )
std r4 , T H R E A D _ T M _ P P R ( r12 )
std r5 , T H R E A D _ T M _ D S C R ( r12 )
2013-02-13 20:21:35 +04:00
/ * MSR a n d f l a g s : W e d o n ' t c h a n g e C R s , a n d w e d o n ' t n e e d t o a l t e r
* MSR.
* /
/ * TM r e g s , i n c l T E X A S R - - t h e s e l i v e i n t h r e a d _ s t r u c t . N o t e t h e y ' v e
* been u p d a t e d b y t h e t r e c l a i m , t o e x p l a i n t o u s e r l a n d t h e f a i l u r e
* cause ( a b o r t e d ) .
* /
mfspr r0 , S P R N _ T E X A S R
mfspr r3 , S P R N _ T F H A R
mfspr r4 , S P R N _ T F I A R
std r0 , T H R E A D _ T M _ T E X A S R ( r12 )
std r3 , T H R E A D _ T M _ T F H A R ( r12 )
std r4 , T H R E A D _ T M _ T F I A R ( r12 )
/* AMR and PPR are checkpointed too, but are unsupported by Linux. */
/* Restore original MSR/IRQ state & clear TM mode */
ld r14 , T M _ F R A M E _ L 0 ( r1 ) / * O r i g M S R * /
li r15 , 0
rldimi r14 , r15 , M S R _ T S _ L G , ( 6 3 - M S R _ T S _ L G ) - 1
mtmsrd r14
REST_ N V G P R S ( r1 )
addi r1 , r1 , T M _ F R A M E _ S I Z E
ld r4 , 8 ( r1 )
ld r0 , 1 6 ( r1 )
mtcr r4
mtlr r0
ld r2 , 4 0 ( r1 )
blr
/ * void t m _ r e c h e c k p o i n t ( s t r u c t t h r e a d _ s t r u c t * t h r e a d ,
* unsigned l o n g o r i g _ m s r )
* - Restore t h e c h e c k p o i n t e d r e g i s t e r s t a t e s a v e d b y t m _ r e c l a i m
* when w e s w i t c h _ t o a p r o c e s s .
*
* Call w i t h I R Q s o f f , s t a c k s g e t a l l o u t o f s y n c f o r
* some p e r i o d s i n h e r e !
* /
_ GLOBAL( t m _ r e c h e c k p o i n t )
mfcr r5
mflr r0
std r5 , 8 ( r1 )
std r0 , 1 6 ( r1 )
std r2 , 4 0 ( r1 )
stdu r1 , - T M _ F R A M E _ S I Z E ( r1 )
/ * We' v e a s t r u c t p t _ r e g s a t [ r1 + S T A C K _ F R A M E _ O V E R H E A D ] .
* This i s u s e d f o r b a c k i n g u p t h e N V G P R s :
* /
SAVE_ N V G P R S ( r1 )
std r1 , P A C A R 1 ( r13 )
/* Load complete register state from ts_ckpt* registers */
addi r7 , r3 , P T _ C K P T _ R E G S / * T h r e a d ' s c k p t _ r e g s * /
/ * Make r7 l o o k l i k e a n e x c e p t i o n f r a m e s o t h a t w e
* can u s e t h e n e a t G P R x ( n ) m a c r o s . r7 i s n o w N O T a p t _ r e g s p t r !
* /
subi r7 , r7 , S T A C K _ F R A M E _ O V E R H E A D
SET_ S C R A T C H 0 ( r1 )
mfmsr r6
/* R4 = original MSR to indicate whether thread used FP/Vector etc. */
/* Enable FP/vec in MSR if necessary! */
lis r5 , M S R _ V E C @h
ori r5 , r5 , M S R _ F P
and. r5 , r4 , r5
beq r e s t o r e _ g p r s / * i f n e i t h e r , s k i p b o t h * /
# ifdef C O N F I G _ V S X
BEGIN_ F T R _ S E C T I O N
oris r5 , r5 , M S R _ V S X @h
END_ F T R _ S E C T I O N _ I F S E T ( C P U _ F T R _ V S X )
# endif
or r5 , r6 , r5 / * S e t M S R . F P + . V S X / . V E C * /
mtmsr r5
2013-04-09 10:18:55 +04:00
# ifdef C O N F I G _ A L T I V E C
2013-02-13 20:21:35 +04:00
/ * FP a n d V E C r e g i s t e r s : T h e s e a r e r e c h e c k p o i n t e d f r o m t h r e a d . f p r [ ]
* and t h r e a d . v r [ ] r e s p e c t i v e l y . T h e t h r e a d . t r a n s a c t _ f p r [ ] v e r s i o n
* is m o r e m o d e r n , a n d w i l l b e l o a d e d s u b s e q u e n t l y b y a n y F P U n a v a i l a b l e
* trap.
* /
andis. r0 , r4 , M S R _ V E C @h
beq d o n t _ r e s t o r e _ v e c
li r5 , T H R E A D _ V S C R
lvx v r0 , r3 , r5
mtvscr v r0
REST_ 3 2 V R S ( 0 , r5 , r3 ) / * r5 s c r a t c h , r3 T H R E A D p t r * /
2013-08-05 08:13:16 +04:00
dont_restore_vec :
2013-02-13 20:21:35 +04:00
ld r5 , T H R E A D _ V R S A V E ( r3 )
mtspr S P R N _ V R S A V E , r5
2013-04-09 10:18:55 +04:00
# endif
2013-02-13 20:21:35 +04:00
andi. r0 , r4 , M S R _ F P
beq d o n t _ r e s t o r e _ f p
lfd f r0 , T H R E A D _ F P S C R ( r3 )
MTFSF_ L ( f r0 )
REST_ 3 2 F P R S _ V S R S ( 0 , R 4 , R 3 )
dont_restore_fp :
mtmsr r6 / * F P / V e c o f f a g a i n ! * /
restore_gprs :
2013-08-09 11:29:31 +04:00
/* ******************** TAR, PPR, DSCR ********** */
ld r4 , T H R E A D _ T M _ T A R ( r3 )
ld r5 , T H R E A D _ T M _ P P R ( r3 )
ld r6 , T H R E A D _ T M _ D S C R ( r3 )
mtspr S P R N _ T A R , r4
mtspr S P R N _ P P R , r5
mtspr S P R N _ D S C R , r6
2013-02-13 20:21:35 +04:00
/* ******************** CR,LR,CCR,MSR ********** */
ld r3 , _ C T R ( r7 )
ld r4 , _ L I N K ( r7 )
ld r5 , _ C C R ( r7 )
ld r6 , _ X E R ( r7 )
mtctr r3
mtlr r4
mtcr r5
mtxer r6
2013-06-28 12:17:09 +04:00
/ * Clear t h e M S R R I s i n c e w e a r e a b o u t t o c h a n g e R 1 . E E i s a l r e a d y o f f
2013-02-13 20:21:35 +04:00
* /
2013-06-28 12:17:09 +04:00
li r4 , 0
mtmsrd r4 , 1
2013-02-13 20:21:35 +04:00
REST_ 4 G P R S ( 0 , r7 ) / * G P R 0 - 3 * /
REST_ G P R ( 4 , r7 ) / * G P R 4 - 6 * /
REST_ G P R ( 5 , r7 )
REST_ G P R ( 6 , r7 )
REST_ 4 G P R S ( 8 , r7 ) / * G P R 8 - 1 1 * /
REST_ 2 G P R S ( 1 2 , r7 ) / * G P R 1 2 - 1 3 * /
REST_ N V G P R S ( r7 ) / * G P R 1 4 - 3 1 * /
ld r7 , G P R 7 ( r7 ) / * G P R 7 * /
/* Commit register state as checkpointed state: */
TRECHKPT
/ * Our t r a n s a c t i o n a l s t a t e h a s n o w c h a n g e d .
*
* Now j u s t g e t o u t o f h e r e . T r a n s a c t i o n a l ( c u r r e n t ) s t a t e w i l l b e
* updated o n c e r e s t o r e i s c a l l e d o n t h e r e t u r n p a t h i n t h e _ s w i t c h - e d
* - to p r o c e s s .
* /
GET_ P A C A ( r13 )
GET_ S C R A T C H 0 ( r1 )
2013-06-28 12:17:09 +04:00
/* R1 is restored, so we are recoverable again. EE is still off */
li r4 , M S R _ R I
mtmsrd r4 , 1
2013-02-13 20:21:35 +04:00
REST_ N V G P R S ( r1 )
addi r1 , r1 , T M _ F R A M E _ S I Z E
ld r4 , 8 ( r1 )
ld r0 , 1 6 ( r1 )
mtcr r4
mtlr r0
ld r2 , 4 0 ( r1 )
blr
/* ****************************************************************** */
