2009-06-03 01:17:37 +04:00
# include < a s m / p r o c e s s o r . h >
2005-09-26 10:04:21 +04:00
# include < a s m / p p c _ a s m . h >
2005-10-10 16:20:10 +04:00
# include < a s m / r e g . h >
2009-06-03 01:17:37 +04:00
# include < a s m / a s m - o f f s e t s . h >
# include < a s m / c p u t a b l e . h >
# include < a s m / t h r e a d _ i n f o . h >
# include < a s m / p a g e . h >
2010-11-18 18:06:17 +03:00
# include < a s m / p t r a c e . h >
2009-06-03 01:17:37 +04:00
/ *
* load_ u p _ a l t i v e c ( u n u s e d , u n u s e d , t s k )
* Disable V M X f o r t h e t a s k w h i c h h a d i t p r e v i o u s l y ,
* and s a v e i t s v e c t o r r e g i s t e r s i n i t s t h r e a d _ s t r u c t .
* Enables t h e V M X f o r u s e i n t h e k e r n e l o n r e t u r n .
* On S M P w e k n o w t h e V M X i s f r e e , s i n c e w e g i v e i t u p e v e r y
* switch ( i e , n o l a z y s a v e o f t h e v e c t o r r e g i s t e r s ) .
* /
_ GLOBAL( l o a d _ u p _ a l t i v e c )
mfmsr r5 / * g r a b t h e c u r r e n t M S R * /
oris r5 ,r5 ,M S R _ V E C @h
MTMSRD( r5 ) / * e n a b l e u s e o f A l t i V e c n o w * /
isync
/ *
* For S M P , w e d o n ' t d o l a z y V M X s w i t c h i n g b e c a u s e i t j u s t g e t s t o o
* horrendously c o m p l e x , e s p e c i a l l y w h e n a t a s k s w i t c h e s f r o m o n e C P U
* to a n o t h e r . I n s t e a d w e c a l l g i v e u p _ a l t v e c i n s w i t c h _ t o .
* VRSAVE i s n ' t d e a l t w i t h h e r e , t h a t i s d o n e i n t h e n o r m a l c o n t e x t
* switch c o d e . N o t e t h a t w e c o u l d r e l y o n v r s a v e v a l u e t o e v e n t u a l l y
* avoid s a v i n g a l l o f t h e V R E G s h e r e . . .
* /
# ifndef C O N F I G _ S M P
LOAD_ R E G _ A D D R B A S E ( r3 , l a s t _ t a s k _ u s e d _ a l t i v e c )
toreal( r3 )
PPC_ L L r4 ,A D D R O F F ( l a s t _ t a s k _ u s e d _ a l t i v e c ) ( r3 )
PPC_ L C M P I 0 ,r4 ,0
beq 1 f
/* Save VMX state to last_task_used_altivec's THREAD struct */
toreal( r4 )
addi r4 ,r4 ,T H R E A D
SAVE_ 3 2 V R S ( 0 ,r5 ,r4 )
mfvscr v r0
li r10 ,T H R E A D _ V S C R
stvx v r0 ,r10 ,r4
/* Disable VMX for last_task_used_altivec */
PPC_ L L r5 ,P T _ R E G S ( r4 )
toreal( r5 )
PPC_ L L r4 ,_ M S R - S T A C K _ F R A M E _ O V E R H E A D ( r5 )
lis r10 ,M S R _ V E C @h
andc r4 ,r4 ,r10
PPC_ S T L r4 ,_ M S R - S T A C K _ F R A M E _ O V E R H E A D ( r5 )
1 :
# endif / * C O N F I G _ S M P * /
/ * Hack : if w e g e t a n a l t i v e c u n a v a i l a b l e t r a p w i t h V R S A V E
* set t o a l l z e r o s , w e a s s u m e t h i s i s a b r o k e n a p p l i c a t i o n
* that f a i l s t o s e t i t p r o p e r l y , a n d t h u s w e s w i t c h i t t o
* all 1 ' s
* /
mfspr r4 ,S P R N _ V R S A V E
2009-12-08 21:45:45 +03:00
cmpwi 0 ,r4 ,0
2009-06-03 01:17:37 +04:00
bne+ 1 f
li r4 ,- 1
mtspr S P R N _ V R S A V E ,r4
1 :
/* enable use of VMX after return */
# ifdef C O N F I G _ P P C 3 2
2009-07-15 00:52:54 +04:00
mfspr r5 ,S P R N _ S P R G _ T H R E A D / * c u r r e n t t a s k ' s T H R E A D ( p h y s ) * /
2009-06-03 01:17:37 +04:00
oris r9 ,r9 ,M S R _ V E C @h
# else
ld r4 ,P A C A C U R R E N T ( r13 )
addi r5 ,r4 ,T H R E A D / * G e t T H R E A D * /
oris r12 ,r12 ,M S R _ V E C @h
std r12 ,_ M S R ( r1 )
# endif
li r4 ,1
li r10 ,T H R E A D _ V S C R
stw r4 ,T H R E A D _ U S E D _ V R ( r5 )
lvx v r0 ,r10 ,r5
mtvscr v r0
REST_ 3 2 V R S ( 0 ,r4 ,r5 )
# ifndef C O N F I G _ S M P
2009-07-10 15:17:36 +04:00
/* Update last_task_used_altivec to 'current' */
2009-06-03 01:17:37 +04:00
subi r4 ,r5 ,T H R E A D / * B a c k t o ' c u r r e n t ' * /
fromreal( r4 )
2009-07-10 15:17:36 +04:00
PPC_ S T L r4 ,A D D R O F F ( l a s t _ t a s k _ u s e d _ a l t i v e c ) ( r3 )
2009-06-03 01:17:37 +04:00
# endif / * C O N F I G _ S M P * /
/* restore registers and return */
blr
2012-04-16 00:56:45 +04:00
_ GLOBAL( g i v e u p _ a l t i v e c _ n o t a s k )
mfmsr r3
andis. r4 ,r3 ,M S R _ V E C @h
bnelr / * A l r e a d y e n a b l e d ? * /
oris r3 ,r3 ,M S R _ V E C @h
SYNC
MTMSRD( r3 ) / * e n a b l e u s e o f V M X n o w * /
isync
blr
2009-06-03 01:17:37 +04:00
/ *
* giveup_ a l t i v e c ( t s k )
* Disable V M X f o r t h e t a s k g i v e n a s t h e a r g u m e n t ,
* and s a v e t h e v e c t o r r e g i s t e r s i n i t s t h r e a d _ s t r u c t .
* Enables t h e V M X f o r u s e i n t h e k e r n e l o n r e t u r n .
* /
_ GLOBAL( g i v e u p _ a l t i v e c )
mfmsr r5
oris r5 ,r5 ,M S R _ V E C @h
SYNC
MTMSRD( r5 ) / * e n a b l e u s e o f V M X n o w * /
isync
PPC_ L C M P I 0 ,r3 ,0
2011-05-09 01:20:19 +04:00
beqlr / * i f n o p r e v i o u s o w n e r , d o n e * /
2009-06-03 01:17:37 +04:00
addi r3 ,r3 ,T H R E A D / * w a n t T H R E A D o f t a s k * /
PPC_ L L r5 ,P T _ R E G S ( r3 )
PPC_ L C M P I 0 ,r5 ,0
SAVE_ 3 2 V R S ( 0 ,r4 ,r3 )
mfvscr v r0
li r4 ,T H R E A D _ V S C R
stvx v r0 ,r4 ,r3
beq 1 f
PPC_ L L r4 ,_ M S R - S T A C K _ F R A M E _ O V E R H E A D ( r5 )
# ifdef C O N F I G _ V S X
BEGIN_ F T R _ S E C T I O N
lis r3 ,( M S R _ V E C | M S R _ V S X ) @h
FTR_ S E C T I O N _ E L S E
lis r3 ,M S R _ V E C @h
ALT_ F T R _ S E C T I O N _ E N D _ I F S E T ( C P U _ F T R _ V S X )
# else
lis r3 ,M S R _ V E C @h
# endif
andc r4 ,r4 ,r3 / * d i s a b l e F P f o r p r e v i o u s t a s k * /
PPC_ S T L r4 ,_ M S R - S T A C K _ F R A M E _ O V E R H E A D ( r5 )
1 :
# ifndef C O N F I G _ S M P
li r5 ,0
LOAD_ R E G _ A D D R B A S E ( r4 ,l a s t _ t a s k _ u s e d _ a l t i v e c )
PPC_ S T L r5 ,A D D R O F F ( l a s t _ t a s k _ u s e d _ a l t i v e c ) ( r4 )
# endif / * C O N F I G _ S M P * /
blr
# ifdef C O N F I G _ V S X
# ifdef C O N F I G _ P P C 3 2
# error T h i s a s m c o d e i s n ' t r e a d y f o r 3 2 - b i t k e r n e l s
# endif
/ *
* load_ u p _ v s x ( u n u s e d , u n u s e d , t s k )
* Disable V S X f o r t h e t a s k w h i c h h a d i t p r e v i o u s l y ,
* and s a v e i t s v e c t o r r e g i s t e r s i n i t s t h r e a d _ s t r u c t .
* Reuse t h e f p a n d v s x s a v e s , b u t f i r s t c h e c k t o s e e i f t h e y h a v e
* been s a v e d a l r e a d y .
* /
_ GLOBAL( l o a d _ u p _ v s x )
/* Load FP and VSX registers if they haven't been done yet */
andi. r5 ,r12 ,M S R _ F P
beql+ l o a d _ u p _ f p u / * s k i p i f a l r e a d y l o a d e d * /
andis. r5 ,r12 ,M S R _ V E C @h
beql+ l o a d _ u p _ a l t i v e c / * s k i p i f a l r e a d y l o a d e d * /
# ifndef C O N F I G _ S M P
ld r3 ,l a s t _ t a s k _ u s e d _ v s x @got(r2)
ld r4 ,0 ( r3 )
cmpdi 0 ,r4 ,0
beq 1 f
/* Disable VSX for last_task_used_vsx */
addi r4 ,r4 ,T H R E A D
ld r5 ,P T _ R E G S ( r4 )
ld r4 ,_ M S R - S T A C K _ F R A M E _ O V E R H E A D ( r5 )
lis r6 ,M S R _ V S X @h
andc r6 ,r4 ,r6
std r6 ,_ M S R - S T A C K _ F R A M E _ O V E R H E A D ( r5 )
1 :
# endif / * C O N F I G _ S M P * /
ld r4 ,P A C A C U R R E N T ( r13 )
addi r4 ,r4 ,T H R E A D / * G e t T H R E A D * /
li r6 ,1
stw r6 ,T H R E A D _ U S E D _ V S R ( r4 ) / * . . . a l s o s e t t h r e a d u s e d v s r * /
/* enable use of VSX after return */
oris r12 ,r12 ,M S R _ V S X @h
std r12 ,_ M S R ( r1 )
# ifndef C O N F I G _ S M P
2009-07-10 15:17:36 +04:00
/* Update last_task_used_vsx to 'current' */
2009-06-03 01:17:37 +04:00
ld r4 ,P A C A C U R R E N T ( r13 )
std r4 ,0 ( r3 )
# endif / * C O N F I G _ S M P * /
b f a s t _ e x c e p t i o n _ r e t u r n
/ *
* _ _ giveup_ v s x ( t s k )
* Disable V S X f o r t h e t a s k g i v e n a s t h e a r g u m e n t .
* Does N O T s a v e v s x r e g i s t e r s .
* Enables t h e V S X f o r u s e i n t h e k e r n e l o n r e t u r n .
* /
_ GLOBAL( _ _ g i v e u p _ v s x )
mfmsr r5
oris r5 ,r5 ,M S R _ V S X @h
mtmsrd r5 / * e n a b l e u s e o f V S X n o w * /
isync
cmpdi 0 ,r3 ,0
beqlr- / * i f n o p r e v i o u s o w n e r , d o n e * /
addi r3 ,r3 ,T H R E A D / * w a n t T H R E A D o f t a s k * /
ld r5 ,P T _ R E G S ( r3 )
cmpdi 0 ,r5 ,0
beq 1 f
ld r4 ,_ M S R - S T A C K _ F R A M E _ O V E R H E A D ( r5 )
lis r3 ,M S R _ V S X @h
andc r4 ,r4 ,r3 / * d i s a b l e V S X f o r p r e v i o u s t a s k * /
std r4 ,_ M S R - S T A C K _ F R A M E _ O V E R H E A D ( r5 )
1 :
# ifndef C O N F I G _ S M P
li r5 ,0
ld r4 ,l a s t _ t a s k _ u s e d _ v s x @got(r2)
std r5 ,0 ( r4 )
# endif / * C O N F I G _ S M P * /
blr
# endif / * C O N F I G _ V S X * /
2005-09-26 10:04:21 +04:00
/ *
* The r o u t i n e s b e l o w a r e i n a s s e m b l e r s o w e c a n c l o s e l y c o n t r o l t h e
* usage o f f l o a t i n g - p o i n t r e g i s t e r s . T h e s e r o u t i n e s m u s t b e c a l l e d
* with p r e e m p t d i s a b l e d .
* /
# ifdef C O N F I G _ P P C 3 2
.data
fpzero :
.long 0
fpone :
.long 0x3f800000 /* 1.0 in single-precision FP */
fphalf :
.long 0x3f000000 /* 0.5 in single-precision FP */
# define L D C O N S T ( f r , n a m e ) \
lis r11 ,n a m e @ha; \
lfs f r ,n a m e @l(r11)
# else
.section " .toc " , " aw"
fpzero :
.tc _ 0 _ 0 [ T C ] ,0
fpone :
.tc _ 3 f f00 0 0 0 _ 0 [ T C ] ,0 x3 f f00 0 0 0 0 0 0 0 0 0 0 0 / * 1 . 0 * /
fphalf :
.tc _ 3 f e 0 0 0 0 0 _ 0 [ T C ] ,0 x3 f e 0 0 0 0 0 0 0 0 0 0 0 0 0 / * 0 . 5 * /
# define L D C O N S T ( f r , n a m e ) \
lfd f r ,n a m e @toc(r2)
# endif
.text
/ *
* Internal r o u t i n e t o e n a b l e f l o a t i n g p o i n t a n d s e t F P S C R t o 0 .
* Don' t c a l l i t f r o m C ; it doesn't use the normal calling convention.
* /
fpenable :
# ifdef C O N F I G _ P P C 3 2
stwu r1 ,- 6 4 ( r1 )
# else
stdu r1 ,- 6 4 ( r1 )
# endif
mfmsr r10
ori r11 ,r10 ,M S R _ F P
mtmsr r11
isync
stfd f r0 ,2 4 ( r1 )
stfd f r1 ,1 6 ( r1 )
stfd f r31 ,8 ( r1 )
LDCONST( f r1 , f p z e r o )
mffs f r31
2006-06-10 14:18:39 +04:00
MTFSF_ L ( f r1 )
2005-09-26 10:04:21 +04:00
blr
fpdisable :
mtlr r12
2006-06-10 14:18:39 +04:00
MTFSF_ L ( f r31 )
2005-09-26 10:04:21 +04:00
lfd f r31 ,8 ( r1 )
lfd f r1 ,1 6 ( r1 )
lfd f r0 ,2 4 ( r1 )
mtmsr r10
isync
addi r1 ,r1 ,6 4
blr
/ *
* Vector a d d , f l o a t i n g p o i n t .
* /
_ GLOBAL( v a d d f p )
mflr r12
bl f p e n a b l e
li r0 ,4
mtctr r0
li r6 ,0
1 : lfsx f r0 ,r4 ,r6
lfsx f r1 ,r5 ,r6
fadds f r0 ,f r0 ,f r1
stfsx f r0 ,r3 ,r6
addi r6 ,r6 ,4
bdnz 1 b
b f p d i s a b l e
/ *
* Vector s u b t r a c t , f l o a t i n g p o i n t .
* /
_ GLOBAL( v s u b f p )
mflr r12
bl f p e n a b l e
li r0 ,4
mtctr r0
li r6 ,0
1 : lfsx f r0 ,r4 ,r6
lfsx f r1 ,r5 ,r6
fsubs f r0 ,f r0 ,f r1
stfsx f r0 ,r3 ,r6
addi r6 ,r6 ,4
bdnz 1 b
b f p d i s a b l e
/ *
* Vector m u l t i p l y a n d a d d , f l o a t i n g p o i n t .
* /
_ GLOBAL( v m a d d f p )
mflr r12
bl f p e n a b l e
stfd f r2 ,3 2 ( r1 )
li r0 ,4
mtctr r0
li r7 ,0
1 : lfsx f r0 ,r4 ,r7
lfsx f r1 ,r5 ,r7
lfsx f r2 ,r6 ,r7
fmadds f r0 ,f r0 ,f r2 ,f r1
stfsx f r0 ,r3 ,r7
addi r7 ,r7 ,4
bdnz 1 b
lfd f r2 ,3 2 ( r1 )
b f p d i s a b l e
/ *
* Vector n e g a t i v e m u l t i p l y a n d s u b t r a c t , f l o a t i n g p o i n t .
* /
_ GLOBAL( v n m s u b f p )
mflr r12
bl f p e n a b l e
stfd f r2 ,3 2 ( r1 )
li r0 ,4
mtctr r0
li r7 ,0
1 : lfsx f r0 ,r4 ,r7
lfsx f r1 ,r5 ,r7
lfsx f r2 ,r6 ,r7
fnmsubs f r0 ,f r0 ,f r2 ,f r1
stfsx f r0 ,r3 ,r7
addi r7 ,r7 ,4
bdnz 1 b
lfd f r2 ,3 2 ( r1 )
b f p d i s a b l e
/ *
* Vector r e c i p r o c a l e s t i m a t e . W e j u s t c o m p u t e 1 . 0 / x .
* r3 - > d e s t i n a t i o n , r4 - > s o u r c e .
* /
_ GLOBAL( v r e f p )
mflr r12
bl f p e n a b l e
li r0 ,4
LDCONST( f r1 , f p o n e )
mtctr r0
li r6 ,0
1 : lfsx f r0 ,r4 ,r6
fdivs f r0 ,f r1 ,f r0
stfsx f r0 ,r3 ,r6
addi r6 ,r6 ,4
bdnz 1 b
b f p d i s a b l e
/ *
* Vector r e c i p r o c a l s q u a r e - r o o t e s t i m a t e , f l o a t i n g p o i n t .
* We u s e t h e f r s q r t e i n s t r u c t i o n f o r t h e i n i t i a l e s t i m a t e f o l l o w e d
* by 2 i t e r a t i o n s o f N e w t o n - R a p h s o n t o g e t s u f f i c i e n t a c c u r a c y .
* r3 - > d e s t i n a t i o n , r4 - > s o u r c e .
* /
_ GLOBAL( v r s q r t e f p )
mflr r12
bl f p e n a b l e
stfd f r2 ,3 2 ( r1 )
stfd f r3 ,4 0 ( r1 )
stfd f r4 ,4 8 ( r1 )
stfd f r5 ,5 6 ( r1 )
li r0 ,4
LDCONST( f r4 , f p o n e )
LDCONST( f r5 , f p h a l f )
mtctr r0
li r6 ,0
1 : lfsx f r0 ,r4 ,r6
frsqrte f r1 ,f r0 / * r = f r s q r t e ( s ) * /
fmuls f r3 ,f r1 ,f r0 / * r * s * /
fmuls f r2 ,f r1 ,f r5 / * r * 0 . 5 * /
fnmsubs f r3 ,f r1 ,f r3 ,f r4 / * 1 - s * r * r * /
fmadds f r1 ,f r2 ,f r3 ,f r1 / * r = r + 0 . 5 * r * ( 1 - s * r * r ) * /
fmuls f r3 ,f r1 ,f r0 / * r * s * /
fmuls f r2 ,f r1 ,f r5 / * r * 0 . 5 * /
fnmsubs f r3 ,f r1 ,f r3 ,f r4 / * 1 - s * r * r * /
fmadds f r1 ,f r2 ,f r3 ,f r1 / * r = r + 0 . 5 * r * ( 1 - s * r * r ) * /
stfsx f r1 ,r3 ,r6
addi r6 ,r6 ,4
bdnz 1 b
lfd f r5 ,5 6 ( r1 )
lfd f r4 ,4 8 ( r1 )
lfd f r3 ,4 0 ( r1 )
lfd f r2 ,3 2 ( r1 )
b f p d i s a b l e
