2017-10-24 14:42:02 +08:00
/ / SPDX- L i c e n s e - I d e n t i f i e r : G P L - 2 . 0
/ / Copyright ( C ) 2 0 0 5 - 2 0 1 7 A n d e s T e c h n o l o g y C o r p o r a t i o n
# include < l i n u x / l i n k a g e . h >
# include < a s m / u n i s t d . h >
# include < a s m / a s s e m b l e r . h >
# include < a s m / n d s32 . h >
# include < a s m / a s m - o f f s e t s . h >
# include < a s m / t h r e a d _ i n f o . h >
# include < a s m / c u r r e n t . h >
2018-11-22 11:14:34 +08:00
# include < a s m / f p u . h >
2017-10-24 14:42:02 +08:00
# ifdef C O N F I G _ H W Z O L
.macro pop_zol
mtusr $ r14 , $ L B
mtusr $ r15 , $ L E
mtusr $ r16 , $ L C
.endm
# endif
.macro restore_user_regs_first
setgie. d
isb
2018-11-22 11:14:34 +08:00
# if d e f i n e d ( C O N F I G _ F P U )
addi $ s p , $ s p , O S P _ O F F S E T
lmw. a d m $ r12 , [ $ s p ] , $ r25 , #0x0
sethi $ p0 , h i 2 0 ( h a s _ f p u )
lbsi $ p0 , [ $ p0 + l o 1 2 ( h a s _ f p u ) ]
beqz $ p0 , 2 f
mtsr $ r25 , $ F U C O P _ C T L
2 :
# else
2017-10-24 14:42:02 +08:00
addi $ s p , $ s p , F U C O P _ C T L _ O F F S E T
lmw. a d m $ r12 , [ $ s p ] , $ r24 , #0x0
2018-11-22 11:14:34 +08:00
# endif
2017-10-24 14:42:02 +08:00
mtsr $ r12 , $ S P _ U S R
mtsr $ r13 , $ I P C
# ifdef C O N F I G _ H W Z O L
pop_ z o l
# endif
mtsr $ r19 , $ P S W
mtsr $ r20 , $ I P S W
mtsr $ r21 , $ P _ I P S W
mtsr $ r22 , $ P _ I P C
mtsr $ r23 , $ P _ P 0
mtsr $ r24 , $ P _ P 1
lmw. a d m $ s p , [ $ s p ] , $ s p , #0xe
.endm
.macro restore_user_regs_last
pop $ p0
cmovn $ s p , $ p0 , $ p0
iret
nop
.endm
.macro restore_user_regs
restore_ u s e r _ r e g s _ f i r s t
lmw. a d m $ r0 , [ $ s p ] , $ r25 , #0x0
addi $ s p , $ s p , O S P _ O F F S E T
restore_ u s e r _ r e g s _ l a s t
.endm
.macro fast_restore_user_regs
restore_ u s e r _ r e g s _ f i r s t
lmw. a d m $ r1 , [ $ s p ] , $ r25 , #0x0
addi $ s p , $ s p , O S P _ O F F S E T - 4
restore_ u s e r _ r e g s _ l a s t
.endm
2019-10-15 21:18:00 +02:00
# ifdef C O N F I G _ P R E E M P T I O N
2017-10-24 14:42:02 +08:00
.macro preempt_stop
.endm
# else
.macro preempt_stop
setgie. d
isb
.endm
# define r e s u m e _ k e r n e l n o _ w o r k _ p e n d i n g
# endif
ENTRY( r e t _ f r o m _ e x c e p t i o n )
preempt_ s t o p
ENTRY( r e t _ f r o m _ i n t r )
/ *
* judge K e r n e l o r u s e r m o d e
*
* /
lwi $ p0 , [ $ s p + ( #I P S W _ O F F S E T ) ] ! C h e c k i f i n n e s t e d i n t e r r u p t
andi $ p0 , $ p0 , #P S W _ m s k I N T L
bnez $ p0 , r e s u m e _ k e r n e l ! d o n e w i t h i r e t
j r e s u m e _ u s e r s p a c e
/ *
* This i s t h e f a s t s y s c a l l r e t u r n p a t h . W e d o a s l i t t l e a s
* possible h e r e , a n d t h i s i n c l u d e s s a v i n g $ r0 b a c k i n t o t h e S V C
* stack.
* fixed : tsk - $ r25 , s y s c a l l # - $ r 7 , s y s c a l l t a b l e p o i n t e r - $ r8
* /
ENTRY( r e t _ f a s t _ s y s c a l l )
gie_ d i s a b l e
lwi $ r1 , [ t s k + #T S K _ T I _ F L A G S ]
andi $ p1 , $ r1 , #_ T I F _ W O R K _ M A S K
bnez $ p1 , f a s t _ w o r k _ p e n d i n g
fast_ r e s t o r e _ u s e r _ r e g s ! i r e t
/ *
* Ok, w e n e e d t o d o e x t r a p r o c e s s i n g ,
* enter t h e s l o w p a t h r e t u r n i n g f r o m s y s c a l l , w h i l e p e n d i n g w o r k .
* /
fast_work_pending :
swi $ r0 , [ $ s p + ( #R 0 _ O F F S E T ) ] ! w h a t i s d i f f e r e n t f r o m r e t _ f r o m _ e x c e p t i o n
work_pending :
andi $ p1 , $ r1 , #_ T I F _ N E E D _ R E S C H E D
bnez $ p1 , w o r k _ r e s c h e d
2020-10-09 15:18:43 -06:00
andi $ p1 , $ r1 , #_ T I F _ S I G P E N D I N G | # _ T I F _ N O T I F Y _ R E S U M E | # _ T I F _ N O T I F Y _ S I G N A L
2017-10-24 14:42:02 +08:00
beqz $ p1 , n o _ w o r k _ p e n d i n g
move $ r0 , $ s p ! ' r e g s '
gie_ e n a b l e
bal d o _ n o t i f y _ r e s u m e
b r e t _ s l o w _ s y s c a l l
work_resched :
bal s c h e d u l e ! p a t h , r e t u r n t o u s e r m o d e
/ *
* " slow" s y s c a l l r e t u r n p a t h .
* /
ENTRY( r e s u m e _ u s e r s p a c e )
ENTRY( r e t _ s l o w _ s y s c a l l )
gie_ d i s a b l e
lwi $ p0 , [ $ s p + ( #I P S W _ O F F S E T ) ] ! C h e c k i f i n n e s t e d i n t e r r u p t
andi $ p0 , $ p0 , #P S W _ m s k I N T L
bnez $ p0 , n o _ w o r k _ p e n d i n g ! d o n e w i t h i r e t
lwi $ r1 , [ t s k + #T S K _ T I _ F L A G S ]
andi $ p1 , $ r1 , #_ T I F _ W O R K _ M A S K
bnez $ p1 , w o r k _ p e n d i n g ! h a n d l e w o r k _ r e s c h e d , s i g _ p e n d
no_work_pending :
# ifdef C O N F I G _ T R A C E _ I R Q F L A G S
lwi $ p0 , [ $ s p + ( #I P S W _ O F F S E T ) ]
andi $ p0 , $ p0 , #0x1
2018-08-23 15:05:46 +08:00
la $ r10 , _ _ t r a c e _ h a r d i r q s _ o f f
la $ r9 , _ _ t r a c e _ h a r d i r q s _ o n
2017-10-24 14:42:02 +08:00
cmovz $ r9 , $ p0 , $ r10
jral $ r9
# endif
restore_ u s e r _ r e g s ! r e t u r n f r o m i r e t
/ *
* preemptive k e r n e l
* /
2019-10-15 21:18:00 +02:00
# ifdef C O N F I G _ P R E E M P T I O N
2017-10-24 14:42:02 +08:00
resume_kernel :
gie_ d i s a b l e
lwi $ t 0 , [ t s k + #T S K _ T I _ P R E E M P T ]
bnez $ t 0 , n o _ w o r k _ p e n d i n g
2019-03-11 22:47:45 +00:00
2017-10-24 14:42:02 +08:00
lwi $ t 0 , [ t s k + #T S K _ T I _ F L A G S ]
andi $ p1 , $ t 0 , #_ T I F _ N E E D _ R E S C H E D
beqz $ p1 , n o _ w o r k _ p e n d i n g
lwi $ t 0 , [ $ s p + ( #I P S W _ O F F S E T ) ] ! I n t e r r u p t s o f f ?
andi $ t 0 , $ t 0 , #1
beqz $ t 0 , n o _ w o r k _ p e n d i n g
jal p r e e m p t _ s c h e d u l e _ i r q
2019-03-11 22:47:45 +00:00
b n o _ w o r k _ p e n d i n g
2017-10-24 14:42:02 +08:00
# endif
/ *
* This i s h o w w e r e t u r n f r o m a f o r k .
* /
ENTRY( r e t _ f r o m _ f o r k )
bal s c h e d u l e _ t a i l
beqz $ r6 , 1 f ! r6 s t o r e s f n f o r k e r n e l t h r e a d
move $ r0 , $ r7 ! p r e p a r e k e r n e l t h r e a d a r g
jral $ r6
1 :
lwi $ r1 , [ t s k + #T S K _ T I _ F L A G S ] ! c h e c k f o r s y s c a l l t r a c i n g
andi $ p1 , $ r1 , #_ T I F _ W O R K _ S Y S C A L L _ L E A V E ! a r e w e t r a c i n g s y s c a l l s ?
beqz $ p1 , r e t _ s l o w _ s y s c a l l
move $ r0 , $ s p
bal s y s c a l l _ t r a c e _ l e a v e
b r e t _ s l o w _ s y s c a l l
