2006-02-08 02:53:50 -08:00
/ * sun4 v _ i v e c . S : S u n 4 v i n t e r r u p t v e c t o r h a n d l i n g .
*
* Copyright ( C ) 2 0 0 6 < d a v e m @davemloft.net>
* /
# include < a s m / c p u d a t a . h >
# include < a s m / i n t r _ q u e u e . h >
2006-06-20 01:20:00 -07:00
# include < a s m / p i l . h >
2006-02-08 02:53:50 -08:00
.text
.align 32
sun4v_cpu_mondo :
/ * Head o f f s e t i n % g 2 , t a i l o f f s e t i n % g 4 .
* If t h e y a r e t h e s a m e , n o w o r k .
* /
mov I N T R Q _ C P U _ M O N D O _ H E A D , % g 2
ldxa [ % g 2 ] A S I _ Q U E U E , % g 2
mov I N T R Q _ C P U _ M O N D O _ T A I L , % g 4
ldxa [ % g 4 ] A S I _ Q U E U E , % g 4
cmp % g 2 , % g 4
be,p n % x c c , s u n 4 v _ c p u _ m o n d o _ q u e u e _ e m p t y
nop
2007-05-25 15:49:59 -07:00
/* Get &trap_block[smp_processor_id()] into %g4. */
ldxa [ % g 0 ] A S I _ S C R A T C H P A D , % g 4
sub % g 4 , T R A P _ P E R _ C P U _ F A U L T _ I N F O , % g 4
2006-02-08 02:53:50 -08:00
sparc64: Measure receiver forward progress to avoid send mondo timeout
A large sun4v SPARC system may have moments of intensive xcall activities,
usually caused by unmapping many pages on many CPUs concurrently. This can
flood receivers with CPU mondo interrupts for an extended period, causing
some unlucky senders to hit send-mondo timeout. This problem gets worse
as cpu count increases because sometimes mappings must be invalidated on
all CPUs, and sometimes all CPUs may gang up on a single CPU.
But a busy system is not a broken system. In the above scenario, as long
as the receiver is making forward progress processing mondo interrupts,
the sender should continue to retry.
This patch implements the receiver's forward progress meter by introducing
a per cpu counter 'cpu_mondo_counter[cpu]' where 'cpu' is in the range
of 0..NR_CPUS. The receiver increments its counter as soon as it receives
a mondo and the sender tracks the receiver's counter. If the receiver has
stopped making forward progress when the retry limit is reached, the sender
declares send-mondo-timeout and panic; otherwise, the receiver is allowed
to keep making forward progress.
In addition, it's been observed that PCIe hotplug events generate Correctable
Errors that are handled by hypervisor and then OS. Hypervisor 'borrows'
a guest cpu strand briefly to provide the service. If the cpu strand is
simultaneously the only cpu targeted by a mondo, it may not be available
for the mondo in 20msec, causing SUN4V mondo timeout. It appears that 1 second
is the agreed wait time between hypervisor and guest OS, this patch makes
the adjustment.
Orabug: 25476541
Orabug: 26417466
Signed-off-by: Jane Chu <jane.chu@oracle.com>
Reviewed-by: Steve Sistare <steven.sistare@oracle.com>
Reviewed-by: Anthony Yznaga <anthony.yznaga@oracle.com>
Reviewed-by: Rob Gardner <rob.gardner@oracle.com>
Reviewed-by: Thomas Tai <thomas.tai@oracle.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-07-11 12:00:54 -06:00
/* Get smp_processor_id() into %g3 */
sethi % h i ( t r a p _ b l o c k ) , % g 5
or % g 5 , % l o ( t r a p _ b l o c k ) , % g 5
sub % g 4 , % g 5 , % g 3
srlx % g 3 , T R A P _ B L O C K _ S Z _ S H I F T , % g 3
/* Increment cpu_mondo_counter[smp_processor_id()] */
sethi % h i ( c p u _ m o n d o _ c o u n t e r ) , % g 5
or % g 5 , % l o ( c p u _ m o n d o _ c o u n t e r ) , % g 5
sllx % g 3 , 3 , % g 3
add % g 5 , % g 3 , % g 5
ldx [ % g 5 ] , % g 3
add % g 3 , 1 , % g 3
stx % g 3 , [ % g 5 ]
2006-02-08 02:53:50 -08:00
/* Get CPU mondo queue base phys address into %g7. */
2007-05-25 15:49:59 -07:00
ldx [ % g 4 + T R A P _ P E R _ C P U _ C P U _ M O N D O _ P A ] , % g 7
2006-02-08 02:53:50 -08:00
/ * Now g e t t h e c r o s s - c a l l a r g u m e n t s a n d h a n d l e r P C , s a m e
* layout a s s u n 4 u :
*
* 1 st 6 4 - b i t w o r d : l o w h a l f i s 3 2 - b i t P C , p u t i n t o % g 3 a n d j m p l t o i t
* high h a l f i s c o n t e x t a r g t o M M U f l u s h e s , i n t o % g 5
* 2 nd 6 4 - b i t w o r d : 6 4 - b i t a r g , l o a d i n t o % g 1
* 3 rd 6 4 - b i t w o r d : 6 4 - b i t a r g , l o a d i n t o % g 7
* /
ldxa [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
add % g 2 , 0 x8 , % g 2
srlx % g 3 , 3 2 , % g 5
ldxa [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 1
add % g 2 , 0 x8 , % g 2
srl % g 3 , 0 , % g 3
ldxa [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 7
add % g 2 , 0 x40 - 0 x8 - 0 x8 , % g 2
/* Update queue head pointer. */
2007-05-25 15:49:59 -07:00
lduw [ % g 4 + T R A P _ P E R _ C P U _ C P U _ M O N D O _ Q M A S K ] , % g 4
2006-02-08 02:53:50 -08:00
and % g 2 , % g 4 , % g 2
mov I N T R Q _ C P U _ M O N D O _ H E A D , % g 4
stxa % g 2 , [ % g 4 ] A S I _ Q U E U E
membar #S y n c
jmpl % g 3 , % g 0
nop
sun4v_cpu_mondo_queue_empty :
retry
sun4v_dev_mondo :
/* Head offset in %g2, tail offset in %g4. */
mov I N T R Q _ D E V I C E _ M O N D O _ H E A D , % g 2
ldxa [ % g 2 ] A S I _ Q U E U E , % g 2
mov I N T R Q _ D E V I C E _ M O N D O _ T A I L , % g 4
ldxa [ % g 4 ] A S I _ Q U E U E , % g 4
cmp % g 2 , % g 4
be,p n % x c c , s u n 4 v _ d e v _ m o n d o _ q u e u e _ e m p t y
nop
2007-05-25 15:49:59 -07:00
/* Get &trap_block[smp_processor_id()] into %g4. */
ldxa [ % g 0 ] A S I _ S C R A T C H P A D , % g 4
sub % g 4 , T R A P _ P E R _ C P U _ F A U L T _ I N F O , % g 4
2006-02-08 02:53:50 -08:00
/* Get DEV mondo queue base phys address into %g5. */
2007-05-25 15:49:59 -07:00
ldx [ % g 4 + T R A P _ P E R _ C P U _ D E V _ M O N D O _ P A ] , % g 5
2006-02-08 02:53:50 -08:00
/* Load IVEC into %g3. */
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
add % g 2 , 0 x40 , % g 2
/ * XXX T h e r e c a n b e a f u l l 6 4 - b y t e b l o c k o f d a t a h e r e .
* XXX T h i s i s h o w w e c a n g e t a t M S I v e c t o r d a t a .
* XXX C u r r e n t w e d o n o t c a p t u r e t h i s , b u t w h e n w e d o w e ' l l
* XXX n e e d t o a d d a 6 4 - b y t e s t o r a g e a r e a i n t h e s t r u c t i n o _ b u c k e t
* XXX o r t h e s t r u c t i r q _ d e s c .
* /
/* Update queue head pointer, this frees up some registers. */
2007-05-25 15:49:59 -07:00
lduw [ % g 4 + T R A P _ P E R _ C P U _ D E V _ M O N D O _ Q M A S K ] , % g 4
2006-02-08 02:53:50 -08:00
and % g 2 , % g 4 , % g 2
mov I N T R Q _ D E V I C E _ M O N D O _ H E A D , % g 4
stxa % g 2 , [ % g 4 ] A S I _ Q U E U E
membar #S y n c
2007-10-13 21:42:46 -07:00
TRAP_ L O A D _ I R Q _ W O R K _ P A ( % g 1 , % g 4 )
2006-02-08 02:53:50 -08:00
2007-10-13 21:51:37 -07:00
/* For VIRQs, cookie is encoded as ~bucket_phys_addr */
brlz,p t % g 3 , 1 f
xnor % g 3 , % g 0 , % g 4
2007-10-13 21:42:46 -07:00
/* Get __pa(&ivector_table[IVEC]) into %g4. */
sethi % h i ( i v e c t o r _ t a b l e _ p a ) , % g 4
ldx [ % g 4 + % l o ( i v e c t o r _ t a b l e _ p a ) ] , % g 4
2007-10-12 02:59:40 -07:00
sllx % g 3 , 4 , % g 3
2006-02-08 02:53:50 -08:00
add % g 4 , % g 3 , % g 4
2007-10-13 21:51:37 -07:00
1 : ldx [ % g 1 ] , % g 2
2007-10-13 21:42:46 -07:00
stxa % g 2 , [ % g 4 ] A S I _ P H Y S _ U S E _ E C
stx % g 4 , [ % g 1 ]
2006-02-08 02:53:50 -08:00
/* Signal the interrupt by setting (1 << pil) in %softint. */
2006-06-20 01:20:00 -07:00
wr % g 0 , 1 < < P I L _ D E V I C E _ I R Q , % s e t _ s o f t i n t
2006-02-08 02:53:50 -08:00
sun4v_dev_mondo_queue_empty :
retry
sun4v_res_mondo :
/* Head offset in %g2, tail offset in %g4. */
mov I N T R Q _ R E S U M _ M O N D O _ H E A D , % g 2
ldxa [ % g 2 ] A S I _ Q U E U E , % g 2
mov I N T R Q _ R E S U M _ M O N D O _ T A I L , % g 4
ldxa [ % g 4 ] A S I _ Q U E U E , % g 4
cmp % g 2 , % g 4
be,p n % x c c , s u n 4 v _ r e s _ m o n d o _ q u e u e _ e m p t y
nop
/* Get &trap_block[smp_processor_id()] into %g3. */
2006-02-10 15:39:51 -08:00
ldxa [ % g 0 ] A S I _ S C R A T C H P A D , % g 3
sub % g 3 , T R A P _ P E R _ C P U _ F A U L T _ I N F O , % g 3
2006-02-08 02:53:50 -08:00
/* Get RES mondo queue base phys address into %g5. */
ldx [ % g 3 + T R A P _ P E R _ C P U _ R E S U M _ M O N D O _ P A ] , % g 5
/* Get RES kernel buffer base phys address into %g7. */
ldx [ % g 3 + T R A P _ P E R _ C P U _ R E S U M _ K B U F _ P A ] , % g 7
/* If the first word is non-zero, queue is full. */
ldxa [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 1
brnz,p n % g 1 , s u n 4 v _ r e s _ m o n d o _ q u e u e _ f u l l
nop
2007-05-25 15:49:59 -07:00
lduw [ % g 3 + T R A P _ P E R _ C P U _ R E S U M _ Q M A S K ] , % g 4
2006-02-08 02:53:50 -08:00
/* Remember this entry's offset in %g1. */
mov % g 2 , % g 1
/* Copy 64-byte queue entry into kernel buffer. */
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
/* Update queue head pointer. */
and % g 2 , % g 4 , % g 2
mov I N T R Q _ R E S U M _ M O N D O _ H E A D , % g 4
stxa % g 2 , [ % g 4 ] A S I _ Q U E U E
membar #S y n c
/ * Disable i n t e r r u p t s a n d s a v e r e g i s t e r s t a t e s o w e c a n c a l l
* C c o d e . T h e e t r a p h a n d l i n g w i l l l e a v e % g 4 i n % l 4 f o r u s
* when i t ' s d o n e .
* /
rdpr % p i l , % g 2
2008-11-23 21:55:29 -08:00
wrpr % g 0 , P I L _ N O R M A L _ M A X , % p i l
2006-02-08 02:53:50 -08:00
mov % g 1 , % g 4
ba,p t % x c c , e t r a p _ i r q
rd % p c , % g 7
2006-11-16 13:38:57 -08:00
# ifdef C O N F I G _ T R A C E _ I R Q F L A G S
call t r a c e _ h a r d i r q s _ o f f
nop
# endif
2006-02-08 02:53:50 -08:00
/* Log the event. */
add % s p , P T R E G S _ O F F , % o 0
call s u n 4 v _ r e s u m _ e r r o r
mov % l 4 , % o 1
/* Return from trap. */
ba,p t % x c c , r t r a p _ i r q
nop
sun4v_res_mondo_queue_empty :
retry
sun4v_res_mondo_queue_full :
/ * The q u e u e i s f u l l , c o n s o l i d a t e o u r d a m a g e b y s e t t i n g
* the h e a d e q u a l t o t h e t a i l . W e ' l l j u s t t r a p a g a i n o t h e r w i s e .
* Call C c o d e t o l o g t h e e v e n t .
* /
mov I N T R Q _ R E S U M _ M O N D O _ H E A D , % g 2
stxa % g 4 , [ % g 2 ] A S I _ Q U E U E
membar #S y n c
rdpr % p i l , % g 2
2008-11-23 21:55:29 -08:00
wrpr % g 0 , P I L _ N O R M A L _ M A X , % p i l
2006-02-08 02:53:50 -08:00
ba,p t % x c c , e t r a p _ i r q
rd % p c , % g 7
2006-11-16 13:38:57 -08:00
# ifdef C O N F I G _ T R A C E _ I R Q F L A G S
call t r a c e _ h a r d i r q s _ o f f
nop
# endif
2006-02-08 02:53:50 -08:00
call s u n 4 v _ r e s u m _ o v e r f l o w
add % s p , P T R E G S _ O F F , % o 0
ba,p t % x c c , r t r a p _ i r q
nop
sun4v_nonres_mondo :
/* Head offset in %g2, tail offset in %g4. */
mov I N T R Q _ N O N R E S U M _ M O N D O _ H E A D , % g 2
ldxa [ % g 2 ] A S I _ Q U E U E , % g 2
mov I N T R Q _ N O N R E S U M _ M O N D O _ T A I L , % g 4
ldxa [ % g 4 ] A S I _ Q U E U E , % g 4
cmp % g 2 , % g 4
be,p n % x c c , s u n 4 v _ n o n r e s _ m o n d o _ q u e u e _ e m p t y
nop
/* Get &trap_block[smp_processor_id()] into %g3. */
2006-02-10 15:39:51 -08:00
ldxa [ % g 0 ] A S I _ S C R A T C H P A D , % g 3
sub % g 3 , T R A P _ P E R _ C P U _ F A U L T _ I N F O , % g 3
2006-02-08 02:53:50 -08:00
/* Get RES mondo queue base phys address into %g5. */
ldx [ % g 3 + T R A P _ P E R _ C P U _ N O N R E S U M _ M O N D O _ P A ] , % g 5
/* Get RES kernel buffer base phys address into %g7. */
ldx [ % g 3 + T R A P _ P E R _ C P U _ N O N R E S U M _ K B U F _ P A ] , % g 7
/* If the first word is non-zero, queue is full. */
ldxa [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 1
brnz,p n % g 1 , s u n 4 v _ n o n r e s _ m o n d o _ q u e u e _ f u l l
nop
2007-05-25 15:49:59 -07:00
lduw [ % g 3 + T R A P _ P E R _ C P U _ N O N R E S U M _ Q M A S K ] , % g 4
2006-02-08 02:53:50 -08:00
/* Remember this entry's offset in %g1. */
mov % g 2 , % g 1
/* Copy 64-byte queue entry into kernel buffer. */
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
ldxa [ % g 5 + % g 2 ] A S I _ P H Y S _ U S E _ E C , % g 3
stxa % g 3 , [ % g 7 + % g 2 ] A S I _ P H Y S _ U S E _ E C
add % g 2 , 0 x08 , % g 2
/* Update queue head pointer. */
and % g 2 , % g 4 , % g 2
mov I N T R Q _ N O N R E S U M _ M O N D O _ H E A D , % g 4
stxa % g 2 , [ % g 4 ] A S I _ Q U E U E
membar #S y n c
/ * Disable i n t e r r u p t s a n d s a v e r e g i s t e r s t a t e s o w e c a n c a l l
* C c o d e . T h e e t r a p h a n d l i n g w i l l l e a v e % g 4 i n % l 4 f o r u s
* when i t ' s d o n e .
* /
rdpr % p i l , % g 2
2008-11-23 21:55:29 -08:00
wrpr % g 0 , P I L _ N O R M A L _ M A X , % p i l
2006-02-08 02:53:50 -08:00
mov % g 1 , % g 4
ba,p t % x c c , e t r a p _ i r q
rd % p c , % g 7
2006-11-16 13:38:57 -08:00
# ifdef C O N F I G _ T R A C E _ I R Q F L A G S
call t r a c e _ h a r d i r q s _ o f f
nop
# endif
2006-02-08 02:53:50 -08:00
/* Log the event. */
add % s p , P T R E G S _ O F F , % o 0
call s u n 4 v _ n o n r e s u m _ e r r o r
mov % l 4 , % o 1
/* Return from trap. */
ba,p t % x c c , r t r a p _ i r q
nop
sun4v_nonres_mondo_queue_empty :
retry
sun4v_nonres_mondo_queue_full :
/ * The q u e u e i s f u l l , c o n s o l i d a t e o u r d a m a g e b y s e t t i n g
* the h e a d e q u a l t o t h e t a i l . W e ' l l j u s t t r a p a g a i n o t h e r w i s e .
* Call C c o d e t o l o g t h e e v e n t .
* /
mov I N T R Q _ N O N R E S U M _ M O N D O _ H E A D , % g 2
stxa % g 4 , [ % g 2 ] A S I _ Q U E U E
membar #S y n c
rdpr % p i l , % g 2
2008-11-23 21:55:29 -08:00
wrpr % g 0 , P I L _ N O R M A L _ M A X , % p i l
2006-02-08 02:53:50 -08:00
ba,p t % x c c , e t r a p _ i r q
rd % p c , % g 7
2006-11-16 13:38:57 -08:00
# ifdef C O N F I G _ T R A C E _ I R Q F L A G S
call t r a c e _ h a r d i r q s _ o f f
nop
# endif
2006-02-08 02:53:50 -08:00
call s u n 4 v _ n o n r e s u m _ o v e r f l o w
add % s p , P T R E G S _ O F F , % o 0
ba,p t % x c c , r t r a p _ i r q
nop
