10d91611f4
Reimplement Book3S idle code in C, moving POWER7/8/9 implementation speific HV idle code to the powernv platform code. Book3S assembly stubs are kept in common code and used only to save the stack frame and non-volatile GPRs before executing architected idle instructions, and restoring the stack and reloading GPRs then returning to C after waking from idle. The complex logic dealing with threads and subcores, locking, SPRs, HMIs, timebase resync, etc., is all done in C which makes it more maintainable. This is not a strict translation to C code, there are some significant differences: - Idle wakeup no longer uses the ->cpu_restore call to reinit SPRs, but saves and restores them itself. - The optimisation where EC=ESL=0 idle modes did not have to save GPRs or change MSR is restored, because it's now simple to do. ESL=1 sleeps that do not lose GPRs can use this optimization too. - KVM secondary entry and cede is now more of a call/return style rather than branchy. nap_state_lost is not required because KVM always returns via NVGPR restoring path. - KVM secondary wakeup from offline sequence is moved entirely into the offline wakeup, which avoids a hwsync in the normal idle wakeup path. Performance measured with context switch ping-pong on different threads or cores, is possibly improved a small amount, 1-3% depending on stop state and core vs thread test for shallow states. Deep states it's in the noise compared with other latencies. KVM improvements: - Idle sleepers now always return to caller rather than branch out to KVM first. - This allows optimisations like very fast return to caller when no state has been lost. - KVM no longer requires nap_state_lost because it controls NVGPR save/restore itself on the way in and out. - The heavy idle wakeup KVM request check can be moved out of the normal host idle code and into the not-performance-critical offline code. - KVM nap code now returns from where it is called, which makes the flow a bit easier to follow. Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com> Signed-off-by: Nicholas Piggin <npiggin@gmail.com> [mpe: Squash the KVM changes in] Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
189 lines
4.8 KiB
ArmAsm
189 lines
4.8 KiB
ArmAsm
/*
|
|
* Copyright 2018, IBM Corporation.
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; either version
|
|
* 2 of the License, or (at your option) any later version.
|
|
*
|
|
* This file contains general idle entry/exit functions to save
|
|
* and restore stack and NVGPRs which allows C code to call idle
|
|
* states that lose GPRs, and it will return transparently with
|
|
* SRR1 wakeup reason return value.
|
|
*
|
|
* The platform / CPU caller must ensure SPRs and any other non-GPR
|
|
* state is saved and restored correctly, handle KVM, interrupts, etc.
|
|
*/
|
|
|
|
#include <asm/ppc_asm.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/ppc-opcode.h>
|
|
#include <asm/cpuidle.h>
|
|
|
|
/*
|
|
* Desired PSSCR in r3
|
|
*
|
|
* No state will be lost regardless of wakeup mechanism (interrupt or NIA).
|
|
*
|
|
* An EC=0 type wakeup will return with a value of 0. SRESET wakeup (which can
|
|
* happen with xscom SRESET and possibly MCE) may clobber volatiles except LR,
|
|
* and must blr, to return to caller with r3 set according to caller's expected
|
|
* return code (for Book3S/64 that is SRR1).
|
|
*/
|
|
_GLOBAL(isa300_idle_stop_noloss)
|
|
mtspr SPRN_PSSCR,r3
|
|
PPC_STOP
|
|
li r3,0
|
|
blr
|
|
|
|
/*
|
|
* Desired PSSCR in r3
|
|
*
|
|
* GPRs may be lost, so they are saved here. Wakeup is by interrupt only.
|
|
* The SRESET wakeup returns to this function's caller by calling
|
|
* idle_return_gpr_loss with r3 set to desired return value.
|
|
*
|
|
* A wakeup without GPR loss may alteratively be handled as in
|
|
* isa300_idle_stop_noloss and blr directly, as an optimisation.
|
|
*
|
|
* The caller is responsible for saving/restoring SPRs, MSR, timebase,
|
|
* etc.
|
|
*/
|
|
_GLOBAL(isa300_idle_stop_mayloss)
|
|
mtspr SPRN_PSSCR,r3
|
|
std r1,PACAR1(r13)
|
|
mflr r4
|
|
mfcr r5
|
|
/* use stack red zone rather than a new frame for saving regs */
|
|
std r2,-8*0(r1)
|
|
std r14,-8*1(r1)
|
|
std r15,-8*2(r1)
|
|
std r16,-8*3(r1)
|
|
std r17,-8*4(r1)
|
|
std r18,-8*5(r1)
|
|
std r19,-8*6(r1)
|
|
std r20,-8*7(r1)
|
|
std r21,-8*8(r1)
|
|
std r22,-8*9(r1)
|
|
std r23,-8*10(r1)
|
|
std r24,-8*11(r1)
|
|
std r25,-8*12(r1)
|
|
std r26,-8*13(r1)
|
|
std r27,-8*14(r1)
|
|
std r28,-8*15(r1)
|
|
std r29,-8*16(r1)
|
|
std r30,-8*17(r1)
|
|
std r31,-8*18(r1)
|
|
std r4,-8*19(r1)
|
|
std r5,-8*20(r1)
|
|
/* 168 bytes */
|
|
PPC_STOP
|
|
b . /* catch bugs */
|
|
|
|
/*
|
|
* Desired return value in r3
|
|
*
|
|
* The idle wakeup SRESET interrupt can call this after calling
|
|
* to return to the idle sleep function caller with r3 as the return code.
|
|
*
|
|
* This must not be used if idle was entered via a _noloss function (use
|
|
* a simple blr instead).
|
|
*/
|
|
_GLOBAL(idle_return_gpr_loss)
|
|
ld r1,PACAR1(r13)
|
|
ld r4,-8*19(r1)
|
|
ld r5,-8*20(r1)
|
|
mtlr r4
|
|
mtcr r5
|
|
/*
|
|
* KVM nap requires r2 to be saved, rather than just restoring it
|
|
* from PACATOC. This could be avoided for that less common case
|
|
* if KVM saved its r2.
|
|
*/
|
|
ld r2,-8*0(r1)
|
|
ld r14,-8*1(r1)
|
|
ld r15,-8*2(r1)
|
|
ld r16,-8*3(r1)
|
|
ld r17,-8*4(r1)
|
|
ld r18,-8*5(r1)
|
|
ld r19,-8*6(r1)
|
|
ld r20,-8*7(r1)
|
|
ld r21,-8*8(r1)
|
|
ld r22,-8*9(r1)
|
|
ld r23,-8*10(r1)
|
|
ld r24,-8*11(r1)
|
|
ld r25,-8*12(r1)
|
|
ld r26,-8*13(r1)
|
|
ld r27,-8*14(r1)
|
|
ld r28,-8*15(r1)
|
|
ld r29,-8*16(r1)
|
|
ld r30,-8*17(r1)
|
|
ld r31,-8*18(r1)
|
|
blr
|
|
|
|
/*
|
|
* This is the sequence required to execute idle instructions, as
|
|
* specified in ISA v2.07 (and earlier). MSR[IR] and MSR[DR] must be 0.
|
|
*
|
|
* The 0(r1) slot is used to save r2 in isa206, so use that here.
|
|
*/
|
|
#define IDLE_STATE_ENTER_SEQ_NORET(IDLE_INST) \
|
|
/* Magic NAP/SLEEP/WINKLE mode enter sequence */ \
|
|
std r2,0(r1); \
|
|
ptesync; \
|
|
ld r2,0(r1); \
|
|
236: cmpd cr0,r2,r2; \
|
|
bne 236b; \
|
|
IDLE_INST; \
|
|
b . /* catch bugs */
|
|
|
|
/*
|
|
* Desired instruction type in r3
|
|
*
|
|
* GPRs may be lost, so they are saved here. Wakeup is by interrupt only.
|
|
* The SRESET wakeup returns to this function's caller by calling
|
|
* idle_return_gpr_loss with r3 set to desired return value.
|
|
*
|
|
* A wakeup without GPR loss may alteratively be handled as in
|
|
* isa300_idle_stop_noloss and blr directly, as an optimisation.
|
|
*
|
|
* The caller is responsible for saving/restoring SPRs, MSR, timebase,
|
|
* etc.
|
|
*
|
|
* This must be called in real-mode (MSR_IDLE).
|
|
*/
|
|
_GLOBAL(isa206_idle_insn_mayloss)
|
|
std r1,PACAR1(r13)
|
|
mflr r4
|
|
mfcr r5
|
|
/* use stack red zone rather than a new frame for saving regs */
|
|
std r2,-8*0(r1)
|
|
std r14,-8*1(r1)
|
|
std r15,-8*2(r1)
|
|
std r16,-8*3(r1)
|
|
std r17,-8*4(r1)
|
|
std r18,-8*5(r1)
|
|
std r19,-8*6(r1)
|
|
std r20,-8*7(r1)
|
|
std r21,-8*8(r1)
|
|
std r22,-8*9(r1)
|
|
std r23,-8*10(r1)
|
|
std r24,-8*11(r1)
|
|
std r25,-8*12(r1)
|
|
std r26,-8*13(r1)
|
|
std r27,-8*14(r1)
|
|
std r28,-8*15(r1)
|
|
std r29,-8*16(r1)
|
|
std r30,-8*17(r1)
|
|
std r31,-8*18(r1)
|
|
std r4,-8*19(r1)
|
|
std r5,-8*20(r1)
|
|
cmpwi r3,PNV_THREAD_NAP
|
|
bne 1f
|
|
IDLE_STATE_ENTER_SEQ_NORET(PPC_NAP)
|
|
1: cmpwi r3,PNV_THREAD_SLEEP
|
|
bne 2f
|
|
IDLE_STATE_ENTER_SEQ_NORET(PPC_SLEEP)
|
|
2: IDLE_STATE_ENTER_SEQ_NORET(PPC_WINKLE)
|
|
|