powerpc/64s: move the hash fault handling logic to C
The fault handling still has some complex logic particularly around hash table handling, in asm. Implement most of this in C. Signed-off-by: Nicholas Piggin <npiggin@gmail.com> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Link: https://lore.kernel.org/r/20210130130852.2952424-6-npiggin@gmail.com
This commit is contained in:
Nicholas Piggin
Michael Ellerman
parent
36f0114140
commit
a4922f5442
@ -456,6 +456,7 @@ static inline unsigned long hpt_hash(unsigned long vpn,
|
||||
|
||||
long hpte_insert_repeating(unsigned long hash, unsigned long vpn, unsigned long pa,
|
||||
unsigned long rlags, unsigned long vflags, int psize, int ssize);
|
||||
int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr);
|
||||
extern int __hash_page_4K(unsigned long ea, unsigned long access,
|
||||
unsigned long vsid, pte_t *ptep, unsigned long trap,
|
||||
unsigned long flags, int ssize, int subpage_prot);
|
||||
|
||||
@ -1401,14 +1401,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
|
||||
*
|
||||
* Handling:
|
||||
* - Hash MMU
|
||||
* Go to do_hash_page first to see if the HPT can be filled from an entry in
|
||||
* the Linux page table. Hash faults can hit in kernel mode in a fairly
|
||||
* Go to do_hash_fault, which attempts to fill the HPT from an entry in the
|
||||
* Linux page table. Hash faults can hit in kernel mode in a fairly
|
||||
* arbitrary state (e.g., interrupts disabled, locks held) when accessing
|
||||
* "non-bolted" regions, e.g., vmalloc space. However these should always be
|
||||
* backed by Linux page tables.
|
||||
* backed by Linux page table entries.
|
||||
*
|
||||
* If none is found, do a Linux page fault. Linux page faults can happen in
|
||||
* kernel mode due to user copy operations of course.
|
||||
* If no entry is found the Linux page fault handler is invoked (by
|
||||
* do_hash_fault). Linux page faults can happen in kernel mode due to user
|
||||
* copy operations of course.
|
||||
*
|
||||
* KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest
|
||||
* MMU context, which may cause a DSI in the host, which must go to the
|
||||
@ -1439,27 +1440,29 @@ EXC_COMMON_BEGIN(data_access_common)
|
||||
GEN_COMMON data_access
|
||||
ld r4,_DAR(r1)
|
||||
ld r5,_DSISR(r1)
|
||||
addi r3,r1,STACK_FRAME_OVERHEAD
|
||||
andis. r0,r5,DSISR_DABRMATCH@h
|
||||
bne- 1f
|
||||
BEGIN_MMU_FTR_SECTION
|
||||
ld r6,_MSR(r1)
|
||||
li r3,0x300
|
||||
b do_hash_page /* Try to handle as hpte fault */
|
||||
bl do_hash_fault
|
||||
MMU_FTR_SECTION_ELSE
|
||||
b handle_page_fault
|
||||
bl do_page_fault
|
||||
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
|
||||
cmpdi r3,0
|
||||
beq+ interrupt_return
|
||||
mr r5,r3
|
||||
addi r3,r1,STACK_FRAME_OVERHEAD
|
||||
ld r4,_DAR(r1)
|
||||
bl __bad_page_fault
|
||||
b interrupt_return
|
||||
|
||||
1: /* We have a data breakpoint exception - handle it */
|
||||
ld r4,_DAR(r1)
|
||||
ld r5,_DSISR(r1)
|
||||
addi r3,r1,STACK_FRAME_OVERHEAD
|
||||
bl do_break
|
||||
1: bl do_break
|
||||
/*
|
||||
* do_break() may have changed the NV GPRS while handling a breakpoint.
|
||||
* If so, we need to restore them with their updated values.
|
||||
*/
|
||||
REST_NVGPRS(r1)
|
||||
b interrupt_return
|
||||
b interrupt_return
|
||||
|
||||
GEN_KVM data_access
|
||||
|
||||
@ -1554,13 +1557,19 @@ EXC_COMMON_BEGIN(instruction_access_common)
|
||||
GEN_COMMON instruction_access
|
||||
ld r4,_DAR(r1)
|
||||
ld r5,_DSISR(r1)
|
||||
addi r3,r1,STACK_FRAME_OVERHEAD
|
||||
BEGIN_MMU_FTR_SECTION
|
||||
ld r6,_MSR(r1)
|
||||
li r3,0x400
|
||||
b do_hash_page /* Try to handle as hpte fault */
|
||||
bl do_hash_fault
|
||||
MMU_FTR_SECTION_ELSE
|
||||
b handle_page_fault
|
||||
bl do_page_fault
|
||||
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
|
||||
cmpdi r3,0
|
||||
beq+ interrupt_return
|
||||
mr r5,r3
|
||||
addi r3,r1,STACK_FRAME_OVERHEAD
|
||||
ld r4,_DAR(r1)
|
||||
bl __bad_page_fault
|
||||
b interrupt_return
|
||||
|
||||
GEN_KVM instruction_access
|
||||
|
||||
@ -3216,83 +3225,3 @@ disable_machine_check:
|
||||
RFI_TO_KERNEL
|
||||
1: mtlr r0
|
||||
blr
|
||||
|
||||
/*
|
||||
* Hash table stuff
|
||||
*/
|
||||
.balign IFETCH_ALIGN_BYTES
|
||||
do_hash_page:
|
||||
#ifdef CONFIG_PPC_BOOK3S_64
|
||||
lis r0,(DSISR_BAD_FAULT_64S | DSISR_KEYFAULT)@h
|
||||
ori r0,r0,DSISR_BAD_FAULT_64S@l
|
||||
and. r0,r5,r0 /* weird error? */
|
||||
bne- handle_page_fault /* if not, try to insert a HPTE */
|
||||
|
||||
/*
|
||||
* If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
|
||||
* don't call hash_page, just fail the fault. This is required to
|
||||
* prevent re-entrancy problems in the hash code, namely perf
|
||||
* interrupts hitting while something holds H_PAGE_BUSY, and taking a
|
||||
* hash fault. See the comment in hash_preload().
|
||||
*/
|
||||
ld r11, PACA_THREAD_INFO(r13)
|
||||
lwz r0,TI_PREEMPT(r11)
|
||||
andis. r0,r0,NMI_MASK@h
|
||||
bne 77f
|
||||
|
||||
/*
|
||||
* r3 contains the trap number
|
||||
* r4 contains the faulting address
|
||||
* r5 contains dsisr
|
||||
* r6 msr
|
||||
*
|
||||
* at return r3 = 0 for success, 1 for page fault, negative for error
|
||||
*/
|
||||
bl __hash_page /* build HPTE if possible */
|
||||
cmpdi r3,0 /* see if __hash_page succeeded */
|
||||
|
||||
/* Success */
|
||||
beq interrupt_return /* Return from exception on success */
|
||||
|
||||
/* Error */
|
||||
blt- 13f
|
||||
|
||||
/* Reload DAR/DSISR into r4/r5 for handle_page_fault */
|
||||
ld r4,_DAR(r1)
|
||||
ld r5,_DSISR(r1)
|
||||
#endif /* CONFIG_PPC_BOOK3S_64 */
|
||||
|
||||
/* Here we have a page fault that hash_page can't handle. */
|
||||
handle_page_fault:
|
||||
addi r3,r1,STACK_FRAME_OVERHEAD
|
||||
bl do_page_fault
|
||||
cmpdi r3,0
|
||||
beq+ interrupt_return
|
||||
mr r5,r3
|
||||
addi r3,r1,STACK_FRAME_OVERHEAD
|
||||
ld r4,_DAR(r1)
|
||||
bl __bad_page_fault
|
||||
b interrupt_return
|
||||
|
||||
#ifdef CONFIG_PPC_BOOK3S_64
|
||||
/* We have a page fault that hash_page could handle but HV refused
|
||||
* the PTE insertion
|
||||
*/
|
||||
13: mr r5,r3
|
||||
addi r3,r1,STACK_FRAME_OVERHEAD
|
||||
ld r4,_DAR(r1)
|
||||
bl low_hash_fault
|
||||
b interrupt_return
|
||||
#endif
|
||||
|
||||
/*
|
||||
* We come here as a result of a DSI at a point where we don't want
|
||||
* to call hash_page, such as when we are accessing memory (possibly
|
||||
* user memory) inside a PMU interrupt that occurred while interrupts
|
||||
* were soft-disabled. We want to invoke the exception handler for
|
||||
* the access, or panic if there isn't a handler.
|
||||
*/
|
||||
77: addi r3,r1,STACK_FRAME_OVERHEAD
|
||||
li r5,SIGSEGV
|
||||
bl bad_page_fault
|
||||
b interrupt_return
|
||||
|
||||
@ -1512,16 +1512,40 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap,
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(hash_page);
|
||||
|
||||
int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr,
|
||||
unsigned long msr)
|
||||
int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
|
||||
{
|
||||
unsigned long access = _PAGE_PRESENT | _PAGE_READ;
|
||||
unsigned long flags = 0;
|
||||
struct mm_struct *mm = current->mm;
|
||||
unsigned int region_id = get_region_id(ea);
|
||||
struct mm_struct *mm;
|
||||
unsigned int region_id;
|
||||
int err;
|
||||
|
||||
if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_KEYFAULT)))
|
||||
goto page_fault;
|
||||
|
||||
/*
|
||||
* If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
|
||||
* don't call hash_page, just fail the fault. This is required to
|
||||
* prevent re-entrancy problems in the hash code, namely perf
|
||||
* interrupts hitting while something holds H_PAGE_BUSY, and taking a
|
||||
* hash fault. See the comment in hash_preload().
|
||||
*
|
||||
* We come here as a result of a DSI at a point where we don't want
|
||||
* to call hash_page, such as when we are accessing memory (possibly
|
||||
* user memory) inside a PMU interrupt that occurred while interrupts
|
||||
* were soft-disabled. We want to invoke the exception handler for
|
||||
* the access, or panic if there isn't a handler.
|
||||
*/
|
||||
if (unlikely(in_nmi())) {
|
||||
bad_page_fault(regs, ea, SIGSEGV);
|
||||
return 0;
|
||||
}
|
||||
|
||||
region_id = get_region_id(ea);
|
||||
if ((region_id == VMALLOC_REGION_ID) || (region_id == IO_REGION_ID))
|
||||
mm = &init_mm;
|
||||
else
|
||||
mm = current->mm;
|
||||
|
||||
if (dsisr & DSISR_NOHPTE)
|
||||
flags |= HPTE_NOHPTE_UPDATE;
|
||||
@ -1537,13 +1561,31 @@ int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr,
|
||||
* 2) user space access kernel space.
|
||||
*/
|
||||
access |= _PAGE_PRIVILEGED;
|
||||
if ((msr & MSR_PR) || (region_id == USER_REGION_ID))
|
||||
if (user_mode(regs) || (region_id == USER_REGION_ID))
|
||||
access &= ~_PAGE_PRIVILEGED;
|
||||
|
||||
if (trap == 0x400)
|
||||
if (regs->trap == 0x400)
|
||||
access |= _PAGE_EXEC;
|
||||
|
||||
return hash_page_mm(mm, ea, access, trap, flags);
|
||||
err = hash_page_mm(mm, ea, access, regs->trap, flags);
|
||||
if (unlikely(err < 0)) {
|
||||
// failed to instert a hash PTE due to an hypervisor error
|
||||
if (user_mode(regs)) {
|
||||
if (IS_ENABLED(CONFIG_PPC_SUBPAGE_PROT) && err == -2)
|
||||
_exception(SIGSEGV, regs, SEGV_ACCERR, ea);
|
||||
else
|
||||
_exception(SIGBUS, regs, BUS_ADRERR, ea);
|
||||
} else {
|
||||
bad_page_fault(regs, ea, SIGBUS);
|
||||
}
|
||||
err = 0;
|
||||
|
||||
} else if (err) {
|
||||
page_fault:
|
||||
err = do_page_fault(regs, ea, dsisr);
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PPC_MM_SLICES
|
||||
@ -1843,27 +1885,6 @@ void flush_hash_range(unsigned long number, int local)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* low_hash_fault is called when we the low level hash code failed
|
||||
* to instert a PTE due to an hypervisor error
|
||||
*/
|
||||
void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc)
|
||||
{
|
||||
enum ctx_state prev_state = exception_enter();
|
||||
|
||||
if (user_mode(regs)) {
|
||||
#ifdef CONFIG_PPC_SUBPAGE_PROT
|
||||
if (rc == -2)
|
||||
_exception(SIGSEGV, regs, SEGV_ACCERR, address);
|
||||
else
|
||||
#endif
|
||||
_exception(SIGBUS, regs, BUS_ADRERR, address);
|
||||
} else
|
||||
bad_page_fault(regs, address, SIGBUS);
|
||||
|
||||
exception_exit(prev_state);
|
||||
}
|
||||
|
||||
long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
|
||||
unsigned long pa, unsigned long rflags,
|
||||
unsigned long vflags, int psize, int ssize)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user