iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

powerpc/64s: move the hash fault handling logic to C

Browse Source 
The fault handling still has some complex logic particularly around
hash table handling, in asm. Implement most of this in C.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20210130130852.2952424-6-npiggin@gmail.com
This commit is contained in:
Nicholas Piggin 2021-01-30 23:08:15 +10:00 committed by Michael Ellerman
parent 36f0114140
commit a4922f5442
3 changed files with 78 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
arch/powerpc/include/asm/book3s/64/mmu-hash.h
View File

@ -456,6 +456,7 @@ static inline unsigned long hpt_hash(unsigned long vpn,
long hpte_insert_repeating(unsigned long hash, unsigned long vpn, unsigned long pa, long hpte_insert_repeating(unsigned long hash, unsigned long vpn, unsigned long pa,
unsigned long rlags, unsigned long vflags, int psize, int ssize); unsigned long rlags, unsigned long vflags, int psize, int ssize);
int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr);
extern int __hash_page_4K(unsigned long ea, unsigned long access, extern int __hash_page_4K(unsigned long ea, unsigned long access,
unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long vsid, pte_t *ptep, unsigned long trap,
unsigned long flags, int ssize, int subpage_prot); unsigned long flags, int ssize, int subpage_prot);

127
arch/powerpc/kernel/exceptions-64s.S
View File

@ -1401,14 +1401,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
* *
* Handling: * Handling:
* - Hash MMU * - Hash MMU
* Go to do_hash_page first to see if the HPT can be filled from an entry in * Go to do_hash_fault, which attempts to fill the HPT from an entry in the
* the Linux page table. Hash faults can hit in kernel mode in a fairly * Linux page table. Hash faults can hit in kernel mode in a fairly
* arbitrary state (e.g., interrupts disabled, locks held) when accessing * arbitrary state (e.g., interrupts disabled, locks held) when accessing
* "non-bolted" regions, e.g., vmalloc space. However these should always be * "non-bolted" regions, e.g., vmalloc space. However these should always be
* backed by Linux page tables. * backed by Linux page table entries.
* *
* If none is found, do a Linux page fault. Linux page faults can happen in * If no entry is found the Linux page fault handler is invoked (by
* kernel mode due to user copy operations of course. * do_hash_fault). Linux page faults can happen in kernel mode due to user
* copy operations of course.
* *
* KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest
* MMU context, which may cause a DSI in the host, which must go to the * MMU context, which may cause a DSI in the host, which must go to the
@ -1439,27 +1440,29 @@ EXC_COMMON_BEGIN(data_access_common)
GEN_COMMON data_access GEN_COMMON data_access
ld r4,_DAR(r1) ld r4,_DAR(r1)
ld r5,_DSISR(r1) ld r5,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
andis. r0,r5,DSISR_DABRMATCH@h andis. r0,r5,DSISR_DABRMATCH@h
bne- 1f bne- 1f
BEGIN_MMU_FTR_SECTION BEGIN_MMU_FTR_SECTION
ld r6,_MSR(r1) bl do_hash_fault
li r3,0x300
b do_hash_page /* Try to handle as hpte fault */
MMU_FTR_SECTION_ELSE MMU_FTR_SECTION_ELSE
b handle_page_fault bl do_page_fault
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
cmpdi r3,0
beq+ interrupt_return
mr r5,r3
addi r3,r1,STACK_FRAME_OVERHEAD
ld r4,_DAR(r1)
bl __bad_page_fault
b interrupt_return
1: /* We have a data breakpoint exception - handle it */ 1: bl do_break
ld r4,_DAR(r1)
ld r5,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_break
/* /*
* do_break() may have changed the NV GPRS while handling a breakpoint. * do_break() may have changed the NV GPRS while handling a breakpoint.
* If so, we need to restore them with their updated values. * If so, we need to restore them with their updated values.
*/ */
REST_NVGPRS(r1) REST_NVGPRS(r1)
b interrupt_return b interrupt_return
GEN_KVM data_access GEN_KVM data_access
@ -1554,13 +1557,19 @@ EXC_COMMON_BEGIN(instruction_access_common)
GEN_COMMON instruction_access GEN_COMMON instruction_access
ld r4,_DAR(r1) ld r4,_DAR(r1)
ld r5,_DSISR(r1) ld r5,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
BEGIN_MMU_FTR_SECTION BEGIN_MMU_FTR_SECTION
ld r6,_MSR(r1) bl do_hash_fault
li r3,0x400
b do_hash_page /* Try to handle as hpte fault */
MMU_FTR_SECTION_ELSE MMU_FTR_SECTION_ELSE
b handle_page_fault bl do_page_fault
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
cmpdi r3,0
beq+ interrupt_return
mr r5,r3
addi r3,r1,STACK_FRAME_OVERHEAD
ld r4,_DAR(r1)
bl __bad_page_fault
b interrupt_return
GEN_KVM instruction_access GEN_KVM instruction_access
@ -3216,83 +3225,3 @@ disable_machine_check:
RFI_TO_KERNEL RFI_TO_KERNEL
1: mtlr r0 1: mtlr r0
blr blr
/*
* Hash table stuff
*/
.balign IFETCH_ALIGN_BYTES
do_hash_page:
#ifdef CONFIG_PPC_BOOK3S_64
lis r0,(DSISR_BAD_FAULT_64S | DSISR_KEYFAULT)@h
ori r0,r0,DSISR_BAD_FAULT_64S@l
and. r0,r5,r0 /* weird error? */
bne- handle_page_fault /* if not, try to insert a HPTE */
/*
* If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
* don't call hash_page, just fail the fault. This is required to
* prevent re-entrancy problems in the hash code, namely perf
* interrupts hitting while something holds H_PAGE_BUSY, and taking a
* hash fault. See the comment in hash_preload().
*/
ld r11, PACA_THREAD_INFO(r13)
lwz r0,TI_PREEMPT(r11)
andis. r0,r0,NMI_MASK@h
bne 77f
/*
* r3 contains the trap number
* r4 contains the faulting address
* r5 contains dsisr
* r6 msr
*
* at return r3 = 0 for success, 1 for page fault, negative for error
*/
bl __hash_page /* build HPTE if possible */
cmpdi r3,0 /* see if __hash_page succeeded */
/* Success */
beq interrupt_return /* Return from exception on success */
/* Error */
blt- 13f
/* Reload DAR/DSISR into r4/r5 for handle_page_fault */
ld r4,_DAR(r1)
ld r5,_DSISR(r1)
#endif /* CONFIG_PPC_BOOK3S_64 */
/* Here we have a page fault that hash_page can't handle. */
handle_page_fault:
addi r3,r1,STACK_FRAME_OVERHEAD
bl do_page_fault
cmpdi r3,0
beq+ interrupt_return
mr r5,r3
addi r3,r1,STACK_FRAME_OVERHEAD
ld r4,_DAR(r1)
bl __bad_page_fault
b interrupt_return
#ifdef CONFIG_PPC_BOOK3S_64
/* We have a page fault that hash_page could handle but HV refused
* the PTE insertion
*/
13: mr r5,r3
addi r3,r1,STACK_FRAME_OVERHEAD
ld r4,_DAR(r1)
bl low_hash_fault
b interrupt_return
#endif
/*
* We come here as a result of a DSI at a point where we don't want
* to call hash_page, such as when we are accessing memory (possibly
* user memory) inside a PMU interrupt that occurred while interrupts
* were soft-disabled. We want to invoke the exception handler for
* the access, or panic if there isn't a handler.
*/
77: addi r3,r1,STACK_FRAME_OVERHEAD
li r5,SIGSEGV
bl bad_page_fault
b interrupt_return

77
arch/powerpc/mm/book3s64/hash_utils.c
View File

@ -1512,16 +1512,40 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap,
} }
EXPORT_SYMBOL_GPL(hash_page); EXPORT_SYMBOL_GPL(hash_page);
int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, int do_hash_fault(struct pt_regs *regs, unsigned long ea, unsigned long dsisr)
unsigned long msr)
{ {
unsigned long access = _PAGE_PRESENT | _PAGE_READ; unsigned long access = _PAGE_PRESENT | _PAGE_READ;
unsigned long flags = 0; unsigned long flags = 0;
struct mm_struct *mm = current->mm; struct mm_struct *mm;
unsigned int region_id = get_region_id(ea); unsigned int region_id;
int err;
if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_KEYFAULT)))
goto page_fault;
/*
* If we are in an "NMI" (e.g., an interrupt when soft-disabled), then
* don't call hash_page, just fail the fault. This is required to
* prevent re-entrancy problems in the hash code, namely perf
* interrupts hitting while something holds H_PAGE_BUSY, and taking a
* hash fault. See the comment in hash_preload().
*
* We come here as a result of a DSI at a point where we don't want
* to call hash_page, such as when we are accessing memory (possibly
* user memory) inside a PMU interrupt that occurred while interrupts
* were soft-disabled. We want to invoke the exception handler for
* the access, or panic if there isn't a handler.
*/
if (unlikely(in_nmi())) {
bad_page_fault(regs, ea, SIGSEGV);
return 0;
}
region_id = get_region_id(ea);
if ((region_id == VMALLOC_REGION_ID) || (region_id == IO_REGION_ID)) if ((region_id == VMALLOC_REGION_ID) || (region_id == IO_REGION_ID))
mm = &init_mm; mm = &init_mm;
else
mm = current->mm;
if (dsisr & DSISR_NOHPTE) if (dsisr & DSISR_NOHPTE)
flags |= HPTE_NOHPTE_UPDATE; flags |= HPTE_NOHPTE_UPDATE;
@ -1537,13 +1561,31 @@ int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr,
* 2) user space access kernel space. * 2) user space access kernel space.
*/ */
access |= _PAGE_PRIVILEGED; access |= _PAGE_PRIVILEGED;
if ((msr & MSR_PR) || (region_id == USER_REGION_ID)) if (user_mode(regs) || (region_id == USER_REGION_ID))
access &= ~_PAGE_PRIVILEGED; access &= ~_PAGE_PRIVILEGED;
if (trap == 0x400) if (regs->trap == 0x400)
access |= _PAGE_EXEC; access |= _PAGE_EXEC;
return hash_page_mm(mm, ea, access, trap, flags); err = hash_page_mm(mm, ea, access, regs->trap, flags);
if (unlikely(err < 0)) {
// failed to instert a hash PTE due to an hypervisor error
if (user_mode(regs)) {
if (IS_ENABLED(CONFIG_PPC_SUBPAGE_PROT) && err == -2)
_exception(SIGSEGV, regs, SEGV_ACCERR, ea);
else
_exception(SIGBUS, regs, BUS_ADRERR, ea);
} else {
bad_page_fault(regs, ea, SIGBUS);
}
err = 0;
} else if (err) {
page_fault:
err = do_page_fault(regs, ea, dsisr);
}
return err;
} }
#ifdef CONFIG_PPC_MM_SLICES #ifdef CONFIG_PPC_MM_SLICES
@ -1843,27 +1885,6 @@ void flush_hash_range(unsigned long number, int local)
} }
} }
/*
* low_hash_fault is called when we the low level hash code failed
* to instert a PTE due to an hypervisor error
*/
void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc)
{
enum ctx_state prev_state = exception_enter();
if (user_mode(regs)) {
#ifdef CONFIG_PPC_SUBPAGE_PROT
if (rc == -2)
_exception(SIGSEGV, regs, SEGV_ACCERR, address);
else
#endif
_exception(SIGBUS, regs, BUS_ADRERR, address);
} else
bad_page_fault(regs, address, SIGBUS);
exception_exit(prev_state);
}
long hpte_insert_repeating(unsigned long hash, unsigned long vpn, long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
unsigned long pa, unsigned long rflags, unsigned long pa, unsigned long rflags,
unsigned long vflags, int psize, int ssize) unsigned long vflags, int psize, int ssize)