iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x86/mm: Pre-allocate P4D/PUD pages for vmalloc area

Browse Source 
Pre-allocate the page-table pages for the vmalloc area at the level
which needs synchronization on x86-64, which is P4D for 5-level and
PUD for 4-level paging.

Doing this at boot makes sure no synchronization of that area is
necessary at runtime. The synchronization takes the pgd_lock and
iterates over all page-tables in the system, so it can take quite long
and is better avoided.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Mike Rapoport <rppt@linux.ibm.com>
Link: https://lore.kernel.org/r/20200721095953.6218-2-joro@8bytes.org
This commit is contained in:
Joerg Roedel 2020-07-21 11:59:51 +02:00 committed by Ingo Molnar
parent 92ed301919
commit 6eb82f9940
1 changed files with 52 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
arch/x86/mm/init_64.c
View File

@ -1238,6 +1238,56 @@ static void __init register_page_bootmem_info(void)
#endif #endif
} }
/*
* Pre-allocates page-table pages for the vmalloc area in the kernel page-table.
* Only the level which needs to be synchronized between all page-tables is
* allocated because the synchronization can be expensive.
*/
static void __init preallocate_vmalloc_pages(void)
{
unsigned long addr;
const char *lvl;
for (addr = VMALLOC_START; addr <= VMALLOC_END; addr = ALIGN(addr + 1, PGDIR_SIZE)) {
pgd_t *pgd = pgd_offset_k(addr);
p4d_t *p4d;
pud_t *pud;
p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d)) {
/* Can only happen with 5-level paging */
p4d = p4d_alloc(&init_mm, pgd, addr);
if (!p4d) {
lvl = "p4d";
goto failed;
}
}
if (pgtable_l5_enabled())
continue;
pud = pud_offset(p4d, addr);
if (pud_none(*pud)) {
/* Ends up here only with 4-level paging */
pud = pud_alloc(&init_mm, p4d, addr);
if (!pud) {
lvl = "pud";
goto failed;
}
}
}
return;
failed:
/*
* The pages have to be there now or they will be missing in
* process page-tables later.
*/
panic("Failed to pre-allocate %s pages for vmalloc area\n", lvl);
}
void __init mem_init(void) void __init mem_init(void)
{ {
pci_iommu_alloc(); pci_iommu_alloc();
@ -1261,6 +1311,8 @@ void __init mem_init(void)
if (get_gate_vma(&init_mm)) if (get_gate_vma(&init_mm))
kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER); kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER);
preallocate_vmalloc_pages();
mem_init_print_info(NULL); mem_init_print_info(NULL);
} }