iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kaiser: x86_cr3_pcid_noflush and x86_cr3_pcid_user

Browse Source 
Mostly this commit is just unshouting X86_CR3_PCID_KERN_VAR and
X86_CR3_PCID_USER_VAR: we usually name variables in lower-case.

But why does x86_cr3_pcid_noflush need to be __aligned(PAGE_SIZE)?
Ah, it's a leftover from when kaiser_add_user_map() once complained
about mapping the same page twice.  Make it __read_mostly instead.
(I'm a little uneasy about all the unrelated data which shares its
page getting user-mapped too, but that was so before, and not a big
deal: though we call it user-mapped, it's not mapped with _PAGE_USER.)

And there is a little change around the two calls to do_nmi().
Previously they set the NOFLUSH bit (if PCID supported) when
forcing to kernel context before do_nmi(); now they also have the
NOFLUSH bit set (if PCID supported) when restoring context after:
nothing done in do_nmi() should require a TLB to be flushed here.

Signed-off-by: Hugh Dickins <hughd@google.com>
Acked-by: Jiri Kosina <jkosina@suse.cz>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
Hugh Dickins
2017-08-27 16:24:27 -07:00
committed by Greg Kroah-Hartman
parent 3b4ce0e1a1
commit 20268a10ff
3 changed files with 16 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
arch/x86/entry/entry_64.S
View File

@ -1290,11 +1290,11 @@ ENTRY(nmi)
/* Unconditionally use kernel CR3 for do_nmi() */ /* Unconditionally use kernel CR3 for do_nmi() */
/* %rax is saved above, so OK to clobber here */ /* %rax is saved above, so OK to clobber here */
movq %cr3, %rax movq %cr3, %rax
/* If PCID enabled, NOFLUSH now and NOFLUSH on return */
orq x86_cr3_pcid_noflush, %rax
pushq %rax pushq %rax
/* mask off "user" bit of pgd address and 12 PCID bits: */ /* mask off "user" bit of pgd address and 12 PCID bits: */
andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax
/* Add back kernel PCID and "no flush" bit */
orq X86_CR3_PCID_KERN_VAR, %rax
movq %rax, %cr3 movq %rax, %cr3
#endif #endif
call do_nmi call do_nmi
@ -1534,11 +1534,11 @@ end_repeat_nmi:
/* Unconditionally use kernel CR3 for do_nmi() */ /* Unconditionally use kernel CR3 for do_nmi() */
/* %rax is saved above, so OK to clobber here */ /* %rax is saved above, so OK to clobber here */
movq %cr3, %rax movq %cr3, %rax
/* If PCID enabled, NOFLUSH now and NOFLUSH on return */
orq x86_cr3_pcid_noflush, %rax
pushq %rax pushq %rax
/* mask off "user" bit of pgd address and 12 PCID bits: */ /* mask off "user" bit of pgd address and 12 PCID bits: */
andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax
/* Add back kernel PCID and "no flush" bit */
orq X86_CR3_PCID_KERN_VAR, %rax
movq %rax, %cr3 movq %rax, %cr3
#endif #endif

11
arch/x86/include/asm/kaiser.h
View File

@ -25,7 +25,7 @@
.macro _SWITCH_TO_KERNEL_CR3 reg .macro _SWITCH_TO_KERNEL_CR3 reg
movq %cr3, \reg movq %cr3, \reg
andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg
orq X86_CR3_PCID_KERN_VAR, \reg orq x86_cr3_pcid_noflush, \reg
movq \reg, %cr3 movq \reg, %cr3
.endm .endm
@ -37,11 +37,10 @@ movq \reg, %cr3
* not enabled): so that the one register can update both memory and cr3. * not enabled): so that the one register can update both memory and cr3.
*/ */
movq %cr3, \reg movq %cr3, \reg
andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg orq PER_CPU_VAR(x86_cr3_pcid_user), \reg
orq PER_CPU_VAR(X86_CR3_PCID_USER_VAR), \reg
js 9f js 9f
/* FLUSH this time, reset to NOFLUSH for next time (if PCID enabled) */ /* FLUSH this time, reset to NOFLUSH for next time (if PCID enabled) */
movb \regb, PER_CPU_VAR(X86_CR3_PCID_USER_VAR+7) movb \regb, PER_CPU_VAR(x86_cr3_pcid_user+7)
9: 9:
movq \reg, %cr3 movq \reg, %cr3
.endm .endm
@ -94,8 +93,8 @@ movq PER_CPU_VAR(unsafe_stack_register_backup), %rax
*/ */
DECLARE_PER_CPU_USER_MAPPED(unsigned long, unsafe_stack_register_backup); DECLARE_PER_CPU_USER_MAPPED(unsigned long, unsafe_stack_register_backup);
extern unsigned long X86_CR3_PCID_KERN_VAR; extern unsigned long x86_cr3_pcid_noflush;
DECLARE_PER_CPU(unsigned long, X86_CR3_PCID_USER_VAR); DECLARE_PER_CPU(unsigned long, x86_cr3_pcid_user);
extern char __per_cpu_user_mapped_start[], __per_cpu_user_mapped_end[]; extern char __per_cpu_user_mapped_start[], __per_cpu_user_mapped_end[];

13
arch/x86/mm/kaiser.c
View File

@ -29,8 +29,8 @@ DEFINE_PER_CPU_USER_MAPPED(unsigned long, unsafe_stack_register_backup);
* This is also handy because systems that do not support PCIDs * This is also handy because systems that do not support PCIDs
* just end up or'ing a 0 into their CR3, which does no harm. * just end up or'ing a 0 into their CR3, which does no harm.
*/ */
__aligned(PAGE_SIZE) unsigned long X86_CR3_PCID_KERN_VAR; unsigned long x86_cr3_pcid_noflush __read_mostly;
DEFINE_PER_CPU(unsigned long, X86_CR3_PCID_USER_VAR); DEFINE_PER_CPU(unsigned long, x86_cr3_pcid_user);
/* /*
* At runtime, the only things we map are some things for CPU * At runtime, the only things we map are some things for CPU
@ -304,7 +304,8 @@ void __init kaiser_init(void)
sizeof(gate_desc) * NR_VECTORS, sizeof(gate_desc) * NR_VECTORS,
__PAGE_KERNEL); __PAGE_KERNEL);
kaiser_add_user_map_early(&X86_CR3_PCID_KERN_VAR, PAGE_SIZE, kaiser_add_user_map_early(&x86_cr3_pcid_noflush,
sizeof(x86_cr3_pcid_noflush),
__PAGE_KERNEL); __PAGE_KERNEL);
} }
@ -384,8 +385,8 @@ void kaiser_setup_pcid(void)
* These variables are used by the entry/exit * These variables are used by the entry/exit
* code to change PCID and pgd and TLB flushing. * code to change PCID and pgd and TLB flushing.
*/ */
X86_CR3_PCID_KERN_VAR = kern_cr3; x86_cr3_pcid_noflush = kern_cr3;
this_cpu_write(X86_CR3_PCID_USER_VAR, user_cr3); this_cpu_write(x86_cr3_pcid_user, user_cr3);
} }
/* /*
@ -395,7 +396,7 @@ void kaiser_setup_pcid(void)
*/ */
void kaiser_flush_tlb_on_return_to_user(void) void kaiser_flush_tlb_on_return_to_user(void)
{ {
this_cpu_write(X86_CR3_PCID_USER_VAR, this_cpu_write(x86_cr3_pcid_user,
X86_CR3_PCID_USER_FLUSH | KAISER_SHADOW_PGD_OFFSET); X86_CR3_PCID_USER_FLUSH | KAISER_SHADOW_PGD_OFFSET);
} }
EXPORT_SYMBOL(kaiser_flush_tlb_on_return_to_user); EXPORT_SYMBOL(kaiser_flush_tlb_on_return_to_user);