ba3a6120a4
Use an atomic XCHG to write TDP MMU SPTEs that have volatile bits, even
if mmu_lock is held for write, as volatile SPTEs can be written by other
tasks/vCPUs outside of mmu_lock. If a vCPU uses the to-be-modified SPTE
to write a page, the CPU can cache the translation as WRITABLE in the TLB
despite it being seen by KVM as !WRITABLE, and/or KVM can clobber the
Accessed/Dirty bits and not properly tag the backing page.
Exempt non-leaf SPTEs from atomic updates as KVM itself doesn't modify
non-leaf SPTEs without holding mmu_lock, they do not have Dirty bits, and
KVM doesn't consume the Accessed bit of non-leaf SPTEs.
Dropping the Dirty and/or Writable bits is most problematic for dirty
logging, as doing so can result in a missed TLB flush and eventually a
missed dirty page. In the unlikely event that the only dirty page(s) is
a clobbered SPTE, clear_dirty_gfn_range() will see the SPTE as not dirty
(based on the Dirty or Writable bit depending on the method) and so not
update the SPTE and ultimately not flush. If the SPTE is cached in the
TLB as writable before it is clobbered, the guest can continue writing
the associated page without ever taking a write-protect fault.
For most (all?) file back memory, dropping the Dirty bit is a non-issue.
The primary MMU write-protects its PTEs on writeback, i.e. KVM's dirty
bit is effectively ignored because the primary MMU will mark that page
dirty when the write-protection is lifted, e.g. when KVM faults the page
back in for write.
The Accessed bit is a complete non-issue. Aside from being unused for
non-leaf SPTEs, KVM doesn't do a TLB flush when aging SPTEs, i.e. the
Accessed bit may be dropped anyways.
Lastly, the Writable bit is also problematic as an extension of the Dirty
bit, as KVM (correctly) treats the Dirty bit as volatile iff the SPTE is
!DIRTY && WRITABLE. If KVM fixes an MMU-writable, but !WRITABLE, SPTE
out of mmu_lock, then it can allow the CPU to set the Dirty bit despite
the SPTE being !WRITABLE when it is checked by KVM. But that all depends
on the Dirty bit being problematic in the first place.
Fixes: 2f2fad0897cb ("kvm: x86/mmu: Add functions to handle changed TDP SPTEs")
Cc: stable@vger.kernel.org
Cc: Ben Gardon <bgardon@google.com>
Cc: David Matlack <dmatlack@google.com>
Cc: Venkatesh Srinivas <venkateshs@google.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Message-Id: <20220423034752.1161007-4-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
119 lines
3.8 KiB
C
119 lines
3.8 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
#ifndef __KVM_X86_MMU_TDP_ITER_H
|
|
#define __KVM_X86_MMU_TDP_ITER_H
|
|
|
|
#include <linux/kvm_host.h>
|
|
|
|
#include "mmu.h"
|
|
#include "spte.h"
|
|
|
|
/*
|
|
* TDP MMU SPTEs are RCU protected to allow paging structures (non-leaf SPTEs)
|
|
* to be zapped while holding mmu_lock for read, and to allow TLB flushes to be
|
|
* batched without having to collect the list of zapped SPs. Flows that can
|
|
* remove SPs must service pending TLB flushes prior to dropping RCU protection.
|
|
*/
|
|
static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep)
|
|
{
|
|
return READ_ONCE(*rcu_dereference(sptep));
|
|
}
|
|
|
|
static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte)
|
|
{
|
|
return xchg(rcu_dereference(sptep), new_spte);
|
|
}
|
|
|
|
static inline void __kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 new_spte)
|
|
{
|
|
WRITE_ONCE(*rcu_dereference(sptep), new_spte);
|
|
}
|
|
|
|
static inline u64 kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 old_spte,
|
|
u64 new_spte, int level)
|
|
{
|
|
/*
|
|
* Atomically write the SPTE if it is a shadow-present, leaf SPTE with
|
|
* volatile bits, i.e. has bits that can be set outside of mmu_lock.
|
|
* The Writable bit can be set by KVM's fast page fault handler, and
|
|
* Accessed and Dirty bits can be set by the CPU.
|
|
*
|
|
* Note, non-leaf SPTEs do have Accessed bits and those bits are
|
|
* technically volatile, but KVM doesn't consume the Accessed bit of
|
|
* non-leaf SPTEs, i.e. KVM doesn't care if it clobbers the bit. This
|
|
* logic needs to be reassessed if KVM were to use non-leaf Accessed
|
|
* bits, e.g. to skip stepping down into child SPTEs when aging SPTEs.
|
|
*/
|
|
if (is_shadow_present_pte(old_spte) && is_last_spte(old_spte, level) &&
|
|
spte_has_volatile_bits(old_spte))
|
|
return kvm_tdp_mmu_write_spte_atomic(sptep, new_spte);
|
|
|
|
__kvm_tdp_mmu_write_spte(sptep, new_spte);
|
|
return old_spte;
|
|
}
|
|
|
|
/*
|
|
* A TDP iterator performs a pre-order walk over a TDP paging structure.
|
|
*/
|
|
struct tdp_iter {
|
|
/*
|
|
* The iterator will traverse the paging structure towards the mapping
|
|
* for this GFN.
|
|
*/
|
|
gfn_t next_last_level_gfn;
|
|
/*
|
|
* The next_last_level_gfn at the time when the thread last
|
|
* yielded. Only yielding when the next_last_level_gfn !=
|
|
* yielded_gfn helps ensure forward progress.
|
|
*/
|
|
gfn_t yielded_gfn;
|
|
/* Pointers to the page tables traversed to reach the current SPTE */
|
|
tdp_ptep_t pt_path[PT64_ROOT_MAX_LEVEL];
|
|
/* A pointer to the current SPTE */
|
|
tdp_ptep_t sptep;
|
|
/* The lowest GFN mapped by the current SPTE */
|
|
gfn_t gfn;
|
|
/* The level of the root page given to the iterator */
|
|
int root_level;
|
|
/* The lowest level the iterator should traverse to */
|
|
int min_level;
|
|
/* The iterator's current level within the paging structure */
|
|
int level;
|
|
/* The address space ID, i.e. SMM vs. regular. */
|
|
int as_id;
|
|
/* A snapshot of the value at sptep */
|
|
u64 old_spte;
|
|
/*
|
|
* Whether the iterator has a valid state. This will be false if the
|
|
* iterator walks off the end of the paging structure.
|
|
*/
|
|
bool valid;
|
|
/*
|
|
* True if KVM dropped mmu_lock and yielded in the middle of a walk, in
|
|
* which case tdp_iter_next() needs to restart the walk at the root
|
|
* level instead of advancing to the next entry.
|
|
*/
|
|
bool yielded;
|
|
};
|
|
|
|
/*
|
|
* Iterates over every SPTE mapping the GFN range [start, end) in a
|
|
* preorder traversal.
|
|
*/
|
|
#define for_each_tdp_pte_min_level(iter, root, min_level, start, end) \
|
|
for (tdp_iter_start(&iter, root, min_level, start); \
|
|
iter.valid && iter.gfn < end; \
|
|
tdp_iter_next(&iter))
|
|
|
|
#define for_each_tdp_pte(iter, root, start, end) \
|
|
for_each_tdp_pte_min_level(iter, root, PG_LEVEL_4K, start, end)
|
|
|
|
tdp_ptep_t spte_to_child_pt(u64 pte, int level);
|
|
|
|
void tdp_iter_start(struct tdp_iter *iter, struct kvm_mmu_page *root,
|
|
int min_level, gfn_t next_last_level_gfn);
|
|
void tdp_iter_next(struct tdp_iter *iter);
|
|
void tdp_iter_restart(struct tdp_iter *iter);
|
|
|
|
#endif /* __KVM_X86_MMU_TDP_ITER_H */
|