2020-09-23 11:31:11 -07:00
// SPDX-License-Identifier: GPL-2.0-only
KVM: x86: Unify pr_fmt to use module name for all KVM modules
Define pr_fmt using KBUILD_MODNAME for all KVM x86 code so that printks
use consistent formatting across common x86, Intel, and AMD code. In
addition to providing consistent print formatting, using KBUILD_MODNAME,
e.g. kvm_amd and kvm_intel, allows referencing SVM and VMX (and SEV and
SGX and ...) as technologies without generating weird messages, and
without causing naming conflicts with other kernel code, e.g. "SEV: ",
"tdx: ", "sgx: " etc.. are all used by the kernel for non-KVM subsystems.
Opportunistically move away from printk() for prints that need to be
modified anyways, e.g. to drop a manual "kvm: " prefix.
Opportunistically convert a few SGX WARNs that are similarly modified to
WARN_ONCE; in the very unlikely event that the WARNs fire, odds are good
that they would fire repeatedly and spam the kernel log without providing
unique information in each print.
Note, defining pr_fmt yields undesirable results for code that uses KVM's
printk wrappers, e.g. vcpu_unimpl(). But, that's a pre-existing problem
as SVM/kvm_amd already defines a pr_fmt, and thankfully use of KVM's
wrappers is relatively limited in KVM x86 code.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Paul Durrant <paul@xen.org>
Message-Id: <20221130230934.1014142-35-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2022-11-30 23:09:18 +00:00
# define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
2020-09-23 11:31:11 -07:00
# include <linux/kvm_host.h>
# include <asm/irq_remapping.h>
# include <asm/cpu.h>
# include "lapic.h"
2021-11-22 19:43:11 -05:00
# include "irq.h"
2020-09-23 11:31:11 -07:00
# include "posted_intr.h"
# include "trace.h"
# include "vmx.h"
/*
2021-10-08 19:12:21 -07:00
* Maintain a per - CPU list of vCPUs that need to be awakened by wakeup_handler ( )
* when a WAKEUP_VECTOR interrupted is posted . vCPUs are added to the list when
* the vCPU is scheduled out and is blocking ( e . g . in HLT ) with IRQs enabled .
* The vCPUs posted interrupt descriptor is updated at the same time to set its
* notification vector to WAKEUP_VECTOR , so that posted interrupt from devices
* wake the target vCPUs . vCPUs are removed from the list and the notification
* vector is reset when the vCPU is scheduled in .
2020-09-23 11:31:11 -07:00
*/
2021-12-08 01:52:16 +00:00
static DEFINE_PER_CPU ( struct list_head , wakeup_vcpus_on_cpu ) ;
2021-10-08 19:12:21 -07:00
/*
* Protect the per - CPU list with a per - CPU spinlock to handle task migration .
* When a blocking vCPU is awakened _and_ migrated to a different pCPU , the
* - > sched_in ( ) path will need to take the vCPU off the list of the _previous_
* CPU . IRQs must be disabled when taking this lock , otherwise deadlock will
* occur if a wakeup IRQ arrives and attempts to acquire the lock .
*/
2021-12-08 01:52:16 +00:00
static DEFINE_PER_CPU ( raw_spinlock_t , wakeup_vcpus_on_cpu_lock ) ;
2020-09-23 11:31:11 -07:00
static inline struct pi_desc * vcpu_to_pi_desc ( struct kvm_vcpu * vcpu )
{
return & ( to_vmx ( vcpu ) - > pi_desc ) ;
}
2022-06-24 11:45:45 -04:00
static int pi_try_set_control ( struct pi_desc * pi_desc , u64 * pold , u64 new )
2021-12-08 01:52:13 +00:00
{
/*
* PID . ON can be set at any time by a different vCPU or by hardware ,
* e . g . a device . PID . control must be written atomically , and the
* update must be retried with a fresh snapshot an ON change causes
* the cmpxchg to fail .
*/
2022-06-24 11:45:45 -04:00
if ( ! try_cmpxchg64 ( & pi_desc - > control , pold , new ) )
2021-12-08 01:52:13 +00:00
return - EBUSY ;
return 0 ;
}
2020-09-23 11:31:11 -07:00
void vmx_vcpu_pi_load ( struct kvm_vcpu * vcpu , int cpu )
{
struct pi_desc * pi_desc = vcpu_to_pi_desc ( vcpu ) ;
2021-12-08 01:52:16 +00:00
struct vcpu_vmx * vmx = to_vmx ( vcpu ) ;
2020-09-23 11:31:11 -07:00
struct pi_desc old , new ;
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
unsigned long flags ;
2020-09-23 11:31:11 -07:00
unsigned int dest ;
/*
2021-10-08 19:12:13 -07:00
* To simplify hot - plug and dynamic toggling of APICv , keep PI . NDST and
* PI . SN up - to - date even if there is no assigned device or if APICv is
* deactivated due to a dynamic inhibit bit , e . g . for Hyper - V ' s SyncIC .
2020-09-23 11:31:11 -07:00
*/
2021-10-08 19:12:13 -07:00
if ( ! enable_apicv | | ! lapic_in_kernel ( vcpu ) )
return ;
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
/*
* If the vCPU wasn ' t on the wakeup list and wasn ' t migrated , then the
* full update can be skipped as neither the vector nor the destination
* needs to be changed .
*/
if ( pi_desc - > nv ! = POSTED_INTR_WAKEUP_VECTOR & & vcpu - > cpu = = cpu ) {
/*
* Clear SN if it was set due to being preempted . Again , do
* this even if there is no assigned device for simplicity .
*/
if ( pi_test_and_clear_sn ( pi_desc ) )
goto after_clear_sn ;
2020-09-23 11:31:11 -07:00
return ;
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
}
local_irq_save ( flags ) ;
2020-09-23 11:31:11 -07:00
/*
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
* If the vCPU was waiting for wakeup , remove the vCPU from the wakeup
* list of the _previous_ pCPU , which will not be the same as the
* current pCPU if the task was migrated .
2020-09-23 11:31:11 -07:00
*/
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
if ( pi_desc - > nv = = POSTED_INTR_WAKEUP_VECTOR ) {
2021-12-08 01:52:16 +00:00
raw_spin_lock ( & per_cpu ( wakeup_vcpus_on_cpu_lock , vcpu - > cpu ) ) ;
list_del ( & vmx - > pi_wakeup_list ) ;
raw_spin_unlock ( & per_cpu ( wakeup_vcpus_on_cpu_lock , vcpu - > cpu ) ) ;
2020-09-23 11:31:11 -07:00
}
2021-10-08 19:12:20 -07:00
dest = cpu_physical_id ( cpu ) ;
if ( ! x2apic_mode )
dest = ( dest < < 8 ) & 0xFF00 ;
2022-06-24 11:45:45 -04:00
old . control = READ_ONCE ( pi_desc - > control ) ;
2020-09-23 11:31:11 -07:00
do {
2022-06-24 11:45:45 -04:00
new . control = old . control ;
2020-09-23 11:31:11 -07:00
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
/*
* Clear SN ( as above ) and refresh the destination APIC ID to
* handle task migration ( @ cpu ! = vcpu - > cpu ) .
*/
2021-10-08 19:12:20 -07:00
new . ndst = dest ;
2020-09-23 11:31:11 -07:00
new . sn = 0 ;
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
/*
* Restore the notification vector ; in the blocking case , the
* descriptor was modified on " put " to use the wakeup vector .
*/
new . nv = POSTED_INTR_VECTOR ;
2022-06-24 11:45:45 -04:00
} while ( pi_try_set_control ( pi_desc , & old . control , new . control ) ) ;
2020-09-23 11:31:11 -07:00
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
local_irq_restore ( flags ) ;
2020-09-23 11:31:11 -07:00
after_clear_sn :
/*
* Clear SN before reading the bitmap . The VT - d firmware
* writes the bitmap and reads SN atomically ( 5.2 .3 in the
* spec ) , so it doesn ' t really have a memory barrier that
* pairs with this , but we cannot do that and we need one .
*/
smp_mb__after_atomic ( ) ;
if ( ! pi_is_pir_empty ( pi_desc ) )
pi_set_on ( pi_desc ) ;
}
2021-11-22 19:43:11 -05:00
static bool vmx_can_use_vtd_pi ( struct kvm * kvm )
{
return irqchip_in_kernel ( kvm ) & & enable_apicv & &
kvm_arch_has_assigned_device ( kvm ) & &
irq_remapping_cap ( IRQ_POSTING_CAP ) ;
}
2020-09-23 11:31:11 -07:00
/*
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
* Put the vCPU on this pCPU ' s list of vCPUs that needs to be awakened and set
* WAKEUP as the notification vector in the PI descriptor .
2020-09-23 11:31:11 -07:00
*/
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
static void pi_enable_wakeup_handler ( struct kvm_vcpu * vcpu )
2020-09-23 11:31:11 -07:00
{
struct pi_desc * pi_desc = vcpu_to_pi_desc ( vcpu ) ;
2021-12-08 01:52:16 +00:00
struct vcpu_vmx * vmx = to_vmx ( vcpu ) ;
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
struct pi_desc old , new ;
2021-10-08 19:12:18 -07:00
unsigned long flags ;
2020-09-23 11:31:11 -07:00
2021-10-08 19:12:18 -07:00
local_irq_save ( flags ) ;
2021-10-08 19:12:15 -07:00
2021-12-08 01:52:16 +00:00
raw_spin_lock ( & per_cpu ( wakeup_vcpus_on_cpu_lock , vcpu - > cpu ) ) ;
list_add_tail ( & vmx - > pi_wakeup_list ,
& per_cpu ( wakeup_vcpus_on_cpu , vcpu - > cpu ) ) ;
raw_spin_unlock ( & per_cpu ( wakeup_vcpus_on_cpu_lock , vcpu - > cpu ) ) ;
2020-09-23 11:31:11 -07:00
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
WARN ( pi_desc - > sn , " PI descriptor SN field set before blocking " ) ;
2021-12-08 01:52:13 +00:00
2022-06-24 11:45:45 -04:00
old . control = READ_ONCE ( pi_desc - > control ) ;
2020-09-23 11:31:11 -07:00
do {
/* set 'NV' to 'wakeup vector' */
2022-06-24 11:45:45 -04:00
new . control = old . control ;
2020-09-23 11:31:11 -07:00
new . nv = POSTED_INTR_WAKEUP_VECTOR ;
2022-06-24 11:45:45 -04:00
} while ( pi_try_set_control ( pi_desc , & old . control , new . control ) ) ;
2020-09-23 11:31:11 -07:00
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
/*
* Send a wakeup IPI to this CPU if an interrupt may have been posted
* before the notification vector was updated , in which case the IRQ
* will arrive on the non - wakeup vector . An IPI is needed as calling
* try_to_wake_up ( ) from - > sched_out ( ) isn ' t allowed ( IRQs are not
* enabled until it is safe to call try_to_wake_up ( ) on the task being
* scheduled out ) .
*/
if ( pi_test_on ( & new ) )
2023-08-09 08:21:45 -07:00
__apic_send_IPI_self ( POSTED_INTR_WAKEUP_VECTOR ) ;
2020-09-23 11:31:11 -07:00
2021-10-08 19:12:18 -07:00
local_irq_restore ( flags ) ;
2020-09-23 11:31:11 -07:00
}
2022-04-19 23:45:10 +08:00
static bool vmx_needs_pi_wakeup ( struct kvm_vcpu * vcpu )
{
/*
* The default posted interrupt vector does nothing when
* invoked outside guest mode . Return whether a blocked vCPU
* can be the target of posted interrupts , as is the case when
* using either IPI virtualization or VT - d PI , so that the
* notification vector is switched to the one that calls
* back to the pi_wakeup_handler ( ) function .
*/
return vmx_can_use_ipiv ( vcpu ) | | vmx_can_use_vtd_pi ( vcpu - > kvm ) ;
}
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
void vmx_vcpu_pi_put ( struct kvm_vcpu * vcpu )
2020-09-23 11:31:11 -07:00
{
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
struct pi_desc * pi_desc = vcpu_to_pi_desc ( vcpu ) ;
2021-10-08 19:12:18 -07:00
2022-04-19 23:45:10 +08:00
if ( ! vmx_needs_pi_wakeup ( vcpu ) )
2020-09-23 11:31:11 -07:00
return ;
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
if ( kvm_vcpu_is_blocking ( vcpu ) & & ! vmx_interrupt_blocked ( vcpu ) )
pi_enable_wakeup_handler ( vcpu ) ;
/*
* Set SN when the vCPU is preempted . Note , the vCPU can both be seen
* as blocking and preempted , e . g . if it ' s preempted between setting
* its wait state and manually scheduling out .
*/
if ( vcpu - > preempted )
pi_set_sn ( pi_desc ) ;
2020-09-23 11:31:11 -07:00
}
/*
* Handler for POSTED_INTERRUPT_WAKEUP_VECTOR .
*/
void pi_wakeup_handler ( void )
{
int cpu = smp_processor_id ( ) ;
2022-04-06 19:25:02 +08:00
struct list_head * wakeup_list = & per_cpu ( wakeup_vcpus_on_cpu , cpu ) ;
raw_spinlock_t * spinlock = & per_cpu ( wakeup_vcpus_on_cpu_lock , cpu ) ;
2021-12-08 01:52:16 +00:00
struct vcpu_vmx * vmx ;
2020-09-23 11:31:11 -07:00
2022-04-06 19:25:02 +08:00
raw_spin_lock ( spinlock ) ;
list_for_each_entry ( vmx , wakeup_list , pi_wakeup_list ) {
2020-09-23 11:31:11 -07:00
2021-12-08 01:52:16 +00:00
if ( pi_test_on ( & vmx - > pi_desc ) )
2021-12-08 01:52:30 +00:00
kvm_vcpu_wake_up ( & vmx - > vcpu ) ;
2020-09-23 11:31:11 -07:00
}
2022-04-06 19:25:02 +08:00
raw_spin_unlock ( spinlock ) ;
2020-09-23 11:31:11 -07:00
}
2020-10-24 04:08:37 -04:00
void __init pi_init_cpu ( int cpu )
2020-09-23 11:31:11 -07:00
{
2021-12-08 01:52:16 +00:00
INIT_LIST_HEAD ( & per_cpu ( wakeup_vcpus_on_cpu , cpu ) ) ;
raw_spin_lock_init ( & per_cpu ( wakeup_vcpus_on_cpu_lock , cpu ) ) ;
2020-09-23 11:31:11 -07:00
}
bool pi_has_pending_interrupt ( struct kvm_vcpu * vcpu )
{
struct pi_desc * pi_desc = vcpu_to_pi_desc ( vcpu ) ;
return pi_test_on ( pi_desc ) | |
( pi_test_sn ( pi_desc ) & & ! pi_is_pir_empty ( pi_desc ) ) ;
}
KVM: VMX: update vcpu posted-interrupt descriptor when assigning device
For VMX, when a vcpu enters HLT emulation, pi_post_block will:
1) Add vcpu to per-cpu list of blocked vcpus.
2) Program the posted-interrupt descriptor "notification vector"
to POSTED_INTR_WAKEUP_VECTOR
With interrupt remapping, an interrupt will set the PIR bit for the
vector programmed for the device on the CPU, test-and-set the
ON bit on the posted interrupt descriptor, and if the ON bit is clear
generate an interrupt for the notification vector.
This way, the target CPU wakes upon a device interrupt and wakes up
the target vcpu.
Problem is that pi_post_block only programs the notification vector
if kvm_arch_has_assigned_device() is true. Its possible for the
following to happen:
1) vcpu V HLTs on pcpu P, kvm_arch_has_assigned_device is false,
notification vector is not programmed
2) device is assigned to VM
3) device interrupts vcpu V, sets ON bit
(notification vector not programmed, so pcpu P remains in idle)
4) vcpu 0 IPIs vcpu V (in guest), but since pi descriptor ON bit is set,
kvm_vcpu_kick is skipped
5) vcpu 0 busy spins on vcpu V's response for several seconds, until
RCU watchdog NMIs all vCPUs.
To fix this, use the start_assignment kvm_x86_ops callback to kick
vcpus out of the halt loop, so the notification vector is
properly reprogrammed to the wakeup vector.
Reported-by: Pei Zhang <pezhang@redhat.com>
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Message-Id: <20210526172014.GA29007@fuller.cnet>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-05-26 14:20:14 -03:00
/*
* Bail out of the block loop if the VM has an assigned
* device , but the blocking vCPU didn ' t reconfigure the
* PI . NV to the wakeup vector , i . e . the assigned device
KVM: VMX: Handle PI descriptor updates during vcpu_put/load
Move the posted interrupt pre/post_block logic into vcpu_put/load
respectively, using the kvm_vcpu_is_blocking() to determining whether or
not the wakeup handler needs to be set (and unset). This avoids updating
the PI descriptor if halt-polling is successful, reduces the number of
touchpoints for updating the descriptor, and eliminates the confusing
behavior of intentionally leaving a "stale" PI.NDST when a blocking vCPU
is scheduled back in after preemption.
The downside is that KVM will do the PID update twice if the vCPU is
preempted after prepare_to_rcuwait() but before schedule(), but that's a
rare case (and non-existent on !PREEMPT kernels).
The notable wart is the need to send a self-IPI on the wakeup vector if
an outstanding notification is pending after configuring the wakeup
vector. Ideally, KVM would just do a kvm_vcpu_wake_up() in this case,
but the scheduler doesn't support waking a task from its preemption
notifier callback, i.e. while the task is right in the middle of
being scheduled out.
Note, setting the wakeup vector before halt-polling is not necessary:
once the pending IRQ will be recorded in the PIR, kvm_vcpu_has_events()
will detect this (via kvm_cpu_get_interrupt(), kvm_apic_get_interrupt(),
apic_has_interrupt_for_ppr() and finally vmx_sync_pir_to_irr()) and
terminate the polling.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Message-Id: <20211208015236.1616697-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-12-08 01:52:14 +00:00
* came along after the initial check in vmx_vcpu_pi_put ( ) .
KVM: VMX: update vcpu posted-interrupt descriptor when assigning device
For VMX, when a vcpu enters HLT emulation, pi_post_block will:
1) Add vcpu to per-cpu list of blocked vcpus.
2) Program the posted-interrupt descriptor "notification vector"
to POSTED_INTR_WAKEUP_VECTOR
With interrupt remapping, an interrupt will set the PIR bit for the
vector programmed for the device on the CPU, test-and-set the
ON bit on the posted interrupt descriptor, and if the ON bit is clear
generate an interrupt for the notification vector.
This way, the target CPU wakes upon a device interrupt and wakes up
the target vcpu.
Problem is that pi_post_block only programs the notification vector
if kvm_arch_has_assigned_device() is true. Its possible for the
following to happen:
1) vcpu V HLTs on pcpu P, kvm_arch_has_assigned_device is false,
notification vector is not programmed
2) device is assigned to VM
3) device interrupts vcpu V, sets ON bit
(notification vector not programmed, so pcpu P remains in idle)
4) vcpu 0 IPIs vcpu V (in guest), but since pi descriptor ON bit is set,
kvm_vcpu_kick is skipped
5) vcpu 0 busy spins on vcpu V's response for several seconds, until
RCU watchdog NMIs all vCPUs.
To fix this, use the start_assignment kvm_x86_ops callback to kick
vcpus out of the halt loop, so the notification vector is
properly reprogrammed to the wakeup vector.
Reported-by: Pei Zhang <pezhang@redhat.com>
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Message-Id: <20210526172014.GA29007@fuller.cnet>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-05-26 14:20:14 -03:00
*/
void vmx_pi_start_assignment ( struct kvm * kvm )
{
if ( ! irq_remapping_cap ( IRQ_POSTING_CAP ) )
return ;
kvm_make_all_cpus_request ( kvm , KVM_REQ_UNBLOCK ) ;
}
2020-09-23 11:31:11 -07:00
/*
2022-01-28 00:51:59 +00:00
* vmx_pi_update_irte - set IRTE for Posted - Interrupts
2020-09-23 11:31:11 -07:00
*
* @ kvm : kvm
* @ host_irq : host irq of the interrupt
* @ guest_irq : gsi of the interrupt
* @ set : set or unset PI
* returns 0 on success , < 0 on failure
*/
2022-01-28 00:51:59 +00:00
int vmx_pi_update_irte ( struct kvm * kvm , unsigned int host_irq ,
uint32_t guest_irq , bool set )
2020-09-23 11:31:11 -07:00
{
struct kvm_kernel_irq_routing_entry * e ;
struct kvm_irq_routing_table * irq_rt ;
struct kvm_lapic_irq irq ;
struct kvm_vcpu * vcpu ;
struct vcpu_data vcpu_info ;
int idx , ret = 0 ;
2021-11-22 19:43:11 -05:00
if ( ! vmx_can_use_vtd_pi ( kvm ) )
2020-09-23 11:31:11 -07:00
return 0 ;
idx = srcu_read_lock ( & kvm - > irq_srcu ) ;
irq_rt = srcu_dereference ( kvm - > irq_routing , & kvm - > irq_srcu ) ;
if ( guest_irq > = irq_rt - > nr_rt_entries | |
hlist_empty ( & irq_rt - > map [ guest_irq ] ) ) {
pr_warn_once ( " no route for guest_irq %u/%u (broken user space?) \n " ,
guest_irq , irq_rt - > nr_rt_entries ) ;
goto out ;
}
hlist_for_each_entry ( e , & irq_rt - > map [ guest_irq ] , link ) {
if ( e - > type ! = KVM_IRQ_ROUTING_MSI )
continue ;
/*
* VT - d PI cannot support posting multicast / broadcast
* interrupts to a vCPU , we still use interrupt remapping
* for these kind of interrupts .
*
* For lowest - priority interrupts , we only support
* those with single CPU as the destination , e . g . user
* configures the interrupts via / proc / irq or uses
* irqbalance to make the interrupts single - CPU .
*
* We will support full lowest - priority interrupt later .
*
* In addition , we can only inject generic interrupts using
* the PI mechanism , refuse to route others through it .
*/
kvm_set_msi_irq ( kvm , e , & irq ) ;
if ( ! kvm_intr_is_single_vcpu ( kvm , & irq , & vcpu ) | |
! kvm_irq_is_postable ( & irq ) ) {
/*
* Make sure the IRTE is in remapped mode if
* we don ' t handle it in posted mode .
*/
ret = irq_set_vcpu_affinity ( host_irq , NULL ) ;
if ( ret < 0 ) {
printk ( KERN_INFO
" failed to back to remapped mode, irq: %u \n " ,
host_irq ) ;
goto out ;
}
continue ;
}
2022-03-04 11:49:33 -08:00
vcpu_info . pi_desc_addr = __pa ( vcpu_to_pi_desc ( vcpu ) ) ;
2020-09-23 11:31:11 -07:00
vcpu_info . vector = irq . vector ;
trace_kvm_pi_irte_update ( host_irq , vcpu - > vcpu_id , e - > gsi ,
vcpu_info . vector , vcpu_info . pi_desc_addr , set ) ;
if ( set )
ret = irq_set_vcpu_affinity ( host_irq , & vcpu_info ) ;
else
ret = irq_set_vcpu_affinity ( host_irq , NULL ) ;
if ( ret < 0 ) {
printk ( KERN_INFO " %s: failed to update PI IRTE \n " ,
__func__ ) ;
goto out ;
}
}
ret = 0 ;
out :
srcu_read_unlock ( & kvm - > irq_srcu , idx ) ;
return ret ;
}
