Kernel/linux-next
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git synced 2024-12-29 17:22:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

KVM: Protect vCPU's "last run PID" with rwlock, not RCU

Browse Source 
To avoid jitter on KVM_RUN due to synchronize_rcu(), use a rwlock instead
of RCU to protect vcpu->pid, a.k.a. the pid of the task last used to a
vCPU.  When userspace is doing M:N scheduling of tasks to vCPUs, e.g. to
run SEV migration helper vCPUs during post-copy, the synchronize_rcu()
needed to change the PID associated with the vCPU can stall for hundreds
of milliseconds, which is problematic for latency sensitive post-copy
operations.

In the directed yield path, do not acquire the lock if it's contended,
i.e. if the associated PID is changing, as that means the vCPU's task is
already running.

Reported-by: Steve Rutherford <srutherford@google.com>
Reviewed-by: Steve Rutherford <srutherford@google.com>
Acked-by: Oliver Upton <oliver.upton@linux.dev>
Link: https://lore.kernel.org/r/20240802200136.329973-3-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>
This commit is contained in:
Sean Christopherson 2024-08-02 13:01:36 -07:00
parent 6cf9ef23d9
commit 3e7f43188e
3 changed files with 28 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/arm64/include/asm/kvm_host.h
View File

@ -1140,7 +1140,7 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
void kvm_arm_halt_guest(struct kvm *kvm); void kvm_arm_halt_guest(struct kvm *kvm);
void kvm_arm_resume_guest(struct kvm *kvm); void kvm_arm_resume_guest(struct kvm *kvm);
#define vcpu_has_run_once(vcpu) !!rcu_access_pointer((vcpu)->pid) #define vcpu_has_run_once(vcpu) (!!READ_ONCE((vcpu)->pid))
#ifndef __KVM_NVHE_HYPERVISOR__ #ifndef __KVM_NVHE_HYPERVISOR__
#define kvm_call_hyp_nvhe(f, ...) \ #define kvm_call_hyp_nvhe(f, ...) \

3
include/linux/kvm_host.h
View File

@ -334,7 +334,8 @@ struct kvm_vcpu {
#ifndef __KVM_HAVE_ARCH_WQP #ifndef __KVM_HAVE_ARCH_WQP
struct rcuwait wait; struct rcuwait wait;
#endif #endif
struct pid __rcu *pid; struct pid *pid;
rwlock_t pid_lock;
int sigset_active; int sigset_active;
sigset_t sigset; sigset_t sigset;
unsigned int halt_poll_ns; unsigned int halt_poll_ns;

39
virt/kvm/kvm_main.c
View File

@ -447,6 +447,7 @@ static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
vcpu->kvm = kvm; vcpu->kvm = kvm;
vcpu->vcpu_id = id; vcpu->vcpu_id = id;
vcpu->pid = NULL; vcpu->pid = NULL;
rwlock_init(&vcpu->pid_lock);
#ifndef __KVM_HAVE_ARCH_WQP #ifndef __KVM_HAVE_ARCH_WQP
rcuwait_init(&vcpu->wait); rcuwait_init(&vcpu->wait);
#endif #endif
@ -474,7 +475,7 @@ static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu)
* the vcpu->pid pointer, and at destruction time all file descriptors * the vcpu->pid pointer, and at destruction time all file descriptors
* are already gone. * are already gone.
*/ */
put_pid(rcu_dereference_protected(vcpu->pid, 1)); put_pid(vcpu->pid);
free_page((unsigned long)vcpu->run); free_page((unsigned long)vcpu->run);
kmem_cache_free(kvm_vcpu_cache, vcpu); kmem_cache_free(kvm_vcpu_cache, vcpu);
@ -3770,15 +3771,17 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
int kvm_vcpu_yield_to(struct kvm_vcpu *target) int kvm_vcpu_yield_to(struct kvm_vcpu *target)
{ {
struct pid *pid;
struct task_struct *task = NULL; struct task_struct *task = NULL;
int ret; int ret;
rcu_read_lock(); if (!read_trylock(&target->pid_lock))
pid = rcu_dereference(target->pid); return 0;
if (pid)
task = get_pid_task(pid, PIDTYPE_PID); if (target->pid)
rcu_read_unlock(); task = get_pid_task(target->pid, PIDTYPE_PID);
read_unlock(&target->pid_lock);
if (!task) if (!task)
return 0; return 0;
ret = yield_to(task, 1); ret = yield_to(task, 1);
@ -4030,9 +4033,9 @@ static int vcpu_get_pid(void *data, u64 *val)
{ {
struct kvm_vcpu *vcpu = data; struct kvm_vcpu *vcpu = data;
rcu_read_lock(); read_lock(&vcpu->pid_lock);
*val = pid_nr(rcu_dereference(vcpu->pid)); *val = pid_nr(vcpu->pid);
rcu_read_unlock(); read_unlock(&vcpu->pid_lock);
return 0; return 0;
} }
@ -4318,7 +4321,14 @@ static long kvm_vcpu_ioctl(struct file *filp,
r = -EINVAL; r = -EINVAL;
if (arg) if (arg)
goto out; goto out;
oldpid = rcu_access_pointer(vcpu->pid);
/*
* Note, vcpu->pid is primarily protected by vcpu->mutex. The
* dedicated r/w lock allows other tasks, e.g. other vCPUs, to
* read vcpu->pid while this vCPU is in KVM_RUN, e.g. to yield
* directly to this vCPU
*/
oldpid = vcpu->pid;
if (unlikely(oldpid != task_pid(current))) { if (unlikely(oldpid != task_pid(current))) {
/* The thread running this VCPU changed. */ /* The thread running this VCPU changed. */
struct pid *newpid; struct pid *newpid;
@ -4328,9 +4338,10 @@ static long kvm_vcpu_ioctl(struct file *filp,
break; break;
newpid = get_task_pid(current, PIDTYPE_PID); newpid = get_task_pid(current, PIDTYPE_PID);
rcu_assign_pointer(vcpu->pid, newpid); write_lock(&vcpu->pid_lock);
if (oldpid) vcpu->pid = newpid;
synchronize_rcu(); write_unlock(&vcpu->pid_lock);
put_pid(oldpid); put_pid(oldpid);
} }
vcpu->wants_to_run = !READ_ONCE(vcpu->run->immediate_exit__unsafe); vcpu->wants_to_run = !READ_ONCE(vcpu->run->immediate_exit__unsafe);