Skip to content

Commit 14c07ad

Browse files
vittyvkbonzini
vittyvk
authored and
bonzini
committed
x86/kvm/mmu: introduce guest_mmu
When EPT is used for nested guest we need to re-init MMU as shadow EPT MMU (nested_ept_init_mmu_context() does that). When we return back from L2 to L1 kvm_mmu_reset_context() in nested_vmx_load_cr3() resets MMU back to normal TDP mode. Add a special 'guest_mmu' so we can use separate root caches; the improved hit rate is not very important for single vCPU performance, but it avoids contention on the mmu_lock for many vCPUs. On the nested CPUID benchmark, with 16 vCPUs, an L2->L1->L2 vmexit goes from 42k to 26k cycles. Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com> Reviewed-by: Sean Christopherson <sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
1 parent 6a82cd1 commit 14c07ad

File tree

3 files changed

+40
-18
lines changed

3 files changed

+40
-18
lines changed

‎arch/x86/include/asm/kvm_host.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,9 @@ struct kvm_vcpu_arch {
548548
/* Non-nested MMU for L1 */
549549
struct kvm_mmu root_mmu;
550550

551+
/* L1 MMU when running nested */
552+
struct kvm_mmu guest_mmu;
553+
551554
/*
552555
* Paging state of an L2 guest (used for nested npt)
553556
*

‎arch/x86/kvm/mmu.c

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4832,7 +4832,10 @@ EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
48324832
static union kvm_mmu_page_role
48334833
kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty)
48344834
{
4835-
union kvm_mmu_page_role role = vcpu->arch.mmu->base_role;
4835+
union kvm_mmu_page_role role;
4836+
4837+
/* Role is inherited from root_mmu */
4838+
role.word = vcpu->arch.root_mmu.base_role.word;
48364839

48374840
role.level = PT64_ROOT_4LEVEL;
48384841
role.direct = false;
@@ -4982,8 +4985,10 @@ EXPORT_SYMBOL_GPL(kvm_mmu_load);
49824985

49834986
void kvm_mmu_unload(struct kvm_vcpu *vcpu)
49844987
{
4985-
kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOTS_ALL);
4986-
WARN_ON(VALID_PAGE(vcpu->arch.mmu->root_hpa));
4988+
kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);
4989+
WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root_hpa));
4990+
kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
4991+
WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root_hpa));
49874992
}
49884993
EXPORT_SYMBOL_GPL(kvm_mmu_unload);
49894994

@@ -5422,13 +5427,18 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)
54225427

54235428
vcpu->arch.mmu = &vcpu->arch.root_mmu;
54245429
vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
5430+
54255431
vcpu->arch.root_mmu.root_hpa = INVALID_PAGE;
54265432
vcpu->arch.root_mmu.translate_gpa = translate_gpa;
5427-
vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
5428-
54295433
for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
54305434
vcpu->arch.root_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
54315435

5436+
vcpu->arch.guest_mmu.root_hpa = INVALID_PAGE;
5437+
vcpu->arch.guest_mmu.translate_gpa = translate_gpa;
5438+
for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
5439+
vcpu->arch.guest_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
5440+
5441+
vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
54325442
return alloc_mmu_pages(vcpu);
54335443
}
54345444

‎arch/x86/kvm/vmx.c

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8423,8 +8423,10 @@ static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
84238423
vmcs_write64(VMCS_LINK_POINTER, -1ull);
84248424
}
84258425

8426-
static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
8426+
static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu)
84278427
{
8428+
struct vcpu_vmx *vmx = to_vmx(vcpu);
8429+
84288430
if (vmx->nested.current_vmptr == -1ull)
84298431
return;
84308432

@@ -8438,19 +8440,23 @@ static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
84388440
vmx->nested.posted_intr_nv = -1;
84398441

84408442
/* Flush VMCS12 to guest memory */
8441-
kvm_vcpu_write_guest_page(&vmx->vcpu,
8443+
kvm_vcpu_write_guest_page(vcpu,
84428444
vmx->nested.current_vmptr >> PAGE_SHIFT,
84438445
vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
84448446

8447+
kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
8448+
84458449
vmx->nested.current_vmptr = -1ull;
84468450
}
84478451

84488452
/*
84498453
* Free whatever needs to be freed from vmx->nested when L1 goes down, or
84508454
* just stops using VMX.
84518455
*/
8452-
static void free_nested(struct vcpu_vmx *vmx)
8456+
static void free_nested(struct kvm_vcpu *vcpu)
84538457
{
8458+
struct vcpu_vmx *vmx = to_vmx(vcpu);
8459+
84548460
if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
84558461
return;
84568462

@@ -8483,6 +8489,8 @@ static void free_nested(struct vcpu_vmx *vmx)
84838489
vmx->nested.pi_desc = NULL;
84848490
}
84858491

8492+
kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
8493+
84868494
free_loaded_vmcs(&vmx->nested.vmcs02);
84878495
}
84888496

@@ -8491,7 +8499,7 @@ static int handle_vmoff(struct kvm_vcpu *vcpu)
84918499
{
84928500
if (!nested_vmx_check_permission(vcpu))
84938501
return 1;
8494-
free_nested(to_vmx(vcpu));
8502+
free_nested(vcpu);
84958503
return nested_vmx_succeed(vcpu);
84968504
}
84978505

@@ -8517,7 +8525,7 @@ static int handle_vmclear(struct kvm_vcpu *vcpu)
85178525
VMXERR_VMCLEAR_VMXON_POINTER);
85188526

85198527
if (vmptr == vmx->nested.current_vmptr)
8520-
nested_release_vmcs12(vmx);
8528+
nested_release_vmcs12(vcpu);
85218529

85228530
kvm_vcpu_write_guest(vcpu,
85238531
vmptr + offsetof(struct vmcs12, launch_state),
@@ -8872,7 +8880,8 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
88728880
VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
88738881
}
88748882

8875-
nested_release_vmcs12(vmx);
8883+
nested_release_vmcs12(vcpu);
8884+
88768885
/*
88778886
* Load VMCS12 from guest memory since it is not already
88788887
* cached.
@@ -10928,12 +10937,10 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
1092810937
*/
1092910938
static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu)
1093010939
{
10931-
struct vcpu_vmx *vmx = to_vmx(vcpu);
10932-
10933-
vcpu_load(vcpu);
10934-
vmx_switch_vmcs(vcpu, &vmx->vmcs01);
10935-
free_nested(vmx);
10936-
vcpu_put(vcpu);
10940+
vcpu_load(vcpu);
10941+
vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01);
10942+
free_nested(vcpu);
10943+
vcpu_put(vcpu);
1093710944
}
1093810945

1093910946
static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
@@ -11300,6 +11307,7 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
1130011307
{
1130111308
WARN_ON(mmu_is_nested(vcpu));
1130211309

11310+
vcpu->arch.mmu = &vcpu->arch.guest_mmu;
1130311311
kvm_init_shadow_ept_mmu(vcpu,
1130411312
to_vmx(vcpu)->nested.msrs.ept_caps &
1130511313
VMX_EPT_EXECUTE_ONLY_BIT,
@@ -11315,6 +11323,7 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
1131511323

1131611324
static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
1131711325
{
11326+
vcpu->arch.mmu = &vcpu->arch.root_mmu;
1131811327
vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
1131911328
}
1132011329

@@ -13731,7 +13740,7 @@ static void vmx_leave_nested(struct kvm_vcpu *vcpu)
1373113740
to_vmx(vcpu)->nested.nested_run_pending = 0;
1373213741
nested_vmx_vmexit(vcpu, -1, 0, 0);
1373313742
}
13734-
free_nested(to_vmx(vcpu));
13743+
free_nested(vcpu);
1373513744
}
1373613745

1373713746
static int vmx_check_intercept(struct kvm_vcpu *vcpu,

0 commit comments

Comments
 (0)