Skip to content

Commit 1b7ce8e

Browse files
sean-jcsean-jc
committed
KVM: Move x86's MMU notifier memslot walkers to generic code
Signed-off-by: Sean Christopherson <seanjc@google.com>
1 parent de82b91 commit 1b7ce8e

File tree

6 files changed

+278
-244
lines changed

6 files changed

+278
-244
lines changed

‎arch/x86/include/asm/kvm_host.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1718,6 +1718,7 @@ asmlinkage void kvm_spurious_fault(void);
17181718
_ASM_EXTABLE(666b, 667b)
17191719

17201720
#define KVM_ARCH_WANT_MMU_NOTIFIER
1721+
#define KVM_ARCH_WANT_NEW_MMU_NOTIFIER_APIS
17211722

17221723
int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
17231724
int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);

‎arch/x86/kvm/mmu/mmu.c

Lines changed: 47 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -1298,26 +1298,25 @@ static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
12981298
return flush;
12991299
}
13001300

1301-
static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1302-
struct kvm_memory_slot *slot, gfn_t gfn, int level,
1303-
unsigned long data)
1301+
static bool kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1302+
struct kvm_memory_slot *slot, gfn_t gfn, int level,
1303+
pte_t unused)
13041304
{
13051305
return kvm_zap_rmapp(kvm, rmap_head, slot);
13061306
}
13071307

1308-
static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1309-
struct kvm_memory_slot *slot, gfn_t gfn, int level,
1310-
unsigned long data)
1308+
static bool kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1309+
struct kvm_memory_slot *slot, gfn_t gfn, int level,
1310+
pte_t pte)
13111311
{
13121312
u64 *sptep;
13131313
struct rmap_iterator iter;
13141314
int need_flush = 0;
13151315
u64 new_spte;
1316-
pte_t *ptep = (pte_t *)data;
13171316
kvm_pfn_t new_pfn;
13181317

1319-
WARN_ON(pte_huge(*ptep));
1320-
new_pfn = pte_pfn(*ptep);
1318+
WARN_ON(pte_huge(pte));
1319+
new_pfn = pte_pfn(pte);
13211320

13221321
restart:
13231322
for_each_rmap_spte(rmap_head, &iter, sptep) {
@@ -1326,7 +1325,7 @@ static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
13261325

13271326
need_flush = 1;
13281327

1329-
if (pte_write(*ptep)) {
1328+
if (pte_write(pte)) {
13301329
pte_list_remove(rmap_head, sptep);
13311330
goto restart;
13321331
} else {
@@ -1414,86 +1413,52 @@ static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
14141413
slot_rmap_walk_okay(_iter_); \
14151414
slot_rmap_walk_next(_iter_))
14161415

1417-
typedef int (*rmap_handler_t)(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1418-
struct kvm_memory_slot *slot, gfn_t gfn,
1419-
int level, unsigned long data);
1416+
typedef bool (*rmap_handler_t)(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1417+
struct kvm_memory_slot *slot, gfn_t gfn,
1418+
int level, pte_t pte);
14201419

1421-
static __always_inline int kvm_handle_hva_range(struct kvm *kvm,
1422-
unsigned long start,
1423-
unsigned long end,
1424-
unsigned long data,
1425-
rmap_handler_t handler)
1420+
static __always_inline bool kvm_handle_gfn_range(struct kvm *kvm,
1421+
struct kvm_gfn_info *info,
1422+
rmap_handler_t handler)
14261423
{
1427-
struct kvm_memslots *slots;
1428-
struct kvm_memory_slot *memslot;
14291424
struct slot_rmap_walk_iterator iterator;
1430-
int ret = 0;
1431-
int i;
1432-
1433-
for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
1434-
slots = __kvm_memslots(kvm, i);
1435-
kvm_for_each_memslot(memslot, slots) {
1436-
unsigned long hva_start, hva_end;
1437-
gfn_t gfn_start, gfn_end;
1425+
bool ret = false;
14381426

1439-
hva_start = max(start, memslot->userspace_addr);
1440-
hva_end = min(end, memslot->userspace_addr +
1441-
(memslot->npages << PAGE_SHIFT));
1442-
if (hva_start >= hva_end)
1443-
continue;
1444-
/*
1445-
* {gfn(page) | page intersects with [hva_start, hva_end)} =
1446-
* {gfn_start, gfn_start+1, ..., gfn_end-1}.
1447-
*/
1448-
gfn_start = hva_to_gfn_memslot(hva_start, memslot);
1449-
gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
1450-
1451-
for_each_slot_rmap_range(memslot, PG_LEVEL_4K,
1452-
KVM_MAX_HUGEPAGE_LEVEL,
1453-
gfn_start, gfn_end - 1,
1454-
&iterator)
1455-
ret |= handler(kvm, iterator.rmap, memslot,
1456-
iterator.gfn, iterator.level, data);
1457-
}
1458-
}
1427+
for_each_slot_rmap_range(info->slot, PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
1428+
info->start, info->end - 1, &iterator)
1429+
ret |= handler(kvm, iterator.rmap, info->slot, iterator.gfn,
1430+
iterator.level, info->pte);
14591431

14601432
return ret;
14611433
}
14621434

1463-
static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
1464-
unsigned long data, rmap_handler_t handler)
1435+
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_info *info)
14651436
{
1466-
return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1467-
}
1468-
1469-
int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end,
1470-
unsigned flags)
1471-
{
1472-
int r;
1437+
bool flush;
14731438

1474-
r = kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
1439+
flush = kvm_handle_gfn_range(kvm, info, kvm_unmap_rmapp);
14751440

14761441
if (is_tdp_mmu_enabled(kvm))
1477-
r |= kvm_tdp_mmu_zap_hva_range(kvm, start, end);
1442+
flush |= kvm_tdp_mmu_unmap_gfn_range(kvm, info, flush);
14781443

1479-
return r;
1444+
return flush;
14801445
}
14811446

1482-
int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1447+
bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_info *info)
14831448
{
1484-
int r;
1449+
bool flush;
14851450

1486-
r = kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1451+
flush = kvm_handle_gfn_range(kvm, info, kvm_set_pte_rmapp);
14871452

14881453
if (is_tdp_mmu_enabled(kvm))
1489-
r |= kvm_tdp_mmu_set_spte_hva(kvm, hva, &pte);
1454+
flush |= kvm_tdp_mmu_set_spte_gfn(kvm, info);
14901455

1491-
return r;
1456+
return flush;
14921457
}
14931458

1494-
static int kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1495-
struct kvm_memory_slot *slot, gfn_t gfn, int level,
1496-
unsigned long data)
1459+
static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1460+
struct kvm_memory_slot *slot, gfn_t gfn, int level,
1461+
pte_t unused)
14971462
{
14981463
u64 *sptep;
14991464
struct rmap_iterator iter;
@@ -1506,9 +1471,9 @@ static int kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
15061471
return young;
15071472
}
15081473

1509-
static int kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1510-
struct kvm_memory_slot *slot, gfn_t gfn,
1511-
int level, unsigned long data)
1474+
static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1475+
struct kvm_memory_slot *slot, gfn_t gfn,
1476+
int level, pte_t unused)
15121477
{
15131478
u64 *sptep;
15141479
struct rmap_iterator iter;
@@ -1530,29 +1495,31 @@ static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
15301495

15311496
rmap_head = gfn_to_rmap(vcpu->kvm, gfn, sp);
15321497

1533-
kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, 0);
1498+
kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, __pte(0));
15341499
kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn,
15351500
KVM_PAGES_PER_HPAGE(sp->role.level));
15361501
}
15371502

1538-
int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
1503+
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_info *info)
15391504
{
1540-
int young = false;
1505+
bool young;
1506+
1507+
young = kvm_handle_gfn_range(kvm, info, kvm_age_rmapp);
15411508

1542-
young = kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
15431509
if (is_tdp_mmu_enabled(kvm))
1544-
young |= kvm_tdp_mmu_age_hva_range(kvm, start, end);
1510+
young |= kvm_tdp_mmu_age_gfn_range(kvm, info);
15451511

15461512
return young;
15471513
}
15481514

1549-
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
1515+
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_info *info)
15501516
{
1551-
int young = false;
1517+
bool young;
1518+
1519+
young = kvm_handle_gfn_range(kvm, info, kvm_test_age_rmapp);
15521520

1553-
young = kvm_handle_hva(kvm, hva, 0, kvm_test_age_rmapp);
15541521
if (is_tdp_mmu_enabled(kvm))
1555-
young |= kvm_tdp_mmu_test_age_hva(kvm, hva);
1522+
young |= kvm_tdp_mmu_test_age_gfn(kvm, info);
15561523

15571524
return young;
15581525
}

0 commit comments

Comments
 (0)