Skip to content

Commit b5baa68

Browse files
dgibsonpaulusmack
dgibson
authored and
paulusmack
committed
KVM: PPC: Book3S HV: KVM-HV HPT resizing implementation
This adds the "guts" of the implementation for the HPT resizing PAPR extension. It has the code to allocate and clear a new HPT, rehash an existing HPT's entries into it, and accomplish the switchover for a KVM guest from the old HPT to the new one. Signed-off-by: David Gibson <david@gibson.dropbear.id.au> Signed-off-by: Paul Mackerras <paulus@ozlabs.org>
1 parent 5e98596 commit b5baa68

File tree

1 file changed

+187
-1
lines changed

1 file changed

+187
-1
lines changed

‎arch/powerpc/kvm/book3s_64_mmu_hv.c

Lines changed: 187 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ struct kvm_resize_hpt {
6666
/* These fields protected by kvm->lock */
6767
int error;
6868
bool prepare_done;
69+
70+
/* Private to the work thread, until prepare_done is true,
71+
* then protected by kvm->resize_hpt_sem */
72+
struct kvm_hpt_info hpt;
6973
};
7074

7175
static void kvmppc_rmap_reset(struct kvm *kvm);
@@ -1209,21 +1213,203 @@ void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,
12091213
*/
12101214
static int resize_hpt_allocate(struct kvm_resize_hpt *resize)
12111215
{
1216+
int rc;
1217+
1218+
rc = kvmppc_allocate_hpt(&resize->hpt, resize->order);
1219+
if (rc < 0)
1220+
return rc;
1221+
1222+
resize_hpt_debug(resize, "resize_hpt_allocate(): HPT @ 0x%lx\n",
1223+
resize->hpt.virt);
1224+
12121225
return 0;
12131226
}
12141227

1228+
static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
1229+
unsigned long idx)
1230+
{
1231+
struct kvm *kvm = resize->kvm;
1232+
struct kvm_hpt_info *old = &kvm->arch.hpt;
1233+
struct kvm_hpt_info *new = &resize->hpt;
1234+
unsigned long old_hash_mask = (1ULL << (old->order - 7)) - 1;
1235+
unsigned long new_hash_mask = (1ULL << (new->order - 7)) - 1;
1236+
__be64 *hptep, *new_hptep;
1237+
unsigned long vpte, rpte, guest_rpte;
1238+
int ret;
1239+
struct revmap_entry *rev;
1240+
unsigned long apsize, psize, avpn, pteg, hash;
1241+
unsigned long new_idx, new_pteg, replace_vpte;
1242+
1243+
hptep = (__be64 *)(old->virt + (idx << 4));
1244+
1245+
/* Guest is stopped, so new HPTEs can't be added or faulted
1246+
* in, only unmapped or altered by host actions. So, it's
1247+
* safe to check this before we take the HPTE lock */
1248+
vpte = be64_to_cpu(hptep[0]);
1249+
if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT))
1250+
return 0; /* nothing to do */
1251+
1252+
while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
1253+
cpu_relax();
1254+
1255+
vpte = be64_to_cpu(hptep[0]);
1256+
1257+
ret = 0;
1258+
if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT))
1259+
/* Nothing to do */
1260+
goto out;
1261+
1262+
/* Unmap */
1263+
rev = &old->rev[idx];
1264+
guest_rpte = rev->guest_rpte;
1265+
1266+
ret = -EIO;
1267+
apsize = hpte_page_size(vpte, guest_rpte);
1268+
if (!apsize)
1269+
goto out;
1270+
1271+
if (vpte & HPTE_V_VALID) {
1272+
unsigned long gfn = hpte_rpn(guest_rpte, apsize);
1273+
int srcu_idx = srcu_read_lock(&kvm->srcu);
1274+
struct kvm_memory_slot *memslot =
1275+
__gfn_to_memslot(kvm_memslots(kvm), gfn);
1276+
1277+
if (memslot) {
1278+
unsigned long *rmapp;
1279+
rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
1280+
1281+
lock_rmap(rmapp);
1282+
kvmppc_unmap_hpte(kvm, idx, rmapp, gfn);
1283+
unlock_rmap(rmapp);
1284+
}
1285+
1286+
srcu_read_unlock(&kvm->srcu, srcu_idx);
1287+
}
1288+
1289+
/* Reload PTE after unmap */
1290+
vpte = be64_to_cpu(hptep[0]);
1291+
1292+
BUG_ON(vpte & HPTE_V_VALID);
1293+
BUG_ON(!(vpte & HPTE_V_ABSENT));
1294+
1295+
ret = 0;
1296+
if (!(vpte & HPTE_V_BOLTED))
1297+
goto out;
1298+
1299+
rpte = be64_to_cpu(hptep[1]);
1300+
psize = hpte_base_page_size(vpte, rpte);
1301+
avpn = HPTE_V_AVPN_VAL(vpte) & ~((psize - 1) >> 23);
1302+
pteg = idx / HPTES_PER_GROUP;
1303+
if (vpte & HPTE_V_SECONDARY)
1304+
pteg = ~pteg;
1305+
1306+
if (!(vpte & HPTE_V_1TB_SEG)) {
1307+
unsigned long offset, vsid;
1308+
1309+
/* We only have 28 - 23 bits of offset in avpn */
1310+
offset = (avpn & 0x1f) << 23;
1311+
vsid = avpn >> 5;
1312+
/* We can find more bits from the pteg value */
1313+
if (psize < (1ULL << 23))
1314+
offset |= ((vsid ^ pteg) & old_hash_mask) * psize;
1315+
1316+
hash = vsid ^ (offset / psize);
1317+
} else {
1318+
unsigned long offset, vsid;
1319+
1320+
/* We only have 40 - 23 bits of seg_off in avpn */
1321+
offset = (avpn & 0x1ffff) << 23;
1322+
vsid = avpn >> 17;
1323+
if (psize < (1ULL << 23))
1324+
offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask) * psize;
1325+
1326+
hash = vsid ^ (vsid << 25) ^ (offset / psize);
1327+
}
1328+
1329+
new_pteg = hash & new_hash_mask;
1330+
if (vpte & HPTE_V_SECONDARY) {
1331+
BUG_ON(~pteg != (hash & old_hash_mask));
1332+
new_pteg = ~new_pteg;
1333+
} else {
1334+
BUG_ON(pteg != (hash & old_hash_mask));
1335+
}
1336+
1337+
new_idx = new_pteg * HPTES_PER_GROUP + (idx % HPTES_PER_GROUP);
1338+
new_hptep = (__be64 *)(new->virt + (new_idx << 4));
1339+
1340+
replace_vpte = be64_to_cpu(new_hptep[0]);
1341+
1342+
if (replace_vpte & (HPTE_V_VALID | HPTE_V_ABSENT)) {
1343+
BUG_ON(new->order >= old->order);
1344+
1345+
if (replace_vpte & HPTE_V_BOLTED) {
1346+
if (vpte & HPTE_V_BOLTED)
1347+
/* Bolted collision, nothing we can do */
1348+
ret = -ENOSPC;
1349+
/* Discard the new HPTE */
1350+
goto out;
1351+
}
1352+
1353+
/* Discard the previous HPTE */
1354+
}
1355+
1356+
new_hptep[1] = cpu_to_be64(rpte);
1357+
new->rev[new_idx].guest_rpte = guest_rpte;
1358+
/* No need for a barrier, since new HPT isn't active */
1359+
new_hptep[0] = cpu_to_be64(vpte);
1360+
unlock_hpte(new_hptep, vpte);
1361+
1362+
out:
1363+
unlock_hpte(hptep, vpte);
1364+
return ret;
1365+
}
1366+
12151367
static int resize_hpt_rehash(struct kvm_resize_hpt *resize)
12161368
{
1217-
return -EIO;
1369+
struct kvm *kvm = resize->kvm;
1370+
unsigned long i;
1371+
int rc;
1372+
1373+
for (i = 0; i < kvmppc_hpt_npte(&kvm->arch.hpt); i++) {
1374+
rc = resize_hpt_rehash_hpte(resize, i);
1375+
if (rc != 0)
1376+
return rc;
1377+
}
1378+
1379+
return 0;
12181380
}
12191381

12201382
static void resize_hpt_pivot(struct kvm_resize_hpt *resize)
12211383
{
1384+
struct kvm *kvm = resize->kvm;
1385+
struct kvm_hpt_info hpt_tmp;
1386+
1387+
/* Exchange the pending tables in the resize structure with
1388+
* the active tables */
1389+
1390+
resize_hpt_debug(resize, "resize_hpt_pivot()\n");
1391+
1392+
spin_lock(&kvm->mmu_lock);
1393+
asm volatile("ptesync" : : : "memory");
1394+
1395+
hpt_tmp = kvm->arch.hpt;
1396+
kvmppc_set_hpt(kvm, &resize->hpt);
1397+
resize->hpt = hpt_tmp;
1398+
1399+
spin_unlock(&kvm->mmu_lock);
1400+
1401+
synchronize_srcu_expedited(&kvm->srcu);
1402+
1403+
resize_hpt_debug(resize, "resize_hpt_pivot() done\n");
12221404
}
12231405

12241406
static void resize_hpt_release(struct kvm *kvm, struct kvm_resize_hpt *resize)
12251407
{
12261408
BUG_ON(kvm->arch.resize_hpt != resize);
1409+
1410+
if (resize->hpt.virt)
1411+
kvmppc_free_hpt(&resize->hpt);
1412+
12271413
kvm->arch.resize_hpt = NULL;
12281414
kfree(resize);
12291415
}

0 commit comments

Comments
 (0)