Skip to content

Commit f090a00

Browse files
Changbin Duzhenyw
Changbin Du
authored and
zhenyw
committed
drm/i915/gvt: Add emulation for BAR2 (aperture) with normal file RW approach
For vfio-pci, if the region support MMAP then it should support both mmap and normal file access. The user-space is free to choose which is being used. For qemu, we just need add 'x-no-mmap=on' for vfio-pci option. Currently GVTg only support MMAP for BAR2. So GVTg will not work when user turn on x-no-mmap option. This patch added file style access for BAR2, aka the GPU aperture. We map the entire aperture partition of active vGPU to kernel space when guest driver try to enable PCI Memory Space. Then we redirect the file RW operation from kvmgt to this mapped area. Link: https://bugzilla.redhat.com/show_bug.cgi?id=1458032 Signed-off-by: Changbin Du <changbin.du@intel.com> Signed-off-by: Zhenyu Wang <zhenyuw@linux.intel.com>
1 parent 5d5fe17 commit f090a00

File tree

4 files changed

+98
-25
lines changed

4 files changed

+98
-25
lines changed

‎drivers/gpu/drm/i915/gvt/cfg_space.c

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -110,28 +110,42 @@ int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset,
110110

111111
static int map_aperture(struct intel_vgpu *vgpu, bool map)
112112
{
113-
u64 first_gfn, first_mfn;
113+
phys_addr_t aperture_pa = vgpu_aperture_pa_base(vgpu);
114+
unsigned long aperture_sz = vgpu_aperture_sz(vgpu);
115+
u64 first_gfn;
114116
u64 val;
115117
int ret;
116118

117119
if (map == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked)
118120
return 0;
119121

122+
if (map) {
123+
vgpu->gm.aperture_va = memremap(aperture_pa, aperture_sz,
124+
MEMREMAP_WC);
125+
if (!vgpu->gm.aperture_va)
126+
return -ENOMEM;
127+
} else {
128+
memunmap(vgpu->gm.aperture_va);
129+
vgpu->gm.aperture_va = NULL;
130+
}
131+
120132
val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_2];
121133
if (val & PCI_BASE_ADDRESS_MEM_TYPE_64)
122134
val = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2);
123135
else
124136
val = *(u32 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2);
125137

126138
first_gfn = (val + vgpu_aperture_offset(vgpu)) >> PAGE_SHIFT;
127-
first_mfn = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT;
128139

129140
ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, first_gfn,
130-
first_mfn,
131-
vgpu_aperture_sz(vgpu) >>
132-
PAGE_SHIFT, map);
133-
if (ret)
141+
aperture_pa >> PAGE_SHIFT,
142+
aperture_sz >> PAGE_SHIFT,
143+
map);
144+
if (ret) {
145+
memunmap(vgpu->gm.aperture_va);
146+
vgpu->gm.aperture_va = NULL;
134147
return ret;
148+
}
135149

136150
vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked = map;
137151
return 0;

‎drivers/gpu/drm/i915/gvt/gvt.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ struct intel_gvt_device_info {
8080
struct intel_vgpu_gm {
8181
u64 aperture_sz;
8282
u64 hidden_sz;
83+
void *aperture_va;
8384
struct drm_mm_node low_gm_node;
8485
struct drm_mm_node high_gm_node;
8586
};
@@ -474,6 +475,13 @@ int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset,
474475
int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
475476
void *p_data, unsigned int bytes);
476477

478+
static inline u64 intel_vgpu_get_bar_gpa(struct intel_vgpu *vgpu, int bar)
479+
{
480+
/* We are 64bit bar. */
481+
return (*(u64 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
482+
PCI_BASE_ADDRESS_MEM_MASK;
483+
}
484+
477485
void intel_gvt_clean_opregion(struct intel_gvt *gvt);
478486
int intel_gvt_init_opregion(struct intel_gvt *gvt);
479487

‎drivers/gpu/drm/i915/gvt/kvmgt.c

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -609,21 +609,20 @@ static void intel_vgpu_release_work(struct work_struct *work)
609609
__intel_vgpu_release(vgpu);
610610
}
611611

612-
static uint64_t intel_vgpu_get_bar0_addr(struct intel_vgpu *vgpu)
612+
static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
613613
{
614614
u32 start_lo, start_hi;
615615
u32 mem_type;
616-
int pos = PCI_BASE_ADDRESS_0;
617616

618-
start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + pos)) &
617+
start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
619618
PCI_BASE_ADDRESS_MEM_MASK;
620-
mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + pos)) &
619+
mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
621620
PCI_BASE_ADDRESS_MEM_TYPE_MASK;
622621

623622
switch (mem_type) {
624623
case PCI_BASE_ADDRESS_MEM_TYPE_64:
625624
start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
626-
+ pos + 4));
625+
+ bar + 4));
627626
break;
628627
case PCI_BASE_ADDRESS_MEM_TYPE_32:
629628
case PCI_BASE_ADDRESS_MEM_TYPE_1M:
@@ -637,6 +636,21 @@ static uint64_t intel_vgpu_get_bar0_addr(struct intel_vgpu *vgpu)
637636
return ((u64)start_hi << 32) | start_lo;
638637
}
639638

639+
static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off,
640+
void *buf, unsigned int count, bool is_write)
641+
{
642+
uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
643+
int ret;
644+
645+
if (is_write)
646+
ret = intel_gvt_ops->emulate_mmio_write(vgpu,
647+
bar_start + off, buf, count);
648+
else
649+
ret = intel_gvt_ops->emulate_mmio_read(vgpu,
650+
bar_start + off, buf, count);
651+
return ret;
652+
}
653+
640654
static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
641655
size_t count, loff_t *ppos, bool is_write)
642656
{
@@ -661,20 +675,14 @@ static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
661675
buf, count);
662676
break;
663677
case VFIO_PCI_BAR0_REGION_INDEX:
664-
if (is_write) {
665-
uint64_t bar0_start = intel_vgpu_get_bar0_addr(vgpu);
666-
667-
ret = intel_gvt_ops->emulate_mmio_write(vgpu,
668-
bar0_start + pos, buf, count);
669-
} else {
670-
uint64_t bar0_start = intel_vgpu_get_bar0_addr(vgpu);
671-
672-
ret = intel_gvt_ops->emulate_mmio_read(vgpu,
673-
bar0_start + pos, buf, count);
674-
}
678+
ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,
679+
buf, count, is_write);
675680
break;
676-
case VFIO_PCI_BAR1_REGION_INDEX:
677681
case VFIO_PCI_BAR2_REGION_INDEX:
682+
ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_2, pos,
683+
buf, count, is_write);
684+
break;
685+
case VFIO_PCI_BAR1_REGION_INDEX:
678686
case VFIO_PCI_BAR3_REGION_INDEX:
679687
case VFIO_PCI_BAR4_REGION_INDEX:
680688
case VFIO_PCI_BAR5_REGION_INDEX:

‎drivers/gpu/drm/i915/gvt/mmio.c

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@
4545
*/
4646
int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa)
4747
{
48-
u64 gttmmio_gpa = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0) &
49-
~GENMASK(3, 0);
48+
u64 gttmmio_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
5049
return gpa - gttmmio_gpa;
5150
}
5251

@@ -57,6 +56,38 @@ int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa)
5756
(reg >= gvt->device_info.gtt_start_offset \
5857
&& reg < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt))
5958

59+
static bool vgpu_gpa_is_aperture(struct intel_vgpu *vgpu, uint64_t gpa)
60+
{
61+
u64 aperture_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_2);
62+
u64 aperture_sz = vgpu_aperture_sz(vgpu);
63+
64+
return gpa >= aperture_gpa && gpa < aperture_gpa + aperture_sz;
65+
}
66+
67+
static int vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t gpa,
68+
void *pdata, unsigned int size, bool is_read)
69+
{
70+
u64 aperture_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_2);
71+
u64 offset = gpa - aperture_gpa;
72+
73+
if (!vgpu_gpa_is_aperture(vgpu, gpa + size - 1)) {
74+
gvt_vgpu_err("Aperture rw out of range, offset %llx, size %d\n",
75+
offset, size);
76+
return -EINVAL;
77+
}
78+
79+
if (!vgpu->gm.aperture_va) {
80+
gvt_vgpu_err("BAR is not enabled\n");
81+
return -ENXIO;
82+
}
83+
84+
if (is_read)
85+
memcpy(pdata, vgpu->gm.aperture_va + offset, size);
86+
else
87+
memcpy(vgpu->gm.aperture_va + offset, pdata, size);
88+
return 0;
89+
}
90+
6091
static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa,
6192
void *p_data, unsigned int bytes, bool read)
6293
{
@@ -133,6 +164,12 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa,
133164
}
134165
mutex_lock(&gvt->lock);
135166

167+
if (vgpu_gpa_is_aperture(vgpu, pa)) {
168+
ret = vgpu_aperture_rw(vgpu, pa, p_data, bytes, true);
169+
mutex_unlock(&gvt->lock);
170+
return ret;
171+
}
172+
136173
if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
137174
struct intel_vgpu_guest_page *gp;
138175

@@ -224,6 +261,12 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa,
224261

225262
mutex_lock(&gvt->lock);
226263

264+
if (vgpu_gpa_is_aperture(vgpu, pa)) {
265+
ret = vgpu_aperture_rw(vgpu, pa, p_data, bytes, false);
266+
mutex_unlock(&gvt->lock);
267+
return ret;
268+
}
269+
227270
if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
228271
struct intel_vgpu_guest_page *gp;
229272

0 commit comments

Comments
 (0)