Skip to content

Commit 5d308fc

Browse files
LuBaolujoergroedel
LuBaolu
authored and
joergroedel
committed
iommu/vt-d: Add 256-bit invalidation descriptor support
Intel vt-d spec rev3.0 requires software to use 256-bit descriptors in invalidation queue. As the spec reads in section 6.5.2: Remapping hardware supporting Scalable Mode Translations (ECAP_REG.SMTS=1) allow software to additionally program the width of the descriptors (128-bits or 256-bits) that will be written into the Queue. Software should setup the Invalidation Queue for 256-bit descriptors before progra- mming remapping hardware for scalable-mode translation as 128-bit descriptors are treated as invalid descriptors (see Table 21 in Section 6.5.2.10) in scalable-mode. This patch adds 256-bit invalidation descriptor support if the hardware presents scalable mode capability. Cc: Ashok Raj <ashok.raj@intel.com> Cc: Jacob Pan <jacob.jun.pan@linux.intel.com> Cc: Kevin Tian <kevin.tian@intel.com> Signed-off-by: Sanjay Kumar <sanjay.k.kumar@intel.com> Signed-off-by: Liu Yi L <yi.l.liu@intel.com> Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com> Signed-off-by: Joerg Roedel <jroedel@suse.de>
1 parent 4f2ed18 commit 5d308fc

File tree

4 files changed

+121
-61
lines changed

4 files changed

+121
-61
lines changed

‎drivers/iommu/dmar.c

Lines changed: 61 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1160,6 +1160,7 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
11601160
int head, tail;
11611161
struct q_inval *qi = iommu->qi;
11621162
int wait_index = (index + 1) % QI_LENGTH;
1163+
int shift = qi_shift(iommu);
11631164

11641165
if (qi->desc_status[wait_index] == QI_ABORT)
11651166
return -EAGAIN;
@@ -1173,13 +1174,19 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
11731174
*/
11741175
if (fault & DMA_FSTS_IQE) {
11751176
head = readl(iommu->reg + DMAR_IQH_REG);
1176-
if ((head >> DMAR_IQ_SHIFT) == index) {
1177-
pr_err("VT-d detected invalid descriptor: "
1178-
"low=%llx, high=%llx\n",
1179-
(unsigned long long)qi->desc[index].low,
1180-
(unsigned long long)qi->desc[index].high);
1181-
memcpy(&qi->desc[index], &qi->desc[wait_index],
1182-
sizeof(struct qi_desc));
1177+
if ((head >> shift) == index) {
1178+
struct qi_desc *desc = qi->desc + head;
1179+
1180+
/*
1181+
* desc->qw2 and desc->qw3 are either reserved or
1182+
* used by software as private data. We won't print
1183+
* out these two qw's for security consideration.
1184+
*/
1185+
pr_err("VT-d detected invalid descriptor: qw0 = %llx, qw1 = %llx\n",
1186+
(unsigned long long)desc->qw0,
1187+
(unsigned long long)desc->qw1);
1188+
memcpy(desc, qi->desc + (wait_index << shift),
1189+
1 << shift);
11831190
writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
11841191
return -EINVAL;
11851192
}
@@ -1191,10 +1198,10 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
11911198
*/
11921199
if (fault & DMA_FSTS_ITE) {
11931200
head = readl(iommu->reg + DMAR_IQH_REG);
1194-
head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1201+
head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
11951202
head |= 1;
11961203
tail = readl(iommu->reg + DMAR_IQT_REG);
1197-
tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1204+
tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
11981205

11991206
writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
12001207

@@ -1222,15 +1229,14 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
12221229
{
12231230
int rc;
12241231
struct q_inval *qi = iommu->qi;
1225-
struct qi_desc *hw, wait_desc;
1232+
int offset, shift, length;
1233+
struct qi_desc wait_desc;
12261234
int wait_index, index;
12271235
unsigned long flags;
12281236

12291237
if (!qi)
12301238
return 0;
12311239

1232-
hw = qi->desc;
1233-
12341240
restart:
12351241
rc = 0;
12361242

@@ -1243,16 +1249,21 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
12431249

12441250
index = qi->free_head;
12451251
wait_index = (index + 1) % QI_LENGTH;
1252+
shift = qi_shift(iommu);
1253+
length = 1 << shift;
12461254

12471255
qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
12481256

1249-
hw[index] = *desc;
1250-
1251-
wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1257+
offset = index << shift;
1258+
memcpy(qi->desc + offset, desc, length);
1259+
wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
12521260
QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1253-
wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1261+
wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
1262+
wait_desc.qw2 = 0;
1263+
wait_desc.qw3 = 0;
12541264

1255-
hw[wait_index] = wait_desc;
1265+
offset = wait_index << shift;
1266+
memcpy(qi->desc + offset, &wait_desc, length);
12561267

12571268
qi->free_head = (qi->free_head + 2) % QI_LENGTH;
12581269
qi->free_cnt -= 2;
@@ -1261,7 +1272,7 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
12611272
* update the HW tail register indicating the presence of
12621273
* new descriptors.
12631274
*/
1264-
writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1275+
writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);
12651276

12661277
while (qi->desc_status[wait_index] != QI_DONE) {
12671278
/*
@@ -1298,8 +1309,10 @@ void qi_global_iec(struct intel_iommu *iommu)
12981309
{
12991310
struct qi_desc desc;
13001311

1301-
desc.low = QI_IEC_TYPE;
1302-
desc.high = 0;
1312+
desc.qw0 = QI_IEC_TYPE;
1313+
desc.qw1 = 0;
1314+
desc.qw2 = 0;
1315+
desc.qw3 = 0;
13031316

13041317
/* should never fail */
13051318
qi_submit_sync(&desc, iommu);
@@ -1310,9 +1323,11 @@ void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
13101323
{
13111324
struct qi_desc desc;
13121325

1313-
desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1326+
desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
13141327
| QI_CC_GRAN(type) | QI_CC_TYPE;
1315-
desc.high = 0;
1328+
desc.qw1 = 0;
1329+
desc.qw2 = 0;
1330+
desc.qw3 = 0;
13161331

13171332
qi_submit_sync(&desc, iommu);
13181333
}
@@ -1331,10 +1346,12 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
13311346
if (cap_read_drain(iommu->cap))
13321347
dr = 1;
13331348

1334-
desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1349+
desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
13351350
| QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1336-
desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1351+
desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
13371352
| QI_IOTLB_AM(size_order);
1353+
desc.qw2 = 0;
1354+
desc.qw3 = 0;
13381355

13391356
qi_submit_sync(&desc, iommu);
13401357
}
@@ -1347,15 +1364,17 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
13471364
if (mask) {
13481365
WARN_ON_ONCE(addr & ((1ULL << (VTD_PAGE_SHIFT + mask)) - 1));
13491366
addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1350-
desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1367+
desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
13511368
} else
1352-
desc.high = QI_DEV_IOTLB_ADDR(addr);
1369+
desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
13531370

13541371
if (qdep >= QI_DEV_IOTLB_MAX_INVS)
13551372
qdep = 0;
13561373

1357-
desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1374+
desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
13581375
QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1376+
desc.qw2 = 0;
1377+
desc.qw3 = 0;
13591378

13601379
qi_submit_sync(&desc, iommu);
13611380
}
@@ -1403,16 +1422,24 @@ static void __dmar_enable_qi(struct intel_iommu *iommu)
14031422
u32 sts;
14041423
unsigned long flags;
14051424
struct q_inval *qi = iommu->qi;
1425+
u64 val = virt_to_phys(qi->desc);
14061426

14071427
qi->free_head = qi->free_tail = 0;
14081428
qi->free_cnt = QI_LENGTH;
14091429

1430+
/*
1431+
* Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
1432+
* is present.
1433+
*/
1434+
if (ecap_smts(iommu->ecap))
1435+
val |= (1 << 11) | 1;
1436+
14101437
raw_spin_lock_irqsave(&iommu->register_lock, flags);
14111438

14121439
/* write zero to the tail reg */
14131440
writel(0, iommu->reg + DMAR_IQT_REG);
14141441

1415-
dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1442+
dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
14161443

14171444
iommu->gcmd |= DMA_GCMD_QIE;
14181445
writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
@@ -1448,8 +1475,12 @@ int dmar_enable_qi(struct intel_iommu *iommu)
14481475

14491476
qi = iommu->qi;
14501477

1451-
1452-
desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1478+
/*
1479+
* Need two pages to accommodate 256 descriptors of 256 bits each
1480+
* if the remapping hardware supports scalable mode translation.
1481+
*/
1482+
desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
1483+
!!ecap_smts(iommu->ecap));
14531484
if (!desc_page) {
14541485
kfree(qi);
14551486
iommu->qi = NULL;

‎drivers/iommu/intel-svm.c

Lines changed: 49 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -161,38 +161,54 @@ static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_d
161161
* because that's the only option the hardware gives us. Despite
162162
* the fact that they are actually only accessible through one. */
163163
if (gl)
164-
desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |
165-
QI_EIOTLB_GRAN(QI_GRAN_ALL_ALL) | QI_EIOTLB_TYPE;
164+
desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
165+
QI_EIOTLB_DID(sdev->did) |
166+
QI_EIOTLB_GRAN(QI_GRAN_ALL_ALL) |
167+
QI_EIOTLB_TYPE;
166168
else
167-
desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |
168-
QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | QI_EIOTLB_TYPE;
169-
desc.high = 0;
169+
desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
170+
QI_EIOTLB_DID(sdev->did) |
171+
QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
172+
QI_EIOTLB_TYPE;
173+
desc.qw1 = 0;
170174
} else {
171175
int mask = ilog2(__roundup_pow_of_two(pages));
172176

173-
desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |
174-
QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | QI_EIOTLB_TYPE;
175-
desc.high = QI_EIOTLB_ADDR(address) | QI_EIOTLB_GL(gl) |
176-
QI_EIOTLB_IH(ih) | QI_EIOTLB_AM(mask);
177+
desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
178+
QI_EIOTLB_DID(sdev->did) |
179+
QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
180+
QI_EIOTLB_TYPE;
181+
desc.qw1 = QI_EIOTLB_ADDR(address) |
182+
QI_EIOTLB_GL(gl) |
183+
QI_EIOTLB_IH(ih) |
184+
QI_EIOTLB_AM(mask);
177185
}
186+
desc.qw2 = 0;
187+
desc.qw3 = 0;
178188
qi_submit_sync(&desc, svm->iommu);
179189

180190
if (sdev->dev_iotlb) {
181-
desc.low = QI_DEV_EIOTLB_PASID(svm->pasid) | QI_DEV_EIOTLB_SID(sdev->sid) |
182-
QI_DEV_EIOTLB_QDEP(sdev->qdep) | QI_DEIOTLB_TYPE;
191+
desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) |
192+
QI_DEV_EIOTLB_SID(sdev->sid) |
193+
QI_DEV_EIOTLB_QDEP(sdev->qdep) |
194+
QI_DEIOTLB_TYPE;
183195
if (pages == -1) {
184-
desc.high = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) | QI_DEV_EIOTLB_SIZE;
196+
desc.qw1 = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) |
197+
QI_DEV_EIOTLB_SIZE;
185198
} else if (pages > 1) {
186199
/* The least significant zero bit indicates the size. So,
187200
* for example, an "address" value of 0x12345f000 will
188201
* flush from 0x123440000 to 0x12347ffff (256KiB). */
189202
unsigned long last = address + ((unsigned long)(pages - 1) << VTD_PAGE_SHIFT);
190203
unsigned long mask = __rounddown_pow_of_two(address ^ last);
191204

192-
desc.high = QI_DEV_EIOTLB_ADDR((address & ~mask) | (mask - 1)) | QI_DEV_EIOTLB_SIZE;
205+
desc.qw1 = QI_DEV_EIOTLB_ADDR((address & ~mask) |
206+
(mask - 1)) | QI_DEV_EIOTLB_SIZE;
193207
} else {
194-
desc.high = QI_DEV_EIOTLB_ADDR(address);
208+
desc.qw1 = QI_DEV_EIOTLB_ADDR(address);
195209
}
210+
desc.qw2 = 0;
211+
desc.qw3 = 0;
196212
qi_submit_sync(&desc, svm->iommu);
197213
}
198214
}
@@ -237,8 +253,11 @@ static void intel_flush_pasid_dev(struct intel_svm *svm, struct intel_svm_dev *s
237253
{
238254
struct qi_desc desc;
239255

240-
desc.high = 0;
241-
desc.low = QI_PC_TYPE | QI_PC_DID(sdev->did) | QI_PC_PASID_SEL | QI_PC_PASID(pasid);
256+
desc.qw0 = QI_PC_TYPE | QI_PC_DID(sdev->did) |
257+
QI_PC_PASID_SEL | QI_PC_PASID(pasid);
258+
desc.qw1 = 0;
259+
desc.qw2 = 0;
260+
desc.qw3 = 0;
242261

243262
qi_submit_sync(&desc, svm->iommu);
244263
}
@@ -667,24 +686,27 @@ static irqreturn_t prq_event_thread(int irq, void *d)
667686
no_pasid:
668687
if (req->lpig) {
669688
/* Page Group Response */
670-
resp.low = QI_PGRP_PASID(req->pasid) |
689+
resp.qw0 = QI_PGRP_PASID(req->pasid) |
671690
QI_PGRP_DID((req->bus << 8) | req->devfn) |
672691
QI_PGRP_PASID_P(req->pasid_present) |
673692
QI_PGRP_RESP_TYPE;
674-
resp.high = QI_PGRP_IDX(req->prg_index) |
675-
QI_PGRP_PRIV(req->private) | QI_PGRP_RESP_CODE(result);
676-
677-
qi_submit_sync(&resp, iommu);
693+
resp.qw1 = QI_PGRP_IDX(req->prg_index) |
694+
QI_PGRP_PRIV(req->private) |
695+
QI_PGRP_RESP_CODE(result);
678696
} else if (req->srr) {
679697
/* Page Stream Response */
680-
resp.low = QI_PSTRM_IDX(req->prg_index) |
681-
QI_PSTRM_PRIV(req->private) | QI_PSTRM_BUS(req->bus) |
682-
QI_PSTRM_PASID(req->pasid) | QI_PSTRM_RESP_TYPE;
683-
resp.high = QI_PSTRM_ADDR(address) | QI_PSTRM_DEVFN(req->devfn) |
698+
resp.qw0 = QI_PSTRM_IDX(req->prg_index) |
699+
QI_PSTRM_PRIV(req->private) |
700+
QI_PSTRM_BUS(req->bus) |
701+
QI_PSTRM_PASID(req->pasid) |
702+
QI_PSTRM_RESP_TYPE;
703+
resp.qw1 = QI_PSTRM_ADDR(address) |
704+
QI_PSTRM_DEVFN(req->devfn) |
684705
QI_PSTRM_RESP_CODE(result);
685-
686-
qi_submit_sync(&resp, iommu);
687706
}
707+
resp.qw2 = 0;
708+
resp.qw3 = 0;
709+
qi_submit_sync(&resp, iommu);
688710

689711
head = (head + sizeof(*req)) & PRQ_RING_MASK;
690712
}

‎drivers/iommu/intel_irq_remapping.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,9 +145,11 @@ static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
145145
{
146146
struct qi_desc desc;
147147

148-
desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
148+
desc.qw0 = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
149149
| QI_IEC_SELECTIVE;
150-
desc.high = 0;
150+
desc.qw1 = 0;
151+
desc.qw2 = 0;
152+
desc.qw3 = 0;
151153

152154
return qi_submit_sync(&desc, iommu);
153155
}

‎include/linux/intel-iommu.h

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -401,13 +401,18 @@ enum {
401401
#define QI_GRAN_NONG_PASID 2
402402
#define QI_GRAN_PSI_PASID 3
403403

404+
#define qi_shift(iommu) (DMAR_IQ_SHIFT + !!ecap_smts((iommu)->ecap))
405+
404406
struct qi_desc {
405-
u64 low, high;
407+
u64 qw0;
408+
u64 qw1;
409+
u64 qw2;
410+
u64 qw3;
406411
};
407412

408413
struct q_inval {
409414
raw_spinlock_t q_lock;
410-
struct qi_desc *desc; /* invalidation queue */
415+
void *desc; /* invalidation queue */
411416
int *desc_status; /* desc status */
412417
int free_head; /* first free entry */
413418
int free_tail; /* last free entry */

0 commit comments

Comments
 (0)