Skip to content

Commit 0bbeb01

Browse files
LuBaolujoergroedel
LuBaolu
authored and
joergroedel
committed
iommu/vt-d: Manage scalalble mode PASID tables
In scalable mode, pasid structure is a two level table with a pasid directory table and a pasid table. Any pasid entry can be identified by a pasid value in below way. 1 9 6 5 0 .-----------------------.-------. | PASID | | '-----------------------'-------' .-------------. | | | | | | | | | | | | | .-----------. | .-------------. | | | |----->| PASID Entry | | | | | '-------------' | | | |Plus | | | .-----------. | | | |---->| DIR Entry |-------->| | | '-----------' '-------------' .---------. |Plus | | | Context | | | | | Entry |------->| | '---------' '-----------' This changes the pasid table APIs to support scalable mode PASID directory and PASID table. It also adds a helper to get the PASID table entry according to the pasid value. Cc: Ashok Raj <ashok.raj@intel.com> Cc: Jacob Pan <jacob.jun.pan@linux.intel.com> Cc: Kevin Tian <kevin.tian@intel.com> Signed-off-by: Sanjay Kumar <sanjay.k.kumar@intel.com> Signed-off-by: Liu Yi L <yi.l.liu@intel.com> Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com> Reviewed-by: Ashok Raj <ashok.raj@intel.com> Signed-off-by: Joerg Roedel <jroedel@suse.de>
1 parent 765b6a9 commit 0bbeb01

File tree

4 files changed

+97
-31
lines changed

4 files changed

+97
-31
lines changed

‎drivers/iommu/intel-iommu.c

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -425,21 +425,24 @@ static LIST_HEAD(device_domain_list);
425425

426426
/*
427427
* Iterate over elements in device_domain_list and call the specified
428-
* callback @fn against each element. This helper should only be used
429-
* in the context where the device_domain_lock has already been holden.
428+
* callback @fn against each element.
430429
*/
431430
int for_each_device_domain(int (*fn)(struct device_domain_info *info,
432431
void *data), void *data)
433432
{
434433
int ret = 0;
434+
unsigned long flags;
435435
struct device_domain_info *info;
436436

437-
assert_spin_locked(&device_domain_lock);
437+
spin_lock_irqsave(&device_domain_lock, flags);
438438
list_for_each_entry(info, &device_domain_list, global) {
439439
ret = fn(info, data);
440-
if (ret)
440+
if (ret) {
441+
spin_unlock_irqrestore(&device_domain_lock, flags);
441442
return ret;
443+
}
442444
}
445+
spin_unlock_irqrestore(&device_domain_lock, flags);
443446

444447
return 0;
445448
}
@@ -2481,16 +2484,18 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
24812484
list_add(&info->global, &device_domain_list);
24822485
if (dev)
24832486
dev->archdata.iommu = info;
2487+
spin_unlock_irqrestore(&device_domain_lock, flags);
24842488

2485-
if (dev && dev_is_pci(dev) && info->pasid_supported) {
2489+
/* PASID table is mandatory for a PCI device in scalable mode. */
2490+
if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
24862491
ret = intel_pasid_alloc_table(dev);
24872492
if (ret) {
2488-
pr_warn("No pasid table for %s, pasid disabled\n",
2489-
dev_name(dev));
2490-
info->pasid_supported = 0;
2493+
pr_err("PASID table allocation for %s failed\n",
2494+
dev_name(dev));
2495+
dmar_remove_one_dev_info(domain, dev);
2496+
return NULL;
24912497
}
24922498
}
2493-
spin_unlock_irqrestore(&device_domain_lock, flags);
24942499

24952500
if (dev && domain_context_mapping(domain, dev)) {
24962501
pr_err("Domain context map for %s failed\n", dev_name(dev));

‎drivers/iommu/intel-pasid.c

Lines changed: 72 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -123,12 +123,13 @@ int intel_pasid_alloc_table(struct device *dev)
123123
struct pasid_table *pasid_table;
124124
struct pasid_table_opaque data;
125125
struct page *pages;
126-
size_t size, count;
126+
int max_pasid = 0;
127127
int ret, order;
128+
int size;
128129

130+
might_sleep();
129131
info = dev->archdata.iommu;
130-
if (WARN_ON(!info || !dev_is_pci(dev) ||
131-
!info->pasid_supported || info->pasid_table))
132+
if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table))
132133
return -EINVAL;
133134

134135
/* DMA alias device already has a pasid table, use it: */
@@ -138,38 +139,59 @@ int intel_pasid_alloc_table(struct device *dev)
138139
if (ret)
139140
goto attach_out;
140141

141-
pasid_table = kzalloc(sizeof(*pasid_table), GFP_ATOMIC);
142+
pasid_table = kzalloc(sizeof(*pasid_table), GFP_KERNEL);
142143
if (!pasid_table)
143144
return -ENOMEM;
144145
INIT_LIST_HEAD(&pasid_table->dev);
145146

146-
size = sizeof(struct pasid_entry);
147-
count = min_t(int, pci_max_pasids(to_pci_dev(dev)), intel_pasid_max_id);
148-
order = get_order(size * count);
147+
if (info->pasid_supported)
148+
max_pasid = min_t(int, pci_max_pasids(to_pci_dev(dev)),
149+
intel_pasid_max_id);
150+
151+
size = max_pasid >> (PASID_PDE_SHIFT - 3);
152+
order = size ? get_order(size) : 0;
149153
pages = alloc_pages_node(info->iommu->node,
150-
GFP_ATOMIC | __GFP_ZERO,
151-
order);
154+
GFP_KERNEL | __GFP_ZERO, order);
152155
if (!pages)
153156
return -ENOMEM;
154157

155158
pasid_table->table = page_address(pages);
156159
pasid_table->order = order;
157-
pasid_table->max_pasid = count;
160+
pasid_table->max_pasid = 1 << (order + PAGE_SHIFT + 3);
158161

159162
attach_out:
160163
device_attach_pasid_table(info, pasid_table);
161164

162165
return 0;
163166
}
164167

168+
/* Get PRESENT bit of a PASID directory entry. */
169+
static inline bool
170+
pasid_pde_is_present(struct pasid_dir_entry *pde)
171+
{
172+
return READ_ONCE(pde->val) & PASID_PTE_PRESENT;
173+
}
174+
175+
/* Get PASID table from a PASID directory entry. */
176+
static inline struct pasid_entry *
177+
get_pasid_table_from_pde(struct pasid_dir_entry *pde)
178+
{
179+
if (!pasid_pde_is_present(pde))
180+
return NULL;
181+
182+
return phys_to_virt(READ_ONCE(pde->val) & PDE_PFN_MASK);
183+
}
184+
165185
void intel_pasid_free_table(struct device *dev)
166186
{
167187
struct device_domain_info *info;
168188
struct pasid_table *pasid_table;
189+
struct pasid_dir_entry *dir;
190+
struct pasid_entry *table;
191+
int i, max_pde;
169192

170193
info = dev->archdata.iommu;
171-
if (!info || !dev_is_pci(dev) ||
172-
!info->pasid_supported || !info->pasid_table)
194+
if (!info || !dev_is_pci(dev) || !info->pasid_table)
173195
return;
174196

175197
pasid_table = info->pasid_table;
@@ -178,6 +200,14 @@ void intel_pasid_free_table(struct device *dev)
178200
if (!list_empty(&pasid_table->dev))
179201
return;
180202

203+
/* Free scalable mode PASID directory tables: */
204+
dir = pasid_table->table;
205+
max_pde = pasid_table->max_pasid >> PASID_PDE_SHIFT;
206+
for (i = 0; i < max_pde; i++) {
207+
table = get_pasid_table_from_pde(&dir[i]);
208+
free_pgtable_page(table);
209+
}
210+
181211
free_pages((unsigned long)pasid_table->table, pasid_table->order);
182212
kfree(pasid_table);
183213
}
@@ -206,25 +236,52 @@ int intel_pasid_get_dev_max_id(struct device *dev)
206236

207237
struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid)
208238
{
239+
struct device_domain_info *info;
209240
struct pasid_table *pasid_table;
241+
struct pasid_dir_entry *dir;
210242
struct pasid_entry *entries;
243+
int dir_index, index;
211244

212245
pasid_table = intel_pasid_get_table(dev);
213246
if (WARN_ON(!pasid_table || pasid < 0 ||
214247
pasid >= intel_pasid_get_dev_max_id(dev)))
215248
return NULL;
216249

217-
entries = pasid_table->table;
250+
dir = pasid_table->table;
251+
info = dev->archdata.iommu;
252+
dir_index = pasid >> PASID_PDE_SHIFT;
253+
index = pasid & PASID_PTE_MASK;
254+
255+
spin_lock(&pasid_lock);
256+
entries = get_pasid_table_from_pde(&dir[dir_index]);
257+
if (!entries) {
258+
entries = alloc_pgtable_page(info->iommu->node);
259+
if (!entries) {
260+
spin_unlock(&pasid_lock);
261+
return NULL;
262+
}
263+
264+
WRITE_ONCE(dir[dir_index].val,
265+
(u64)virt_to_phys(entries) | PASID_PTE_PRESENT);
266+
}
267+
spin_unlock(&pasid_lock);
218268

219-
return &entries[pasid];
269+
return &entries[index];
220270
}
221271

222272
/*
223273
* Interfaces for PASID table entry manipulation:
224274
*/
225275
static inline void pasid_clear_entry(struct pasid_entry *pe)
226276
{
227-
WRITE_ONCE(pe->val, 0);
277+
WRITE_ONCE(pe->val[0], 0);
278+
WRITE_ONCE(pe->val[1], 0);
279+
WRITE_ONCE(pe->val[2], 0);
280+
WRITE_ONCE(pe->val[3], 0);
281+
WRITE_ONCE(pe->val[4], 0);
282+
WRITE_ONCE(pe->val[5], 0);
283+
WRITE_ONCE(pe->val[6], 0);
284+
WRITE_ONCE(pe->val[7], 0);
228285
}
229286

230287
void intel_pasid_clear_entry(struct device *dev, int pasid)

‎drivers/iommu/intel-pasid.h

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,20 @@
1111
#define __INTEL_PASID_H
1212

1313
#define PASID_MIN 0x1
14-
#define PASID_MAX 0x20000
14+
#define PASID_MAX 0x100000
15+
#define PASID_PTE_MASK 0x3F
16+
#define PASID_PTE_PRESENT 1
17+
#define PDE_PFN_MASK PAGE_MASK
18+
#define PASID_PDE_SHIFT 6
1519

16-
struct pasid_entry {
20+
struct pasid_dir_entry {
1721
u64 val;
1822
};
1923

24+
struct pasid_entry {
25+
u64 val[8];
26+
};
27+
2028
/* The representative of a PASID table */
2129
struct pasid_table {
2230
void *table; /* pasid table pointer */

‎drivers/iommu/intel-svm.c

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,6 @@ int intel_svm_init(struct intel_iommu *iommu)
6565

6666
order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max);
6767
if (ecap_dis(iommu->ecap)) {
68-
/* Just making it explicit... */
69-
BUILD_BUG_ON(sizeof(struct pasid_entry) != sizeof(struct pasid_state_entry));
7068
pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order);
7169
if (pages)
7270
iommu->pasid_state_table = page_address(pages);
@@ -405,9 +403,7 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
405403
pasid_entry_val |= PASID_ENTRY_FLPM_5LP;
406404

407405
entry = intel_pasid_get_entry(dev, svm->pasid);
408-
entry->val = pasid_entry_val;
409-
410-
wmb();
406+
WRITE_ONCE(entry->val[0], pasid_entry_val);
411407

412408
/*
413409
* Flush PASID cache when a PASID table entry becomes

0 commit comments

Comments
 (0)