Skip to content

Commit bc3ec75

Browse files
author
Christoph Hellwig
committed
dma-mapping: merge direct and noncoherent ops
All the cache maintainance is already stubbed out when not enabled, but merging the two allows us to nicely handle the case where cache maintainance is required for some devices, but not others. Signed-off-by: Christoph Hellwig <hch@lst.de> Acked-by: Paul Burton <paul.burton@mips.com> # MIPS parts
1 parent f3ecc0f commit bc3ec75

File tree

29 files changed

+160
-192
lines changed

29 files changed

+160
-192
lines changed

‎arch/arc/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ config ARC
1717
select BUILDTIME_EXTABLE_SORT
1818
select CLONE_BACKWARDS
1919
select COMMON_CLK
20-
select DMA_NONCOHERENT_OPS
20+
select DMA_DIRECT_OPS
2121
select DMA_NONCOHERENT_MMAP
2222
select GENERIC_ATOMIC64 if !ISA_ARCV2 || !(ARC_HAS_LL64 && ARC_HAS_LLSC)
2323
select GENERIC_CLOCKEVENTS

‎arch/arc/mm/dma.c

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -167,21 +167,19 @@ void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
167167
}
168168

169169
/*
170-
* Plug in coherent or noncoherent dma ops
170+
* Plug in direct dma map ops.
171171
*/
172172
void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
173173
const struct iommu_ops *iommu, bool coherent)
174174
{
175175
/*
176176
* IOC hardware snoops all DMA traffic keeping the caches consistent
177177
* with memory - eliding need for any explicit cache maintenance of
178-
* DMA buffers - so we can use dma_direct cache ops.
178+
* DMA buffers.
179179
*/
180-
if (is_isa_arcv2() && ioc_enable && coherent) {
181-
set_dma_ops(dev, &dma_direct_ops);
182-
dev_info(dev, "use dma_direct_ops cache ops\n");
183-
} else {
184-
set_dma_ops(dev, &dma_noncoherent_ops);
185-
dev_info(dev, "use dma_noncoherent_ops cache ops\n");
186-
}
180+
if (is_isa_arcv2() && ioc_enable && coherent)
181+
dev->dma_coherent = true;
182+
183+
dev_info(dev, "use %sncoherent DMA ops\n",
184+
dev->dma_coherent ? "" : "non");
187185
}

‎arch/arm/mm/dma-mapping-nommu.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size,
4747
*/
4848

4949
if (attrs & DMA_ATTR_NON_CONSISTENT)
50-
return dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
50+
return dma_direct_alloc_pages(dev, size, dma_handle, gfp,
51+
attrs);
5152

5253
ret = dma_alloc_from_global_coherent(size, dma_handle);
5354

@@ -70,7 +71,7 @@ static void arm_nommu_dma_free(struct device *dev, size_t size,
7071
unsigned long attrs)
7172
{
7273
if (attrs & DMA_ATTR_NON_CONSISTENT) {
73-
dma_direct_free(dev, size, cpu_addr, dma_addr, attrs);
74+
dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
7475
} else {
7576
int ret = dma_release_from_global_coherent(get_order(size),
7677
cpu_addr);

‎arch/c6x/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ config C6X
99
select ARCH_HAS_SYNC_DMA_FOR_CPU
1010
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
1111
select CLKDEV_LOOKUP
12-
select DMA_NONCOHERENT_OPS
12+
select DMA_DIRECT_OPS
1313
select GENERIC_ATOMIC64
1414
select GENERIC_IRQ_SHOW
1515
select HAVE_ARCH_TRACEHOOK

‎arch/hexagon/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ config HEXAGON
3030
select GENERIC_CLOCKEVENTS_BROADCAST
3131
select MODULES_USE_ELF_RELA
3232
select GENERIC_CPU_DEVICES
33-
select DMA_NONCOHERENT_OPS
33+
select DMA_DIRECT_OPS
3434
---help---
3535
Qualcomm Hexagon is a processor architecture designed for high
3636
performance and low power across a wide variety of applications.

‎arch/m68k/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ config M68K
2626
select MODULES_USE_ELF_RELA
2727
select OLD_SIGSUSPEND3
2828
select OLD_SIGACTION
29-
select DMA_NONCOHERENT_OPS if HAS_DMA
29+
select DMA_DIRECT_OPS if HAS_DMA
3030
select HAVE_MEMBLOCK
3131
select ARCH_DISCARD_MEMBLOCK
3232
select NO_BOOTMEM

‎arch/microblaze/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ config MICROBLAZE
1111
select TIMER_OF
1212
select CLONE_BACKWARDS3
1313
select COMMON_CLK
14-
select DMA_NONCOHERENT_OPS
14+
select DMA_DIRECT_OPS
1515
select DMA_NONCOHERENT_MMAP
1616
select GENERIC_ATOMIC64
1717
select GENERIC_CLOCKEVENTS

‎arch/mips/Kconfig

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1121,7 +1121,6 @@ config DMA_NONCOHERENT
11211121
select NEED_DMA_MAP_STATE
11221122
select DMA_NONCOHERENT_MMAP
11231123
select DMA_NONCOHERENT_CACHE_SYNC
1124-
select DMA_NONCOHERENT_OPS
11251124

11261125
config SYS_HAS_EARLY_PRINTK
11271126
bool

‎arch/mips/include/asm/dma-mapping.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
1212
return &jazz_dma_ops;
1313
#elif defined(CONFIG_SWIOTLB)
1414
return &swiotlb_dma_ops;
15-
#elif defined(CONFIG_DMA_NONCOHERENT_OPS)
16-
return &dma_noncoherent_ops;
1715
#else
1816
return &dma_direct_ops;
1917
#endif

‎arch/mips/jazz/jazzdma.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -564,13 +564,13 @@ static void *jazz_dma_alloc(struct device *dev, size_t size,
564564
{
565565
void *ret;
566566

567-
ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
567+
ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
568568
if (!ret)
569569
return NULL;
570570

571571
*dma_handle = vdma_alloc(virt_to_phys(ret), size);
572572
if (*dma_handle == VDMA_ERROR) {
573-
dma_direct_free(dev, size, ret, *dma_handle, attrs);
573+
dma_direct_free_pages(dev, size, ret, *dma_handle, attrs);
574574
return NULL;
575575
}
576576

@@ -587,7 +587,7 @@ static void jazz_dma_free(struct device *dev, size_t size, void *vaddr,
587587
vdma_free(dma_handle);
588588
if (!(attrs & DMA_ATTR_NON_CONSISTENT))
589589
vaddr = (void *)CAC_ADDR((unsigned long)vaddr);
590-
return dma_direct_free(dev, size, vaddr, dma_handle, attrs);
590+
dma_direct_free_pages(dev, size, vaddr, dma_handle, attrs);
591591
}
592592

593593
static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page,

‎arch/mips/mm/dma-noncoherent.c

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,6 @@
2929
*/
3030
static inline bool cpu_needs_post_dma_flush(struct device *dev)
3131
{
32-
if (dev_is_dma_coherent(dev))
33-
return false;
34-
3532
switch (boot_cpu_type()) {
3633
case CPU_R10000:
3734
case CPU_R12000:
@@ -52,11 +49,8 @@ void *arch_dma_alloc(struct device *dev, size_t size,
5249
{
5350
void *ret;
5451

55-
ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
56-
if (!ret)
57-
return NULL;
58-
59-
if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
52+
ret = dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs);
53+
if (!ret && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
6054
dma_cache_wback_inv((unsigned long) ret, size);
6155
ret = (void *)UNCAC_ADDR(ret);
6256
}
@@ -67,9 +61,9 @@ void *arch_dma_alloc(struct device *dev, size_t size,
6761
void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
6862
dma_addr_t dma_addr, unsigned long attrs)
6963
{
70-
if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !dev_is_dma_coherent(dev))
64+
if (!(attrs & DMA_ATTR_NON_CONSISTENT))
7165
cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr);
72-
dma_direct_free(dev, size, cpu_addr, dma_addr, attrs);
66+
dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs);
7367
}
7468

7569
int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma,
@@ -78,16 +72,11 @@ int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma,
7872
{
7973
unsigned long user_count = vma_pages(vma);
8074
unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
81-
unsigned long addr = (unsigned long)cpu_addr;
75+
unsigned long addr = CAC_ADDR((unsigned long)cpu_addr);
8276
unsigned long off = vma->vm_pgoff;
83-
unsigned long pfn;
77+
unsigned long pfn = page_to_pfn(virt_to_page((void *)addr));
8478
int ret = -ENXIO;
8579

86-
if (!dev_is_dma_coherent(dev))
87-
addr = CAC_ADDR(addr);
88-
89-
pfn = page_to_pfn(virt_to_page((void *)addr));
90-
9180
if (attrs & DMA_ATTR_WRITE_COMBINE)
9281
vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
9382
else
@@ -167,8 +156,7 @@ static inline void dma_sync_phys(phys_addr_t paddr, size_t size,
167156
void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
168157
size_t size, enum dma_data_direction dir)
169158
{
170-
if (!dev_is_dma_coherent(dev))
171-
dma_sync_phys(paddr, size, dir);
159+
dma_sync_phys(paddr, size, dir);
172160
}
173161

174162
void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
@@ -183,6 +171,5 @@ void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size,
183171
{
184172
BUG_ON(direction == DMA_NONE);
185173

186-
if (!dev_is_dma_coherent(dev))
187-
dma_sync_virt(vaddr, size, direction);
174+
dma_sync_virt(vaddr, size, direction);
188175
}

‎arch/nds32/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ config NDS32
1111
select CLKSRC_MMIO
1212
select CLONE_BACKWARDS
1313
select COMMON_CLK
14-
select DMA_NONCOHERENT_OPS
14+
select DMA_DIRECT_OPS
1515
select GENERIC_ATOMIC64
1616
select GENERIC_CPU_DEVICES
1717
select GENERIC_CLOCKEVENTS

‎arch/nios2/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ config NIOS2
44
select ARCH_HAS_SYNC_DMA_FOR_CPU
55
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
66
select ARCH_NO_SWAP
7-
select DMA_NONCOHERENT_OPS
7+
select DMA_DIRECT_OPS
88
select TIMER_OF
99
select GENERIC_ATOMIC64
1010
select GENERIC_CLOCKEVENTS

‎arch/openrisc/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
config OPENRISC
88
def_bool y
99
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
10-
select DMA_NONCOHERENT_OPS
10+
select DMA_DIRECT_OPS
1111
select OF
1212
select OF_EARLY_FLATTREE
1313
select IRQ_DOMAIN

‎arch/parisc/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ config PA11
186186
depends on PA7000 || PA7100LC || PA7200 || PA7300LC
187187
select ARCH_HAS_SYNC_DMA_FOR_CPU
188188
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
189-
select DMA_NONCOHERENT_OPS
189+
select DMA_DIRECT_OPS
190190
select DMA_NONCOHERENT_CACHE_SYNC
191191

192192
config PREFETCH

‎arch/parisc/kernel/setup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ void __init dma_ops_init(void)
102102
case pcxl: /* falls through */
103103
case pcxs:
104104
case pcxt:
105-
hppa_dma_ops = &dma_noncoherent_ops;
105+
hppa_dma_ops = &dma_direct_ops;
106106
break;
107107
default:
108108
break;

‎arch/sh/Kconfig

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ config SUPERH
77
select ARCH_NO_COHERENT_DMA_MMAP if !MMU
88
select HAVE_PATA_PLATFORM
99
select CLKDEV_LOOKUP
10+
select DMA_DIRECT_OPS
1011
select HAVE_IDE if HAS_IOPORT_MAP
1112
select HAVE_MEMBLOCK
1213
select HAVE_MEMBLOCK_NODE_MAP
@@ -158,13 +159,11 @@ config SWAP_IO_SPACE
158159
bool
159160

160161
config DMA_COHERENT
161-
select DMA_DIRECT_OPS
162162
bool
163163

164164
config DMA_NONCOHERENT
165165
def_bool !DMA_COHERENT
166166
select ARCH_HAS_SYNC_DMA_FOR_DEVICE
167-
select DMA_NONCOHERENT_OPS
168167

169168
config PGTABLE_LEVELS
170169
default 3 if X2TLB

‎arch/sparc/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ config SPARC
5151
config SPARC32
5252
def_bool !64BIT
5353
select ARCH_HAS_SYNC_DMA_FOR_CPU
54-
select DMA_NONCOHERENT_OPS
54+
select DMA_DIRECT_OPS
5555
select GENERIC_ATOMIC64
5656
select CLZ_TAB
5757
select HAVE_UID16

‎arch/sparc/include/asm/dma-mapping.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
1414
{
1515
#ifdef CONFIG_SPARC_LEON
1616
if (sparc_cpu_model == sparc_leon)
17-
return &dma_noncoherent_ops;
17+
return &dma_direct_ops;
1818
#endif
1919
#if defined(CONFIG_SPARC32) && defined(CONFIG_PCI)
2020
if (bus == &pci_bus_type)
21-
return &dma_noncoherent_ops;
21+
return &dma_direct_ops;
2222
#endif
2323
return dma_ops;
2424
}

‎arch/x86/kernel/amd_gart_64.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -482,7 +482,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
482482
{
483483
void *vaddr;
484484

485-
vaddr = dma_direct_alloc(dev, size, dma_addr, flag, attrs);
485+
vaddr = dma_direct_alloc_pages(dev, size, dma_addr, flag, attrs);
486486
if (!vaddr ||
487487
!force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24))
488488
return vaddr;
@@ -494,7 +494,7 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
494494
goto out_free;
495495
return vaddr;
496496
out_free:
497-
dma_direct_free(dev, size, vaddr, *dma_addr, attrs);
497+
dma_direct_free_pages(dev, size, vaddr, *dma_addr, attrs);
498498
return NULL;
499499
}
500500

@@ -504,7 +504,7 @@ gart_free_coherent(struct device *dev, size_t size, void *vaddr,
504504
dma_addr_t dma_addr, unsigned long attrs)
505505
{
506506
gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
507-
dma_direct_free(dev, size, vaddr, dma_addr, attrs);
507+
dma_direct_free_pages(dev, size, vaddr, dma_addr, attrs);
508508
}
509509

510510
static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr)

‎arch/xtensa/Kconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ config XTENSA
1212
select BUILDTIME_EXTABLE_SORT
1313
select CLONE_BACKWARDS
1414
select COMMON_CLK
15-
select DMA_NONCOHERENT_OPS
15+
select DMA_DIRECT_OPS
1616
select GENERIC_ATOMIC64
1717
select GENERIC_CLOCKEVENTS
1818
select GENERIC_IRQ_SHOW

‎include/asm-generic/dma-mapping.h

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,7 @@
44

55
static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
66
{
7-
/*
8-
* Use the non-coherent ops if available. If an architecture wants a
9-
* more fine-grained selection of operations it will have to implement
10-
* get_arch_dma_ops itself or use the per-device dma_ops.
11-
*/
12-
#ifdef CONFIG_DMA_NONCOHERENT_OPS
13-
return &dma_noncoherent_ops;
14-
#else
157
return &dma_direct_ops;
16-
#endif
178
}
189

1910
#endif /* _ASM_GENERIC_DMA_MAPPING_H */

‎include/linux/dma-direct.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,10 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
5959
gfp_t gfp, unsigned long attrs);
6060
void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
6161
dma_addr_t dma_addr, unsigned long attrs);
62+
void *dma_direct_alloc_pages(struct device *dev, size_t size,
63+
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs);
64+
void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
65+
dma_addr_t dma_addr, unsigned long attrs);
6266
dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
6367
unsigned long offset, size_t size, enum dma_data_direction dir,
6468
unsigned long attrs);

‎include/linux/dma-mapping.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,6 @@ struct dma_map_ops {
136136
};
137137

138138
extern const struct dma_map_ops dma_direct_ops;
139-
extern const struct dma_map_ops dma_noncoherent_ops;
140139
extern const struct dma_map_ops dma_virt_ops;
141140

142141
#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))

‎include/linux/dma-noncoherent.h

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,9 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
2424
gfp_t gfp, unsigned long attrs);
2525
void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
2626
dma_addr_t dma_addr, unsigned long attrs);
27-
28-
#ifdef CONFIG_DMA_NONCOHERENT_MMAP
2927
int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma,
3028
void *cpu_addr, dma_addr_t dma_addr, size_t size,
3129
unsigned long attrs);
32-
#else
33-
#define arch_dma_mmap NULL
34-
#endif /* CONFIG_DMA_NONCOHERENT_MMAP */
3530

3631
#ifdef CONFIG_DMA_NONCOHERENT_CACHE_SYNC
3732
void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size,

0 commit comments

Comments
 (0)