Skip to content

Commit b417417

Browse files
author
Christoph Hellwig
committed
dma-mapping: inline the fast path dma-direct calls
Inline the single page map/unmap/sync dma-direct calls into the now out of line generic wrappers. This restores the behavior of a single function call that we had before moving the generic calls out of line. Besides the dma-mapping callers there are just a few callers in IOMMU drivers that have a bypass mode, and more of those are going to be switched to the generic bypass soon. Signed-off-by: Christoph Hellwig <[email protected]> Tested-by: Alexey Kardashevskiy <[email protected]> Reviewed-by: Alexey Kardashevskiy <[email protected]>
1 parent d3fa60d commit b417417

File tree

2 files changed

+69
-88
lines changed

2 files changed

+69
-88
lines changed

‎include/linux/dma-direct.h

Lines changed: 69 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
/* SPDX-License-Identifier: GPL-2.0 */
2+
/*
3+
* Internals of the DMA direct mapping implementation. Only for use by the
4+
* DMA mapping code and IOMMU drivers.
5+
*/
26
#ifndef _LINUX_DMA_DIRECT_H
37
#define _LINUX_DMA_DIRECT_H 1
48

59
#include <linux/dma-mapping.h>
10+
#include <linux/dma-noncoherent.h>
611
#include <linux/memblock.h> /* for min_low_pfn */
712
#include <linux/mem_encrypt.h>
13+
#include <linux/swiotlb.h>
814

915
extern unsigned int zone_dma_bits;
1016

@@ -87,25 +93,17 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
8793
unsigned long attrs);
8894
int dma_direct_supported(struct device *dev, u64 mask);
8995
bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr);
90-
dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
91-
unsigned long offset, size_t size, enum dma_data_direction dir,
92-
unsigned long attrs);
9396
int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
9497
enum dma_data_direction dir, unsigned long attrs);
9598
dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,
9699
size_t size, enum dma_data_direction dir, unsigned long attrs);
100+
size_t dma_direct_max_mapping_size(struct device *dev);
97101

98102
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
99103
defined(CONFIG_SWIOTLB)
100-
void dma_direct_sync_single_for_device(struct device *dev,
101-
dma_addr_t addr, size_t size, enum dma_data_direction dir);
102-
void dma_direct_sync_sg_for_device(struct device *dev,
103-
struct scatterlist *sgl, int nents, enum dma_data_direction dir);
104+
void dma_direct_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
105+
int nents, enum dma_data_direction dir);
104106
#else
105-
static inline void dma_direct_sync_single_for_device(struct device *dev,
106-
dma_addr_t addr, size_t size, enum dma_data_direction dir)
107-
{
108-
}
109107
static inline void dma_direct_sync_sg_for_device(struct device *dev,
110108
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
111109
{
@@ -115,34 +113,82 @@ static inline void dma_direct_sync_sg_for_device(struct device *dev,
115113
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
116114
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
117115
defined(CONFIG_SWIOTLB)
118-
void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
119-
size_t size, enum dma_data_direction dir, unsigned long attrs);
120116
void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
121117
int nents, enum dma_data_direction dir, unsigned long attrs);
122-
void dma_direct_sync_single_for_cpu(struct device *dev,
123-
dma_addr_t addr, size_t size, enum dma_data_direction dir);
124118
void dma_direct_sync_sg_for_cpu(struct device *dev,
125119
struct scatterlist *sgl, int nents, enum dma_data_direction dir);
126120
#else
127-
static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
128-
size_t size, enum dma_data_direction dir, unsigned long attrs)
129-
{
130-
}
131121
static inline void dma_direct_unmap_sg(struct device *dev,
132122
struct scatterlist *sgl, int nents, enum dma_data_direction dir,
133123
unsigned long attrs)
134124
{
135125
}
126+
static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
127+
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
128+
{
129+
}
130+
#endif
131+
132+
static inline void dma_direct_sync_single_for_device(struct device *dev,
133+
dma_addr_t addr, size_t size, enum dma_data_direction dir)
134+
{
135+
phys_addr_t paddr = dma_to_phys(dev, addr);
136+
137+
if (unlikely(is_swiotlb_buffer(paddr)))
138+
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
139+
140+
if (!dev_is_dma_coherent(dev))
141+
arch_sync_dma_for_device(paddr, size, dir);
142+
}
143+
136144
static inline void dma_direct_sync_single_for_cpu(struct device *dev,
137145
dma_addr_t addr, size_t size, enum dma_data_direction dir)
138146
{
147+
phys_addr_t paddr = dma_to_phys(dev, addr);
148+
149+
if (!dev_is_dma_coherent(dev)) {
150+
arch_sync_dma_for_cpu(paddr, size, dir);
151+
arch_sync_dma_for_cpu_all();
152+
}
153+
154+
if (unlikely(is_swiotlb_buffer(paddr)))
155+
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
139156
}
140-
static inline void dma_direct_sync_sg_for_cpu(struct device *dev,
141-
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
157+
158+
static inline dma_addr_t dma_direct_map_page(struct device *dev,
159+
struct page *page, unsigned long offset, size_t size,
160+
enum dma_data_direction dir, unsigned long attrs)
142161
{
162+
phys_addr_t phys = page_to_phys(page) + offset;
163+
dma_addr_t dma_addr = phys_to_dma(dev, phys);
164+
165+
if (unlikely(swiotlb_force == SWIOTLB_FORCE))
166+
return swiotlb_map(dev, phys, size, dir, attrs);
167+
168+
if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
169+
if (swiotlb_force != SWIOTLB_NO_FORCE)
170+
return swiotlb_map(dev, phys, size, dir, attrs);
171+
172+
dev_WARN_ONCE(dev, 1,
173+
"DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
174+
&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
175+
return DMA_MAPPING_ERROR;
176+
}
177+
178+
if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
179+
arch_sync_dma_for_device(phys, size, dir);
180+
return dma_addr;
143181
}
144-
#endif
145182

146-
size_t dma_direct_max_mapping_size(struct device *dev);
183+
static inline void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
184+
size_t size, enum dma_data_direction dir, unsigned long attrs)
185+
{
186+
phys_addr_t phys = dma_to_phys(dev, addr);
147187

188+
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
189+
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
190+
191+
if (unlikely(is_swiotlb_buffer(phys)))
192+
swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);
193+
}
148194
#endif /* _LINUX_DMA_DIRECT_H */

‎kernel/dma/direct.c

Lines changed: 0 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,9 @@
1010
#include <linux/dma-direct.h>
1111
#include <linux/scatterlist.h>
1212
#include <linux/dma-contiguous.h>
13-
#include <linux/dma-noncoherent.h>
1413
#include <linux/pfn.h>
1514
#include <linux/vmalloc.h>
1615
#include <linux/set_memory.h>
17-
#include <linux/swiotlb.h>
1816

1917
/*
2018
* Most architectures use ZONE_DMA for the first 16 Megabytes, but some use it
@@ -304,18 +302,6 @@ void dma_direct_free(struct device *dev, size_t size,
304302

305303
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
306304
defined(CONFIG_SWIOTLB)
307-
void dma_direct_sync_single_for_device(struct device *dev,
308-
dma_addr_t addr, size_t size, enum dma_data_direction dir)
309-
{
310-
phys_addr_t paddr = dma_to_phys(dev, addr);
311-
312-
if (unlikely(is_swiotlb_buffer(paddr)))
313-
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE);
314-
315-
if (!dev_is_dma_coherent(dev))
316-
arch_sync_dma_for_device(paddr, size, dir);
317-
}
318-
319305
void dma_direct_sync_sg_for_device(struct device *dev,
320306
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
321307
{
@@ -339,20 +325,6 @@ void dma_direct_sync_sg_for_device(struct device *dev,
339325
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
340326
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \
341327
defined(CONFIG_SWIOTLB)
342-
void dma_direct_sync_single_for_cpu(struct device *dev,
343-
dma_addr_t addr, size_t size, enum dma_data_direction dir)
344-
{
345-
phys_addr_t paddr = dma_to_phys(dev, addr);
346-
347-
if (!dev_is_dma_coherent(dev)) {
348-
arch_sync_dma_for_cpu(paddr, size, dir);
349-
arch_sync_dma_for_cpu_all();
350-
}
351-
352-
if (unlikely(is_swiotlb_buffer(paddr)))
353-
swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU);
354-
}
355-
356328
void dma_direct_sync_sg_for_cpu(struct device *dev,
357329
struct scatterlist *sgl, int nents, enum dma_data_direction dir)
358330
{
@@ -374,18 +346,6 @@ void dma_direct_sync_sg_for_cpu(struct device *dev,
374346
arch_sync_dma_for_cpu_all();
375347
}
376348

377-
void dma_direct_unmap_page(struct device *dev, dma_addr_t addr,
378-
size_t size, enum dma_data_direction dir, unsigned long attrs)
379-
{
380-
phys_addr_t phys = dma_to_phys(dev, addr);
381-
382-
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
383-
dma_direct_sync_single_for_cpu(dev, addr, size, dir);
384-
385-
if (unlikely(is_swiotlb_buffer(phys)))
386-
swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs);
387-
}
388-
389349
void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
390350
int nents, enum dma_data_direction dir, unsigned long attrs)
391351
{
@@ -398,31 +358,6 @@ void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
398358
}
399359
#endif
400360

401-
dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
402-
unsigned long offset, size_t size, enum dma_data_direction dir,
403-
unsigned long attrs)
404-
{
405-
phys_addr_t phys = page_to_phys(page) + offset;
406-
dma_addr_t dma_addr = phys_to_dma(dev, phys);
407-
408-
if (unlikely(swiotlb_force == SWIOTLB_FORCE))
409-
return swiotlb_map(dev, phys, size, dir, attrs);
410-
411-
if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
412-
if (swiotlb_force != SWIOTLB_NO_FORCE)
413-
return swiotlb_map(dev, phys, size, dir, attrs);
414-
415-
dev_WARN_ONCE(dev, 1,
416-
"DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
417-
&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
418-
return DMA_MAPPING_ERROR;
419-
}
420-
421-
if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
422-
arch_sync_dma_for_device(phys, size, dir);
423-
return dma_addr;
424-
}
425-
426361
int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
427362
enum dma_data_direction dir, unsigned long attrs)
428363
{

0 commit comments

Comments
 (0)