From a41ef1e455a9796be8cb986f0616f52453ac8e4b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Nov 2017 07:32:51 -0800 Subject: dma-mapping: take dma_pfn_offset into account in dma_max_pfn This makes sure the generic version can be used with architectures / devices that have a DMA offset in the direct mapping. Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy --- include/linux/dma-mapping.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 81ed9b2d84dc..d84951865be7 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -692,7 +692,7 @@ static inline int dma_set_seg_boundary(struct device *dev, unsigned long mask) #ifndef dma_max_pfn static inline unsigned long dma_max_pfn(struct device *dev) { - return *dev->dma_mask >> PAGE_SHIFT; + return (*dev->dma_mask >> PAGE_SHIFT) + dev->dma_pfn_offset; } #endif -- cgit v1.2.3 From ea8c64ace86647260ec4255f483e5844d62af2df Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 10 Jan 2018 16:21:13 +0100 Subject: dma-mapping: move swiotlb arch helpers to a new header phys_to_dma, dma_to_phys and dma_capable are helpers published by architecture code for use of swiotlb and xen-swiotlb only. Drivers are not supposed to use these directly, but use the DMA API instead. Move these to a new asm/dma-direct.h helper, included by a linux/dma-direct.h wrapper that provides the default linear mapping unless the architecture wants to override it. In the MIPS case the existing dma-coherent.h is reused for now as untangling it will take a bit of work. Signed-off-by: Christoph Hellwig Acked-by: Robin Murphy --- include/linux/dma-direct.h | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 include/linux/dma-direct.h (limited to 'include') diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h new file mode 100644 index 000000000000..2cc1b6558944 --- /dev/null +++ b/include/linux/dma-direct.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_DMA_DIRECT_H +#define _LINUX_DMA_DIRECT_H 1 + +#include + +#ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA +#include +#else +static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) +{ + dma_addr_t dev_addr = (dma_addr_t)paddr; + + return dev_addr - ((dma_addr_t)dev->dma_pfn_offset << PAGE_SHIFT); +} + +static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr) +{ + phys_addr_t paddr = (phys_addr_t)dev_addr; + + return paddr + ((phys_addr_t)dev->dma_pfn_offset << PAGE_SHIFT); +} + +static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) +{ + if (!dev->dma_mask) + return false; + + return addr + size - 1 <= *dev->dma_mask; +} +#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ +#endif /* _LINUX_DMA_DIRECT_H */ -- cgit v1.2.3 From b49efd76248242169f28ffd20ada05064d01ed9f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 22:11:31 +0100 Subject: dma-mapping: move dma_mark_clean to dma-direct.h And unlike the other helpers we don't require a as this helper is a special case for ia64 only, and this keeps it as simple as possible. Signed-off-by: Christoph Hellwig --- include/linux/dma-direct.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'include') diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index 2cc1b6558944..10e924b7cba7 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -29,4 +29,13 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) return addr + size - 1 <= *dev->dma_mask; } #endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ + +#ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN +void dma_mark_clean(void *addr, size_t size); +#else +static inline void dma_mark_clean(void *addr, size_t size) +{ +} +#endif /* CONFIG_ARCH_HAS_DMA_MARK_CLEAN */ + #endif /* _LINUX_DMA_DIRECT_H */ -- cgit v1.2.3 From 205e1b7f51e4af2643eb1d61a6503e415cd1e014 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 22 Dec 2017 14:50:47 +0100 Subject: dma-mapping: warn when there is no coherent_dma_mask These days all devices should have a DMA coherent mask, and most dma_ops implementations rely on that fact. But just to be sure add an assert to ring the warning bell if that is not the case. Signed-off-by: Christoph Hellwig Reviewed-by: Vladimir Murzin Reviewed-by: Konrad Rzeszutek Wilk --- include/linux/dma-mapping.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index d84951865be7..16add66d5b68 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -513,6 +513,7 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size, void *cpu_addr; BUG_ON(!ops); + WARN_ON_ONCE(dev && !dev->coherent_dma_mask); if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) return cpu_addr; -- cgit v1.2.3 From 57bf5a8963f80fb3828c46c3e3a5b2dd790e09a7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 22 Dec 2017 16:05:15 +0100 Subject: dma-mapping: clear harmful GFP_* flags in common code Lift the code from x86 so that we behave consistently. In the future we should probably warn if any of these is set. Signed-off-by: Christoph Hellwig Acked-by: Jesper Nilsson Acked-by: Geert Uytterhoeven [m68k] --- include/linux/dma-mapping.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include') diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 16add66d5b68..d036e78b9ae9 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -518,6 +518,13 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size, if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) return cpu_addr; + /* + * Let the implementation decide on the zone to allocate from, and + * decide on the way of zeroing the memory given that the memory + * returned should always be zeroed. + */ + flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_ZERO); + if (!arch_dma_alloc_attrs(&dev, &flag)) return NULL; if (!ops->alloc) -- cgit v1.2.3 From cea9d03c822cf1b8e90cc4fc51be6d248fb5d776 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Dec 2017 11:01:41 +0100 Subject: dma-mapping: add an arch_dma_supported hook To implement the x86 forbid_dac and iommu_sac_force we want an arch hook so that it can apply the global options across all dma_map_ops implementations. Signed-off-by: Christoph Hellwig Reviewed-by: Konrad Rzeszutek Wilk --- include/linux/dma-mapping.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'include') diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index d036e78b9ae9..46542ad9d709 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -576,6 +576,14 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) return 0; } +/* + * This is a hack for the legacy x86 forbid_dac and iommu_sac_force. Please + * don't use this is new code. + */ +#ifndef arch_dma_supported +#define arch_dma_supported(dev, mask) (1) +#endif + static inline void dma_check_mask(struct device *dev, u64 mask) { if (sme_active() && (mask < (((u64)sme_get_me_mask() << 1) - 1))) @@ -588,6 +596,9 @@ static inline int dma_supported(struct device *dev, u64 mask) if (!ops) return 0; + if (!arch_dma_supported(dev, mask)) + return 0; + if (!ops->dma_supported) return 1; return ops->dma_supported(dev, mask); -- cgit v1.2.3 From c5cd037d1c8044fbd131c57822a67a1576eb16e9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Dec 2017 10:56:51 +0100 Subject: dma-mapping: provide a generic asm/dma-mapping.h For architectures that just use the generic dma_noop_ops we can provide a generic version of dma-mapping.h. Signed-off-by: Christoph Hellwig --- include/asm-generic/dma-mapping.h | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 include/asm-generic/dma-mapping.h (limited to 'include') diff --git a/include/asm-generic/dma-mapping.h b/include/asm-generic/dma-mapping.h new file mode 100644 index 000000000000..164031531d85 --- /dev/null +++ b/include/asm-generic/dma-mapping.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_GENERIC_DMA_MAPPING_H +#define _ASM_GENERIC_DMA_MAPPING_H + +static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) +{ + return &dma_noop_ops; +} + +#endif /* _ASM_GENERIC_DMA_MAPPING_H */ -- cgit v1.2.3 From 002e67454f61bb67d8071ac4d0cacb86a01d18e0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 16:30:23 +0100 Subject: dma-direct: rename dma_noop to dma_direct The trivial direct mapping implementation already does a virtual to physical translation which isn't strictly a noop, and will soon learn to do non-direct but linear physical to dma translations through the device offset and a few small tricks. Rename it to a better fitting name. Signed-off-by: Christoph Hellwig Reviewed-by: Vladimir Murzin --- include/asm-generic/dma-mapping.h | 2 +- include/linux/dma-mapping.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/asm-generic/dma-mapping.h b/include/asm-generic/dma-mapping.h index 164031531d85..880a292d792f 100644 --- a/include/asm-generic/dma-mapping.h +++ b/include/asm-generic/dma-mapping.h @@ -4,7 +4,7 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) { - return &dma_noop_ops; + return &dma_direct_ops; } #endif /* _ASM_GENERIC_DMA_MAPPING_H */ diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 46542ad9d709..34fe8463d10e 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -136,7 +136,7 @@ struct dma_map_ops { int is_phys; }; -extern const struct dma_map_ops dma_noop_ops; +extern const struct dma_map_ops dma_direct_ops; extern const struct dma_map_ops dma_virt_ops; #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1)) -- cgit v1.2.3 From 19dca8c0efa30e0a45e79f237060d0f307045752 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Dec 2017 13:46:06 +0100 Subject: dma-direct: make dma_direct_{alloc,free} available to other implementations So that they don't need to indirect through the operation vector. Signed-off-by: Christoph Hellwig Reviewed-by: Vladimir Murzin --- include/linux/dma-direct.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include') diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index 10e924b7cba7..4788bf0bf683 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -38,4 +38,9 @@ static inline void dma_mark_clean(void *addr, size_t size) } #endif /* CONFIG_ARCH_HAS_DMA_MARK_CLEAN */ +void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t gfp, unsigned long attrs); +void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_addr, unsigned long attrs); + #endif /* _LINUX_DMA_DIRECT_H */ -- cgit v1.2.3 From 1a9777a8a01fb88659a3dda48080c95c34cab7d3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Dec 2017 15:04:32 +0100 Subject: dma-direct: reject too small dma masks Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy --- include/linux/dma-direct.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index 4788bf0bf683..bcdb1a3e4b1f 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -42,5 +42,6 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs); +int dma_direct_supported(struct device *dev, u64 mask); #endif /* _LINUX_DMA_DIRECT_H */ -- cgit v1.2.3 From 7f2c8bbd321f18e4ccfd262748bd58fb7d4bb1db Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Dec 2017 14:14:54 +0100 Subject: swiotlb: rename swiotlb_free to swiotlb_exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christoph Hellwig Acked-by: Christian König Reviewed-by: Konrad Rzeszutek Wilk --- include/linux/swiotlb.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 24ed817082ee..606375e35d87 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -115,10 +115,10 @@ extern int swiotlb_dma_supported(struct device *hwdev, u64 mask); #ifdef CONFIG_SWIOTLB -extern void __init swiotlb_free(void); +extern void __init swiotlb_exit(void); unsigned int swiotlb_max_segment(void); #else -static inline void swiotlb_free(void) { } +static inline void swiotlb_exit(void) { } static inline unsigned int swiotlb_max_segment(void) { return 0; } #endif -- cgit v1.2.3 From 251533eb35b22f9e87a440b14c6f20e2626397b3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 16:44:16 +0100 Subject: swiotlb: add common swiotlb_map_ops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently all architectures that want to use swiotlb have to implement their own dma_map_ops instances. Provide a generic one based on the x86 implementation which first calls into dma_direct to try a full blown direct mapping implementation (including e.g. CMA) before falling back allocating from the swiotlb buffer. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- include/linux/swiotlb.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 606375e35d87..5b1f2a00491c 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -66,6 +66,12 @@ extern void swiotlb_tbl_sync_single(struct device *hwdev, enum dma_sync_target target); /* Accessory functions. */ + +void *swiotlb_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle, + gfp_t flags, unsigned long attrs); +void swiotlb_free(struct device *dev, size_t size, void *vaddr, + dma_addr_t dma_addr, unsigned long attrs); + extern void *swiotlb_alloc_coherent(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags); @@ -126,4 +132,6 @@ extern void swiotlb_print_info(void); extern int is_swiotlb_buffer(phys_addr_t paddr); extern void swiotlb_set_max_segment(unsigned int); +extern const struct dma_map_ops swiotlb_dma_ops; + #endif /* __LINUX_SWIOTLB_H */ -- cgit v1.2.3