From b1d2dc009dece4cd7e629419b52266ba51960a6b Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Thu, 23 May 2019 21:06:32 -0700 Subject: dma-contiguous: add dma_{alloc,free}_contiguous() helpers Both dma_alloc_from_contiguous() and dma_release_from_contiguous() are very simply implemented, but requiring callers to pass certain parameters like count and align, and taking a boolean parameter to check __GFP_NOWARN in the allocation flags. So every function call duplicates similar work: unsigned long order = get_order(size); size_t count = size >> PAGE_SHIFT; page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN); [...] dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); Additionally, as CMA can be used only in the context which permits sleeping, most of callers do a gfpflags_allow_blocking() check and a corresponding fallback allocation of normal pages upon any false result: if (gfpflags_allow_blocking(flag)) page = dma_alloc_from_contiguous(); if (!page) page = alloc_pages(); [...] if (!dma_release_from_contiguous(dev, page, count)) __free_pages(page, get_order(size)); So this patch simplifies those function calls by abstracting these operations into the two new functions: dma_{alloc,free}_contiguous. As some callers of dma_{alloc,release}_from_contiguous() might be complicated, this patch just implements these two new functions to kernel/dma/direct.c only as an initial step. Suggested-by: Christoph Hellwig Signed-off-by: Nicolin Chen Tested-by: dann frazier Signed-off-by: Christoph Hellwig --- include/linux/dma-contiguous.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'include/linux') diff --git a/include/linux/dma-contiguous.h b/include/linux/dma-contiguous.h index 6665fa03c0d1..428f3b7b1c42 100644 --- a/include/linux/dma-contiguous.h +++ b/include/linux/dma-contiguous.h @@ -111,6 +111,8 @@ struct page *dma_alloc_from_contiguous(struct device *dev, size_t count, unsigned int order, bool no_warn); bool dma_release_from_contiguous(struct device *dev, struct page *pages, int count); +struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp); +void dma_free_contiguous(struct device *dev, struct page *page, size_t size); #else @@ -153,6 +155,17 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, return false; } +static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size, + gfp_t gfp) +{ + return NULL; +} + +static inline void dma_free_contiguous(struct device *dev, struct page *page, + size_t size) +{ +} + #endif #endif -- cgit v1.2.3 From dd3dcede9fa0a0b661ac1f24843f4a1b1317fdb6 Mon Sep 17 00:00:00 2001 From: Nicolin Chen Date: Wed, 29 May 2019 17:54:25 -0700 Subject: dma-contiguous: fix !CONFIG_DMA_CMA version of dma_{alloc, free}_contiguous() Commit fdaeec198ada ("dma-contiguous: add dma_{alloc,free}_contiguous() helpers") adds a pair of new helper functions so as to abstract code in the dma-direct (and other places in the future), however it breaks QEMU boot feature using x86_64 defconfig. That's because x86_64 defconfig has CONFIG_DMA_CMA=n so those two newly introduced helper functions are empty in their !CONFIG_DMA_CMA version, while previously the platform independent dma-direct code had fallback alloc_pages_node() and __free_pages(). So this patch fixes it by adding alloc_pages_node() and __free_pages() in the !CONFIG_DMA_CMA version of the two helper functions. Tested with below QEMU command: qemu-system-x86_64 -m 512m \ -drive file=images/x86_64/rootfs.ext4,format=raw,if=ide \ -append 'console=ttyS0 root=/dev/sda' -nographic \ -kernel arch/x86_64/boot/bzImage with the rootfs from the below link: https://github.com/ClangBuiltLinux/continuous-integration/raw/master/images/x86_64/rootfs.ext4 Fixes: fdaeec198ada ("dma-contiguous: add dma_{alloc,free}_contiguous() helpers") Reported-by: Nathan Chancellor Signed-off-by: Nicolin Chen Signed-off-by: Christoph Hellwig --- include/linux/dma-contiguous.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/dma-contiguous.h b/include/linux/dma-contiguous.h index 428f3b7b1c42..c05d4e661489 100644 --- a/include/linux/dma-contiguous.h +++ b/include/linux/dma-contiguous.h @@ -50,6 +50,7 @@ #ifdef __KERNEL__ #include +#include struct cma; struct page; @@ -155,15 +156,20 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, return false; } +/* Use fallback alloc() and free() when CONFIG_DMA_CMA=n */ static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) { - return NULL; + int node = dev ? dev_to_node(dev) : NUMA_NO_NODE; + size_t align = get_order(PAGE_ALIGN(size)); + + return alloc_pages_node(node, gfp, align); } static inline void dma_free_contiguous(struct device *dev, struct page *page, size_t size) { + __free_pages(page, get_order(size)); } #endif -- cgit v1.2.3 From da83a722959a82733c3ca60030cc364ca2318c5a Mon Sep 17 00:00:00 2001 From: Fredrik Noring Date: Wed, 29 May 2019 13:28:39 +0300 Subject: lib/genalloc: add gen_pool_dma_zalloc() for zeroed DMA allocations gen_pool_dma_zalloc() is a zeroed memory variant of gen_pool_dma_alloc(). Also document the return values of both, and indicate NULL as a "%NULL" constant. Signed-off-by: Fredrik Noring Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- include/linux/genalloc.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/genalloc.h b/include/linux/genalloc.h index dd0a452373e7..6c62eeca754f 100644 --- a/include/linux/genalloc.h +++ b/include/linux/genalloc.h @@ -121,6 +121,7 @@ extern unsigned long gen_pool_alloc_algo(struct gen_pool *, size_t, genpool_algo_t algo, void *data); extern void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); +void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); extern void gen_pool_free(struct gen_pool *, unsigned long, size_t); extern void gen_pool_for_each_chunk(struct gen_pool *, void (*)(struct gen_pool *, struct gen_pool_chunk *, void *), void *); -- cgit v1.2.3 From b0310c2f09bbe8aebefb97ed67949a3a7092aca6 Mon Sep 17 00:00:00 2001 From: Laurentiu Tudor Date: Wed, 29 May 2019 13:28:40 +0300 Subject: USB: use genalloc for USB HCs with local memory For HCs that have local memory, replace the current DMA API usage with a genalloc generic allocator to manage the mappings for these devices. To help users, introduce a new HCD API, usb_hcd_setup_local_mem() that will setup up the genalloc backing up the device local memory. It will be used in subsequent patches. This is in preparation for dropping the existing "coherent" dma mem declaration APIs. The current implementation was relying on a short circuit in the DMA API that in the end, was acting as an allocator for these type of devices. Signed-off-by: Laurentiu Tudor Tested-by: Fredrik Noring Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- include/linux/usb/hcd.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux') diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index bb57b5af4700..127560a4bfa0 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -216,6 +216,9 @@ struct usb_hcd { #define HC_IS_RUNNING(state) ((state) & __ACTIVE) #define HC_IS_SUSPENDED(state) ((state) & __SUSPEND) + /* memory pool for HCs having local memory, or %NULL */ + struct gen_pool *localmem_pool; + /* more shared queuing code would be good; it should support * smarter scheduling, handle transaction translators, etc; * input size of periodic table to an interrupt scheduler. @@ -461,6 +464,8 @@ extern int usb_add_hcd(struct usb_hcd *hcd, unsigned int irqnum, unsigned long irqflags); extern void usb_remove_hcd(struct usb_hcd *hcd); extern int usb_hcd_find_raw_port_number(struct usb_hcd *hcd, int port1); +int usb_hcd_setup_local_mem(struct usb_hcd *hcd, phys_addr_t phys_addr, + dma_addr_t dma, size_t size); struct platform_device; extern void usb_hcd_platform_shutdown(struct platform_device *dev); -- cgit v1.2.3 From 2d7a3dc3e24f43504b1f25eae8195e600f4cce8b Mon Sep 17 00:00:00 2001 From: Laurentiu Tudor Date: Wed, 29 May 2019 13:28:43 +0300 Subject: USB: drop HCD_LOCAL_MEM flag With the addition of the local memory allocator, the HCD_LOCAL_MEM flag can be dropped and the checks against it replaced with a check for the localmem_pool ptr being initialized. Signed-off-by: Laurentiu Tudor Tested-by: Fredrik Noring Reviewed-by: Greg Kroah-Hartman Signed-off-by: Christoph Hellwig --- include/linux/usb/hcd.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 127560a4bfa0..bab27ccc8ff5 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -256,7 +256,6 @@ struct hc_driver { int flags; #define HCD_MEMORY 0x0001 /* HC regs use memory (else I/O) */ -#define HCD_LOCAL_MEM 0x0002 /* HC needs local memory */ #define HCD_SHARED 0x0004 /* Two (or more) usb_hcds share HW */ #define HCD_USB11 0x0010 /* USB 1.1 */ #define HCD_USB2 0x0020 /* USB 2.0 */ -- cgit v1.2.3 From c30700db9eaabb35e0b123301df35a6846e6b6b4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 3 Jun 2019 08:43:51 +0200 Subject: dma-direct: provide generic support for uncached kernel segments A few architectures support uncached kernel segments. In that case we get an uncached mapping for a given physica address by using an offset in the uncached segement. Implement support for this scheme in the generic dma-direct code instead of duplicating it in arch hooks. Signed-off-by: Christoph Hellwig --- include/linux/dma-noncoherent.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 9741767e400f..7e0126a04e02 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -80,4 +80,7 @@ static inline void arch_dma_prep_coherent(struct page *page, size_t size) } #endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */ +void *uncached_kernel_address(void *addr); +void *cached_kernel_address(void *addr); + #endif /* _LINUX_DMA_NONCOHERENT_H */ -- cgit v1.2.3 From 4b85faed211ccfbcc7f3adf1cd62f0b00d1a172b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 14 Jun 2019 16:06:10 +0200 Subject: dma-mapping: add a dma_alloc_need_uncached helper Check if we need to allocate uncached memory for a device given the allocation flags. Switch over the uncached segment check to this helper to deal with architectures that do not support the dma_cache_sync operation and thus should not returned cacheable memory for DMA_ATTR_NON_CONSISTENT allocations. Signed-off-by: Christoph Hellwig --- include/linux/dma-noncoherent.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'include/linux') diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 7e0126a04e02..732919ac5c11 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -20,6 +20,20 @@ static inline bool dev_is_dma_coherent(struct device *dev) } #endif /* CONFIG_ARCH_HAS_DMA_COHERENCE_H */ +/* + * Check if an allocation needs to be marked uncached to be coherent. + */ +static inline bool dma_alloc_need_uncached(struct device *dev, + unsigned long attrs) +{ + if (dev_is_dma_coherent(dev)) + return false; + if (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) && + (attrs & DMA_ATTR_NON_CONSISTENT)) + return false; + return true; +} + void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, -- cgit v1.2.3 From d98849aff87911013aadb730138ab728b52fc547 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 14 Jun 2019 16:17:27 +0200 Subject: dma-direct: handle DMA_ATTR_NO_KERNEL_MAPPING in common code DMA_ATTR_NO_KERNEL_MAPPING is generally implemented by allocating normal cacheable pages or CMA memory, and then returning the page pointer as the opaque handle. Lift that code from the xtensa and generic dma remapping implementations into the generic dma-direct code so that we don't even call arch_dma_alloc for these allocations. Signed-off-by: Christoph Hellwig --- include/linux/dma-noncoherent.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 732919ac5c11..53ee36ecdf37 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -28,6 +28,8 @@ static inline bool dma_alloc_need_uncached(struct device *dev, { if (dev_is_dma_coherent(dev)) return false; + if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) + return false; if (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) && (attrs & DMA_ATTR_NON_CONSISTENT)) return false; -- cgit v1.2.3 From cf394fc5f7155c24efb584979e81427575ab3539 Mon Sep 17 00:00:00 2001 From: Fredrik Noring Date: Tue, 25 Jun 2019 17:05:58 +0200 Subject: lib/genalloc.c: Add algorithm, align and zeroed family of DMA allocators Provide the algorithm option to DMA allocators as well, along with convenience variants for zeroed and aligned memory. The following four functions are added: - gen_pool_dma_alloc_algo() - gen_pool_dma_alloc_align() - gen_pool_dma_zalloc_algo() - gen_pool_dma_zalloc_align() Signed-off-by: Fredrik Noring Tested-by: Guenter Roeck Signed-off-by: Christoph Hellwig --- include/linux/genalloc.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/genalloc.h b/include/linux/genalloc.h index 6c62eeca754f..ed641337df87 100644 --- a/include/linux/genalloc.h +++ b/include/linux/genalloc.h @@ -121,7 +121,15 @@ extern unsigned long gen_pool_alloc_algo(struct gen_pool *, size_t, genpool_algo_t algo, void *data); extern void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); -void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); +extern void *gen_pool_dma_alloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data); +extern void *gen_pool_dma_alloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align); +extern void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma); +extern void *gen_pool_dma_zalloc_algo(struct gen_pool *pool, size_t size, + dma_addr_t *dma, genpool_algo_t algo, void *data); +extern void *gen_pool_dma_zalloc_align(struct gen_pool *pool, size_t size, + dma_addr_t *dma, int align); extern void gen_pool_free(struct gen_pool *, unsigned long, size_t); extern void gen_pool_for_each_chunk(struct gen_pool *, void (*)(struct gen_pool *, struct gen_pool_chunk *, void *), void *); -- cgit v1.2.3 From 15ffe5e1acf5fe1512e98b20702e46ce9f25e2f7 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 8 Jul 2019 12:55:27 -0700 Subject: dma-mapping: mark dma_alloc_need_uncached as __always_inline Without the __always_inline at least i386 configs that have CONFIG_OPTIMIZE_INLINING set seem fail to inline dma_alloc_need_uncached, leading to a linker error because of undefined symbols. Reported-by: Randy Dunlap Signed-off-by: Christoph Hellwig Acked-by: Randy Dunlap # build-tested --- include/linux/dma-noncoherent.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h index 53ee36ecdf37..3813211a9aad 100644 --- a/include/linux/dma-noncoherent.h +++ b/include/linux/dma-noncoherent.h @@ -23,7 +23,7 @@ static inline bool dev_is_dma_coherent(struct device *dev) /* * Check if an allocation needs to be marked uncached to be coherent. */ -static inline bool dma_alloc_need_uncached(struct device *dev, +static __always_inline bool dma_alloc_need_uncached(struct device *dev, unsigned long attrs) { if (dev_is_dma_coherent(dev)) -- cgit v1.2.3