From b36f09c3c441a6e59eab9315032e7d546571de3f Mon Sep 17 00:00:00 2001 From: Lars-Peter Clausen Date: Tue, 20 Oct 2015 11:46:28 +0200 Subject: dmaengine: Add transfer termination synchronization support The DMAengine API has a long standing race condition that is inherent to the API itself. Calling dmaengine_terminate_all() is supposed to stop and abort any pending or active transfers that have previously been submitted. Unfortunately it is possible that this operation races against a currently running (or with some drivers also scheduled) completion callback. Since the API allows dmaengine_terminate_all() to be called from atomic context as well as from within a completion callback it is not possible to synchronize to the execution of the completion callback from within dmaengine_terminate_all() itself. This means that a user of the DMAengine API does not know when it is safe to free resources used in the completion callback, which can result in a use-after-free race condition. This patch addresses the issue by introducing an explicit synchronization primitive to the DMAengine API called dmaengine_synchronize(). The existing dmaengine_terminate_all() is deprecated in favor of dmaengine_terminate_sync() and dmaengine_terminate_async(). The former aborts all pending and active transfers and synchronizes to the current context, meaning it will wait until all running completion callbacks have finished. This means it is only possible to call this function from non-atomic context. The later function does not synchronize, but can still be used in atomic context or from within a complete callback. It has to be followed up by dmaengine_synchronize() before a client can free the resources used in a completion callback. In addition to this the semantics of the device_terminate_all() callback are slightly relaxed by this patch. It is now OK for a driver to only schedule the termination of the active transfer, but does not necessarily have to wait until the DMA controller has completely stopped. The driver must ensure though that the controller has stopped and no longer accesses any memory when the device_synchronize() callback returns. This was in part done since most drivers do not pay attention to this anyway at the moment and to emphasize that this needs to be done when the device_synchronize() callback is implemented. But it also helps with implementing support for devices where stopping the controller can require operations that may sleep. Signed-off-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 90 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) (limited to 'include/linux/dmaengine.h') diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index c47c68e535e8..4662d9aa6d5a 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -654,6 +654,8 @@ enum dmaengine_alignment { * paused. Returns 0 or an error code * @device_terminate_all: Aborts all transfers on a channel. Returns 0 * or an error code + * @device_synchronize: Synchronizes the termination of a transfers to the + * current context. * @device_tx_status: poll for transaction completion, the optional * txstate parameter can be supplied with a pointer to get a * struct with auxiliary transfer status information, otherwise the call @@ -737,6 +739,7 @@ struct dma_device { int (*device_pause)(struct dma_chan *chan); int (*device_resume)(struct dma_chan *chan); int (*device_terminate_all)(struct dma_chan *chan); + void (*device_synchronize)(struct dma_chan *chan); enum dma_status (*device_tx_status)(struct dma_chan *chan, dma_cookie_t cookie, @@ -828,6 +831,13 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_sg( src_sg, src_nents, flags); } +/** + * dmaengine_terminate_all() - Terminate all active DMA transfers + * @chan: The channel for which to terminate the transfers + * + * This function is DEPRECATED use either dmaengine_terminate_sync() or + * dmaengine_terminate_async() instead. + */ static inline int dmaengine_terminate_all(struct dma_chan *chan) { if (chan->device->device_terminate_all) @@ -836,6 +846,86 @@ static inline int dmaengine_terminate_all(struct dma_chan *chan) return -ENOSYS; } +/** + * dmaengine_terminate_async() - Terminate all active DMA transfers + * @chan: The channel for which to terminate the transfers + * + * Calling this function will terminate all active and pending descriptors + * that have previously been submitted to the channel. It is not guaranteed + * though that the transfer for the active descriptor has stopped when the + * function returns. Furthermore it is possible the complete callback of a + * submitted transfer is still running when this function returns. + * + * dmaengine_synchronize() needs to be called before it is safe to free + * any memory that is accessed by previously submitted descriptors or before + * freeing any resources accessed from within the completion callback of any + * perviously submitted descriptors. + * + * This function can be called from atomic context as well as from within a + * complete callback of a descriptor submitted on the same channel. + * + * If none of the two conditions above apply consider using + * dmaengine_terminate_sync() instead. + */ +static inline int dmaengine_terminate_async(struct dma_chan *chan) +{ + if (chan->device->device_terminate_all) + return chan->device->device_terminate_all(chan); + + return -EINVAL; +} + +/** + * dmaengine_synchronize() - Synchronize DMA channel termination + * @chan: The channel to synchronize + * + * Synchronizes to the DMA channel termination to the current context. When this + * function returns it is guaranteed that all transfers for previously issued + * descriptors have stopped and and it is safe to free the memory assoicated + * with them. Furthermore it is guaranteed that all complete callback functions + * for a previously submitted descriptor have finished running and it is safe to + * free resources accessed from within the complete callbacks. + * + * The behavior of this function is undefined if dma_async_issue_pending() has + * been called between dmaengine_terminate_async() and this function. + * + * This function must only be called from non-atomic context and must not be + * called from within a complete callback of a descriptor submitted on the same + * channel. + */ +static inline void dmaengine_synchronize(struct dma_chan *chan) +{ + if (chan->device->device_synchronize) + chan->device->device_synchronize(chan); +} + +/** + * dmaengine_terminate_sync() - Terminate all active DMA transfers + * @chan: The channel for which to terminate the transfers + * + * Calling this function will terminate all active and pending transfers + * that have previously been submitted to the channel. It is similar to + * dmaengine_terminate_async() but guarantees that the DMA transfer has actually + * stopped and that all complete callbacks have finished running when the + * function returns. + * + * This function must only be called from non-atomic context and must not be + * called from within a complete callback of a descriptor submitted on the same + * channel. + */ +static inline int dmaengine_terminate_sync(struct dma_chan *chan) +{ + int ret; + + ret = dmaengine_terminate_async(chan); + if (ret) + return ret; + + dmaengine_synchronize(chan); + + return 0; +} + static inline int dmaengine_pause(struct dma_chan *chan) { if (chan->device->device_pause) -- cgit v1.2.3 From 9eeacd3a2f17438d9d286ff2f78c4709a4148be7 Mon Sep 17 00:00:00 2001 From: Robert Jarzmik Date: Tue, 13 Oct 2015 21:54:29 +0200 Subject: dmaengine: enable DMA_CTRL_REUSE In the current state, the capability of transfer reuse can neither be set by a slave dmaengine driver, nor used by a client driver, because the capability is not available to dma_get_slave_caps(). Fix this by adding a way to declare the capability. Fixes: 272420214d26 ("dmaengine: Add DMA_CTRL_REUSE") Signed-off-by: Robert Jarzmik Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux/dmaengine.h') diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index c47c68e535e8..6f94b5cbd97c 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -659,6 +659,7 @@ enum dmaengine_alignment { * struct with auxiliary transfer status information, otherwise the call * will just return a simple status code * @device_issue_pending: push pending transactions to hardware + * @descriptor_reuse: a submitted transfer can be resubmitted after completion */ struct dma_device { @@ -681,6 +682,7 @@ struct dma_device { u32 src_addr_widths; u32 dst_addr_widths; u32 directions; + bool descriptor_reuse; enum dma_residue_granularity residue_granularity; int (*device_alloc_chan_resources)(struct dma_chan *chan); -- cgit v1.2.3 From b1d6ab1aa8cdc23b89bcd578ea8d5e3c501a13d9 Mon Sep 17 00:00:00 2001 From: Lars-Peter Clausen Date: Mon, 23 Nov 2015 11:06:43 +0100 Subject: dmaengine: Add might_sleep() to dmaengine_synchronize() Implementations of dmaengine_synchronize() are allowed to sleep, hence the function must not be called to from atomic context. Add might_sleep() to dmaengine_synchronize() to make it easier to detect non-compliant callers. Suggested-by: Andy Shevchenko Signed-off-by: Lars-Peter Clausen Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux/dmaengine.h') diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 4662d9aa6d5a..2f69e1d93f92 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -895,6 +895,8 @@ static inline int dmaengine_terminate_async(struct dma_chan *chan) */ static inline void dmaengine_synchronize(struct dma_chan *chan) { + might_sleep(); + if (chan->device->device_synchronize) chan->device->device_synchronize(chan); } -- cgit v1.2.3 From a8135d0d79e9d0ad3a4ff494fceeaae838becf38 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Mon, 14 Dec 2015 22:47:40 +0200 Subject: dmaengine: core: Introduce new, universal API to request a channel The two API function can cover most, if not all current APIs used to request a channel. With minimal effort dmaengine drivers, platforms and dmaengine user drivers can be converted to use the two function. struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask); To request any channel matching with the requested capabilities, can be used to request channel for memcpy, memset, xor, etc where no hardware synchronization is needed. struct dma_chan *dma_request_chan(struct device *dev, const char *name); To request a slave channel. The dma_request_chan() will try to find the channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode it will use a filter lookup table and retrieves the needed information from the dma_slave_map provided by the DMA drivers. This legacy mode needs changes in platform code, in dmaengine drivers and finally the dmaengine user drivers can be converted: For each dmaengine driver an array of DMA device, slave and the parameter for the filter function needs to be added: static const struct dma_slave_map da830_edma_map[] = { { "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) }, { "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) }, { "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) }, { "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) }, { "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) }, { "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) }, { "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) }, { "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) }, { "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) }, { "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) }, { "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) }, { "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) }, }; This information is going to be needed by the dmaengine driver, so modification to the platform_data is needed, and the driver map should be added to the pdata of the DMA driver: da8xx_edma0_pdata.slave_map = da830_edma_map; da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map); The DMA driver then needs to configure the needed device -> filter_fn mapping before it registers with dma_async_device_register() : ecc->dma_slave.filter_map.map = info->slave_map; ecc->dma_slave.filter_map.mapcnt = info->slavecnt; ecc->dma_slave.filter_map.fn = edma_filter_fn; When neither DT or ACPI lookup is available the dma_request_chan() will try to match the requester's device name with the filter_map's list of device names, when a match found it will use the information from the dma_slave_map to get the channel with the dma_get_channel() internal function. Signed-off-by: Peter Ujfalusi Reviewed-by: Arnd Bergmann Signed-off-by: Vinod Koul --- include/linux/dmaengine.h | 51 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 7 deletions(-) (limited to 'include/linux/dmaengine.h') diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index c47c68e535e8..d50a6b51a73d 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -606,12 +606,39 @@ enum dmaengine_alignment { DMAENGINE_ALIGN_64_BYTES = 6, }; +/** + * struct dma_slave_map - associates slave device and it's slave channel with + * parameter to be used by a filter function + * @devname: name of the device + * @slave: slave channel name + * @param: opaque parameter to pass to struct dma_filter.fn + */ +struct dma_slave_map { + const char *devname; + const char *slave; + void *param; +}; + +/** + * struct dma_filter - information for slave device/channel to filter_fn/param + * mapping + * @fn: filter function callback + * @mapcnt: number of slave device/channel in the map + * @map: array of channel to filter mapping data + */ +struct dma_filter { + dma_filter_fn fn; + int mapcnt; + const struct dma_slave_map *map; +}; + /** * struct dma_device - info on the entity supplying DMA services * @chancnt: how many DMA channels are supported * @privatecnt: how many DMA channels are requested by dma_request_channel * @channels: the list of struct dma_chan * @global_node: list_head for global dma_device_list + * @filter: information for device/slave to filter function/param mapping * @cap_mask: one or more dma_capability flags * @max_xor: maximum number of xor sources, 0 if no capability * @max_pq: maximum number of PQ sources and PQ-continue capability @@ -666,6 +693,7 @@ struct dma_device { unsigned int privatecnt; struct list_head channels; struct list_head global_node; + struct dma_filter filter; dma_cap_mask_t cap_mask; unsigned short max_xor; unsigned short max_pq; @@ -1140,9 +1168,11 @@ enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx); void dma_issue_pending_all(void); struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, dma_filter_fn fn, void *fn_param); -struct dma_chan *dma_request_slave_channel_reason(struct device *dev, - const char *name); struct dma_chan *dma_request_slave_channel(struct device *dev, const char *name); + +struct dma_chan *dma_request_chan(struct device *dev, const char *name); +struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask); + void dma_release_channel(struct dma_chan *chan); int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps); #else @@ -1166,16 +1196,21 @@ static inline struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, { return NULL; } -static inline struct dma_chan *dma_request_slave_channel_reason( - struct device *dev, const char *name) -{ - return ERR_PTR(-ENODEV); -} static inline struct dma_chan *dma_request_slave_channel(struct device *dev, const char *name) { return NULL; } +static inline struct dma_chan *dma_request_chan(struct device *dev, + const char *name) +{ + return ERR_PTR(-ENODEV); +} +static inline struct dma_chan *dma_request_chan_by_mask( + const dma_cap_mask_t *mask) +{ + return ERR_PTR(-ENODEV); +} static inline void dma_release_channel(struct dma_chan *chan) { } @@ -1186,6 +1221,8 @@ static inline int dma_get_slave_caps(struct dma_chan *chan, } #endif +#define dma_request_slave_channel_reason(dev, name) dma_request_chan(dev, name) + static inline int dmaengine_desc_set_reuse(struct dma_async_tx_descriptor *tx) { struct dma_slave_caps caps; -- cgit v1.2.3