diff options
Diffstat (limited to 'drivers/dma')
28 files changed, 3292 insertions, 376 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 66cda7cc9f7a..ae6a682c9f76 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -376,29 +376,6 @@ config K3_DMA Support the DMA engine for Hisilicon K3 platform devices. -config LOONGSON1_APB_DMA - tristate "Loongson1 APB DMA support" - depends on MACH_LOONGSON32 || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - This selects support for the APB DMA controller in Loongson1 SoCs, - which is required by Loongson1 NAND and audio support. - -config LOONGSON2_APB_DMA - tristate "Loongson2 APB DMA support" - depends on LOONGARCH || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support for the Loongson2 APB DMA controller driver. The - DMA controller is having single DMA channel which can be - configured for different peripherals like audio, nand, sdio - etc which is in APB bus. - - This DMA controller transfers data from memory to peripheral fifo. - It does not support memory to memory data transfer. - config LPC18XX_DMAMUX bool "NXP LPC18xx/43xx DMA MUX for PL080" depends on ARCH_LPC18XX || COMPILE_TEST @@ -505,7 +482,7 @@ config MV_XOR_V2 platforms. config MXS_DMA - bool "MXS DMA support" + tristate "MXS DMA support" depends on ARCH_MXS || ARCH_MXC || COMPILE_TEST select STMP_DEVICE select DMA_ENGINE @@ -610,6 +587,15 @@ config SPRD_DMA help Enable support for the on-chip DMA controller on Spreadtrum platform. +config SWITCHTEC_DMA + tristate "Switchtec PSX/PFX Switch DMA Engine Support" + depends on PCI + select DMA_ENGINE + help + Some Switchtec PSX/PFX PCIe Switches support additional DMA engines. + These are exposed via an extra function on the switch's upstream + port. + config TXX9_DMAC tristate "Toshiba TXx9 SoC DMA support" depends on MACH_TX49XX @@ -774,6 +760,8 @@ source "drivers/dma/fsl-dpaa2-qdma/Kconfig" source "drivers/dma/lgm/Kconfig" +source "drivers/dma/loongson/Kconfig" + source "drivers/dma/stm32/Kconfig" # clients diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index a54d7688392b..14aa086629d5 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -49,8 +49,6 @@ obj-$(CONFIG_INTEL_IDMA64) += idma64.o obj-$(CONFIG_INTEL_IOATDMA) += ioat/ obj-y += idxd/ obj-$(CONFIG_K3_DMA) += k3dma.o -obj-$(CONFIG_LOONGSON1_APB_DMA) += loongson1-apb-dma.o -obj-$(CONFIG_LOONGSON2_APB_DMA) += loongson2-apb-dma.o obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o obj-$(CONFIG_LPC32XX_DMAMUX) += lpc32xx-dmamux.o obj-$(CONFIG_MILBEAUT_HDMAC) += milbeaut-hdmac.o @@ -74,6 +72,7 @@ obj-$(CONFIG_SF_PDMA) += sf-pdma/ obj-$(CONFIG_SOPHGO_CV1800B_DMAMUX) += cv1800b-dmamux.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o obj-$(CONFIG_SPRD_DMA) += sprd-dma.o +obj-$(CONFIG_SWITCHTEC_DMA) += switchtec_dma.o obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o obj-$(CONFIG_TEGRA186_GPC_DMA) += tegra186-gpc-dma.o obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o @@ -87,6 +86,7 @@ obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/ obj-$(CONFIG_INTEL_LDMA) += lgm/ obj-y += amd/ +obj-y += loongson/ obj-y += mediatek/ obj-y += qcom/ obj-y += stm32/ diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index eb65872c5d5c..45c2c8e4bc45 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -134,6 +134,7 @@ struct axi_dmac_desc { struct axi_dmac_chan *chan; bool cyclic; + bool cyclic_eot; bool have_partial_xfer; unsigned int num_submitted; @@ -162,6 +163,7 @@ struct axi_dmac_chan { bool hw_cyclic; bool hw_2d; bool hw_sg; + bool hw_cyclic_hotfix; }; struct axi_dmac { @@ -227,29 +229,94 @@ static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr) return true; } +static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) +{ + return list_first_entry_or_null(&chan->active_descs, + struct axi_dmac_desc, vdesc.node); +} + +static struct axi_dmac_desc *axi_dmac_get_next_desc(struct axi_dmac *dmac, + struct axi_dmac_chan *chan) +{ + struct axi_dmac_desc *active = axi_dmac_active_desc(chan); + struct virt_dma_desc *vdesc; + struct axi_dmac_desc *desc; + unsigned int val; + + /* + * Just play safe and ignore any SOF if we have an active cyclic transfer + * flagged to end. We'll start it as soon as the current cyclic one ends. + */ + if (active && active->cyclic_eot) + return NULL; + + /* + * It means a SW cyclic transfer is in place so we should just return + * the same descriptor. SW cyclic transfer termination is handled + * in axi_dmac_transfer_done(). + */ + if (chan->next_desc) + return chan->next_desc; + + vdesc = vchan_next_desc(&chan->vchan); + if (!vdesc) + return NULL; + + if (active && active->cyclic && !(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) { + struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev; + + dev_warn(dev, "Discarding non EOT transfer after cyclic\n"); + list_del(&vdesc->node); + return NULL; + } + + list_move_tail(&vdesc->node, &chan->active_descs); + desc = to_axi_dmac_desc(vdesc); + chan->next_desc = desc; + + if (!active || !active->cyclic) + return desc; + + active->cyclic_eot = true; + + if (chan->hw_sg) { + unsigned long flags = AXI_DMAC_HW_FLAG_IRQ | AXI_DMAC_HW_FLAG_LAST; + /* + * Let's then stop the current cyclic transfer by making sure we + * get an EOT interrupt and to open the cyclic loop by marking + * the last segment. + */ + active->sg[active->num_sgs - 1].hw->flags = flags; + return NULL; + } + + /* + * Clear the cyclic bit if there's no Scatter-Gather HW so that we get + * at the end of the transfer. + */ + val = axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS); + val &= ~AXI_DMAC_FLAG_CYCLIC; + axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, val); + + return NULL; +} + static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) { struct axi_dmac *dmac = chan_to_axi_dmac(chan); - struct virt_dma_desc *vdesc; struct axi_dmac_desc *desc; struct axi_dmac_sg *sg; unsigned int flags = 0; unsigned int val; + desc = axi_dmac_get_next_desc(dmac, chan); + if (!desc) + return; + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); if (val) /* Queue is full, wait for the next SOT IRQ */ return; - desc = chan->next_desc; - - if (!desc) { - vdesc = vchan_next_desc(&chan->vchan); - if (!vdesc) - return; - list_move_tail(&vdesc->node, &chan->active_descs); - desc = to_axi_dmac_desc(vdesc); - chan->next_desc = desc; - } sg = &desc->sg[desc->num_submitted]; /* Already queued in cyclic mode. Wait for it to finish */ @@ -291,10 +358,12 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) * call, enable hw cyclic mode to avoid unnecessary interrupts. */ if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) { - if (chan->hw_sg) + if (chan->hw_sg) { desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ; - else if (desc->num_sgs == 1) + } else if (desc->num_sgs == 1) { + chan->next_desc = NULL; flags |= AXI_DMAC_FLAG_CYCLIC; + } } if (chan->hw_partial_xfer) @@ -312,12 +381,6 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); } -static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) -{ - return list_first_entry_or_null(&chan->active_descs, - struct axi_dmac_desc, vdesc.node); -} - static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan, struct axi_dmac_sg *sg) { @@ -398,6 +461,61 @@ static void axi_dmac_compute_residue(struct axi_dmac_chan *chan, } } +static bool axi_dmac_handle_cyclic_eot(struct axi_dmac_chan *chan, + struct axi_dmac_desc *active) +{ + struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev; + struct virt_dma_desc *vdesc; + + /* wrap around */ + active->num_completed = 0; + + if (active->cyclic_eot) { + /* + * It means an HW cyclic transfer was marked to stop. And we + * know we have something to schedule, so start the next + * transfer now the cyclic one is done. + */ + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + + if (chan->hw_cyclic_hotfix) { + struct axi_dmac *dmac = chan_to_axi_dmac(chan); + /* + * In older IP cores, ending a cyclic transfer by clearing + * the CYCLIC flag does not guarantee a graceful end. + * It can happen that some data (of the next frame) is + * already prefetched and will be wrongly visible in the + * next transfer. To workaround this, we need to reenable + * the core so everything is flushed. Newer cores handles + * this correctly and do not require this "hotfix". The + * SG IP also does not require this. + */ + dev_dbg(dev, "HW cyclic hotfix\n"); + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0); + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); + } + + return true; + } + + vdesc = vchan_next_desc(&chan->vchan); + if (!vdesc) + return false; + if (!(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) { + dev_warn(dev, "Discarding non EOT transfer after cyclic\n"); + list_del(&vdesc->node); + return false; + } + + /* then let's end the cyclic transfer */ + chan->next_desc = NULL; + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + + return true; +} + static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, unsigned int completed_transfers) { @@ -416,6 +534,7 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, if (chan->hw_sg) { if (active->cyclic) { vchan_cyclic_callback(&active->vdesc); + start_next = axi_dmac_handle_cyclic_eot(chan, active); } else { list_del(&active->vdesc.node); vchan_cookie_complete(&active->vdesc); @@ -445,7 +564,8 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, if (active->num_completed == active->num_sgs || sg->partial_len) { if (active->cyclic) { - active->num_completed = 0; /* wrap around */ + /* keep start_next as is, if already true... */ + start_next |= axi_dmac_handle_cyclic_eot(chan, active); } else { list_del(&active->vdesc.node); vchan_cookie_complete(&active->vdesc); @@ -657,7 +777,12 @@ axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs, vecs[i].len, dsg); } - desc->cyclic = false; + desc->cyclic = flags & DMA_PREP_REPEAT; + if (desc->cyclic) { + /* Chain the last descriptor to the first, and remove its "last" flag */ + desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST; + desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys; + } return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); } @@ -1053,6 +1178,9 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) chan->length_align_mask = chan->address_align_mask; } + if (version < ADI_AXI_PCORE_VER(4, 6, 0) && !chan->hw_sg) + chan->hw_cyclic_hotfix = true; + return 0; } diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 27a8980b03dd..405bd2fbb4a3 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -31,29 +31,29 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/platform_device.h> -#include <linux/dma-mapping.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> +#include <linux/acpi.h> +#include <linux/acpi_dma.h> #include <linux/device.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/hardirq.h> -#include <linux/spinlock.h> +#include <linux/idr.h> +#include <linux/init.h> +#include <linux/jiffies.h> +#include <linux/mempool.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/numa.h> #include <linux/of.h> -#include <linux/property.h> +#include <linux/of_dma.h> #include <linux/percpu.h> -#include <linux/rcupdate.h> -#include <linux/mutex.h> -#include <linux/jiffies.h> +#include <linux/platform_device.h> +#include <linux/property.h> #include <linux/rculist.h> -#include <linux/idr.h> +#include <linux/rcupdate.h> #include <linux/slab.h> -#include <linux/acpi.h> -#include <linux/acpi_dma.h> -#include <linux/of_dma.h> -#include <linux/mempool.h> -#include <linux/numa.h> +#include <linux/spinlock.h> #include "dmaengine.h" @@ -765,7 +765,7 @@ struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, mutex_lock(&dma_list_mutex); list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { /* Finds a DMA controller with matching device node */ - if (np && device->dev->of_node && np != device->dev->of_node) + if (np && !device_match_of_node(device->dev, np)) continue; chan = find_candidate(device, mask, fn, fn_param); @@ -943,12 +943,14 @@ static void dmaenginem_release_channel(void *chan) struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name) { - struct dma_chan *chan = dma_request_chan(dev, name); - int ret = 0; + struct dma_chan *chan; + int ret; - if (!IS_ERR(chan)) - ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan); + chan = dma_request_chan(dev, name); + if (IS_ERR(chan)) + return chan; + ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan); if (ret) return ERR_PTR(ret); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index 5d74bc29cf89..4d53f077e9d2 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -50,6 +50,7 @@ #define AXI_DMA_FLAG_HAS_APB_REGS BIT(0) #define AXI_DMA_FLAG_HAS_RESETS BIT(1) #define AXI_DMA_FLAG_USE_CFG2 BIT(2) +#define AXI_DMA_FLAG_ARG0_AS_CHAN BIT(3) static inline void axi_dma_iowrite32(struct axi_dma_chip *chip, u32 reg, u32 val) @@ -342,8 +343,8 @@ static void axi_desc_put(struct axi_dma_desc *desc) kfree(desc); atomic_sub(descs_put, &chan->descs_allocated); dev_vdbg(chan2dev(chan), "%s: %d descs put, %d still allocated\n", - axi_chan_name(chan), descs_put, - atomic_read(&chan->descs_allocated)); + axi_chan_name(chan), descs_put, + atomic_read(&chan->descs_allocated)); } static void vchan_desc_put(struct virt_dma_desc *vdesc) @@ -353,7 +354,7 @@ static void vchan_desc_put(struct virt_dma_desc *vdesc) static enum dma_status dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, - struct dma_tx_state *txstate) + struct dma_tx_state *txstate) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); struct virt_dma_desc *vdesc; @@ -419,6 +420,7 @@ static void dw_axi_dma_set_byte_halfword(struct axi_dma_chan *chan, bool set) iowrite32(val, chan->chip->apb_regs + offset); } + /* Called in chan locked context */ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, struct axi_dma_desc *first) @@ -491,7 +493,7 @@ static void axi_chan_start_first_queued(struct axi_dma_chan *chan) desc = vd_to_axi_desc(vd); dev_vdbg(chan2dev(chan), "%s: started %u\n", axi_chan_name(chan), - vd->tx.cookie); + vd->tx.cookie); axi_chan_block_xfer_start(chan, desc); } @@ -592,8 +594,6 @@ static void dw_axi_dma_set_hw_channel(struct axi_dma_chan *chan, bool set) (chan->id * DMA_APB_HS_SEL_BIT_SIZE)); reg_value |= (val << (chan->id * DMA_APB_HS_SEL_BIT_SIZE)); lo_hi_writeq(reg_value, chip->apb_regs + DMAC_APB_HW_HS_SEL_0); - - return; } /* @@ -1162,7 +1162,7 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) axi_chan_irq_clear(chan, status); dev_vdbg(chip->dev, "%s %u IRQ status: 0x%08x\n", - axi_chan_name(chan), i, status); + axi_chan_name(chan), i, status); if (status & DWAXIDMAC_IRQ_ALL_ERR) axi_chan_handle_err(chan, status); @@ -1358,16 +1358,27 @@ static int __maybe_unused axi_dma_runtime_resume(struct device *dev) static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { + unsigned int handshake = dma_spec->args[0]; struct dw_axi_dma *dw = ofdma->of_dma_data; - struct axi_dma_chan *chan; + struct axi_dma_chan *chan = NULL; struct dma_chan *dchan; - dchan = dma_get_any_slave_channel(&dw->dma); + if (dw->hdata->use_handshake_as_channel_number) { + if (handshake >= dw->hdata->nr_channels) + return NULL; + + chan = &dw->chan[handshake]; + dchan = dma_get_slave_channel(&chan->vc.chan); + } else { + dchan = dma_get_any_slave_channel(&dw->dma); + } + if (!dchan) return NULL; - chan = dchan_to_axi_dma_chan(dchan); - chan->hw_handshake_num = dma_spec->args[0]; + if (!chan) + chan = dchan_to_axi_dma_chan(dchan); + chan->hw_handshake_num = handshake; return dchan; } @@ -1451,7 +1462,7 @@ static int axi_req_irqs(struct platform_device *pdev, struct axi_dma_chip *chip) if (chip->irq[i] < 0) return chip->irq[i]; ret = devm_request_irq(chip->dev, chip->irq[i], dw_axi_dma_interrupt, - IRQF_SHARED, KBUILD_MODNAME, chip); + IRQF_SHARED, KBUILD_MODNAME, chip); if (ret < 0) return ret; } @@ -1506,6 +1517,8 @@ static int dw_probe(struct platform_device *pdev) return ret; } + chip->dw->hdata->use_handshake_as_channel_number = !!(flags & AXI_DMA_FLAG_ARG0_AS_CHAN); + chip->dw->hdata->use_cfg2 = !!(flags & AXI_DMA_FLAG_USE_CFG2); chip->core_clk = devm_clk_get(chip->dev, "core-clk"); @@ -1645,7 +1658,7 @@ static void dw_remove(struct platform_device *pdev) of_dma_controller_free(chip->dev->of_node); list_for_each_entry_safe(chan, _chan, &dw->dma.channels, - vc.chan.device_node) { + vc.chan.device_node) { list_del(&chan->vc.chan.device_node); tasklet_kill(&chan->vc.task); } @@ -1662,6 +1675,9 @@ static const struct of_device_id dw_dma_of_id_table[] = { .compatible = "intel,kmb-axi-dma", .data = (void *)AXI_DMA_FLAG_HAS_APB_REGS, }, { + .compatible = "sophgo,cv1800b-axi-dma", + .data = (void *)AXI_DMA_FLAG_ARG0_AS_CHAN, + }, { .compatible = "starfive,jh7110-axi-dma", .data = (void *)(AXI_DMA_FLAG_HAS_RESETS | AXI_DMA_FLAG_USE_CFG2), }, { diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index b842e6a8d90d..67cc199e24d1 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -34,6 +34,7 @@ struct dw_axi_dma_hcfg { bool reg_map_8_channels; bool restrict_axi_burst_len; bool use_cfg2; + bool use_handshake_as_channel_number; }; struct axi_dma_chan { diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 5397dbda4f72..c2feb3adc79f 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -223,6 +223,43 @@ static int dw_edma_device_config(struct dma_chan *dchan, struct dma_slave_config *config) { struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + bool cfg_non_ll; + int non_ll = 0; + + chan->non_ll = false; + if (chan->dw->chip->mf == EDMA_MF_HDMA_NATIVE) { + if (config->peripheral_config && + config->peripheral_size != sizeof(int)) { + dev_err(dchan->device->dev, + "config param peripheral size mismatch\n"); + return -EINVAL; + } + + /* + * When there is no valid LLP base address available then the + * default DMA ops will use the non-LL mode. + * + * Cases where LL mode is enabled and client wants to use the + * non-LL mode then also client can do so via providing the + * peripheral_config param. + */ + cfg_non_ll = chan->dw->chip->cfg_non_ll; + if (config->peripheral_config) { + non_ll = *(int *)config->peripheral_config; + + if (cfg_non_ll && !non_ll) { + dev_err(dchan->device->dev, "invalid configuration\n"); + return -EINVAL; + } + } + + if (cfg_non_ll || non_ll) + chan->non_ll = true; + } else if (config->peripheral_config) { + dev_err(dchan->device->dev, + "peripheral config param applicable only for HDMA\n"); + return -EINVAL; + } memcpy(&chan->config, config, sizeof(*config)); chan->configured = true; @@ -358,6 +395,7 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) struct dw_edma_desc *desc; u64 src_addr, dst_addr; size_t fsz = 0; + u32 bursts_max; u32 cnt = 0; int i; @@ -415,6 +453,13 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) return NULL; } + /* + * For non-LL mode, only a single burst can be handled + * in a single chunk unlike LL mode where multiple bursts + * can be configured in a single chunk. + */ + bursts_max = chan->non_ll ? 1 : chan->ll_max; + desc = dw_edma_alloc_desc(chan); if (unlikely(!desc)) goto err_alloc; @@ -450,7 +495,7 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) break; - if (chunk->bursts_alloc == chan->ll_max) { + if (chunk->bursts_alloc == bursts_max) { chunk = dw_edma_alloc_chunk(desc); if (unlikely(!chunk)) goto err_alloc; @@ -663,7 +708,96 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) chan->status = EDMA_ST_IDLE; } -static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) +static void dw_edma_emul_irq_ack(struct irq_data *d) +{ + struct dw_edma *dw = irq_data_get_irq_chip_data(d); + + dw_edma_core_ack_emulated_irq(dw); +} + +/* + * irq_chip implementation for interrupt-emulation doorbells. + * + * The emulated source has no mask/unmask mechanism. With handle_level_irq(), + * the flow is therefore: + * 1) .irq_ack() deasserts the source + * 2) registered handlers (if any) are dispatched + * Since deassertion is already done in .irq_ack(), handlers do not need to take + * care of it, hence IRQCHIP_ONESHOT_SAFE. + */ +static struct irq_chip dw_edma_emul_irqchip = { + .name = "dw-edma-emul", + .irq_ack = dw_edma_emul_irq_ack, + .flags = IRQCHIP_ONESHOT_SAFE | IRQCHIP_SKIP_SET_WAKE, +}; + +static int dw_edma_emul_irq_alloc(struct dw_edma *dw) +{ + struct dw_edma_chip *chip = dw->chip; + int virq; + + chip->db_irq = 0; + chip->db_offset = ~0; + + /* + * Only meaningful when the core provides the deassert sequence + * for interrupt emulation. + */ + if (!dw->core->ack_emulated_irq) + return 0; + + /* + * Allocate a single, requestable Linux virtual IRQ number. + * Use >= 1 so that 0 can remain a "not available" sentinel. + */ + virq = irq_alloc_desc(NUMA_NO_NODE); + if (virq < 0) + return virq; + + irq_set_chip_and_handler(virq, &dw_edma_emul_irqchip, handle_level_irq); + irq_set_chip_data(virq, dw); + irq_set_noprobe(virq); + + chip->db_irq = virq; + chip->db_offset = dw_edma_core_db_offset(dw); + + return 0; +} + +static void dw_edma_emul_irq_free(struct dw_edma *dw) +{ + struct dw_edma_chip *chip = dw->chip; + + if (!chip) + return; + if (chip->db_irq <= 0) + return; + + irq_free_descs(chip->db_irq, 1); + chip->db_irq = 0; + chip->db_offset = ~0; +} + +static inline irqreturn_t dw_edma_interrupt_emulated(void *data) +{ + struct dw_edma_irq *dw_irq = data; + struct dw_edma *dw = dw_irq->dw; + int db_irq = dw->chip->db_irq; + + if (db_irq > 0) { + /* + * Interrupt emulation may assert the IRQ line without updating the + * normal DONE/ABORT status bits. With a shared IRQ handler we + * cannot reliably detect such events by status registers alone, so + * always perform the core-specific deassert sequence. + */ + generic_handle_irq(db_irq); + return IRQ_HANDLED; + } + return IRQ_NONE; +} + +static inline irqreturn_t dw_edma_interrupt_write_inner(int irq, void *data) { struct dw_edma_irq *dw_irq = data; @@ -672,7 +806,7 @@ static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) dw_edma_abort_interrupt); } -static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) +static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data) { struct dw_edma_irq *dw_irq = data; @@ -681,12 +815,33 @@ static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) dw_edma_abort_interrupt); } -static irqreturn_t dw_edma_interrupt_common(int irq, void *data) +static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) +{ + irqreturn_t ret = IRQ_NONE; + + ret |= dw_edma_interrupt_write_inner(irq, data); + ret |= dw_edma_interrupt_emulated(data); + + return ret; +} + +static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) { irqreturn_t ret = IRQ_NONE; - ret |= dw_edma_interrupt_write(irq, data); - ret |= dw_edma_interrupt_read(irq, data); + ret |= dw_edma_interrupt_read_inner(irq, data); + ret |= dw_edma_interrupt_emulated(data); + + return ret; +} + +static inline irqreturn_t dw_edma_interrupt_common(int irq, void *data) +{ + irqreturn_t ret = IRQ_NONE; + + ret |= dw_edma_interrupt_write_inner(irq, data); + ret |= dw_edma_interrupt_read_inner(irq, data); + ret |= dw_edma_interrupt_emulated(data); return ret; } @@ -977,6 +1132,11 @@ int dw_edma_probe(struct dw_edma_chip *chip) if (err) return err; + /* Allocate a dedicated virtual IRQ for interrupt-emulation doorbells */ + err = dw_edma_emul_irq_alloc(dw); + if (err) + dev_warn(dev, "Failed to allocate emulation IRQ: %d\n", err); + /* Setup write/read channels */ err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc); if (err) @@ -992,6 +1152,7 @@ int dw_edma_probe(struct dw_edma_chip *chip) err_irq_free: for (i = (dw->nr_irqs - 1); i >= 0; i--) free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); + dw_edma_emul_irq_free(dw); return err; } @@ -1014,6 +1175,7 @@ int dw_edma_remove(struct dw_edma_chip *chip) /* Free irqs */ for (i = (dw->nr_irqs - 1); i >= 0; i--) free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); + dw_edma_emul_irq_free(dw); /* Deregister eDMA device */ dma_async_device_unregister(&dw->dma); diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index 71894b9e0b15..902574b1ba86 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -86,6 +86,7 @@ struct dw_edma_chan { u8 configured; struct dma_slave_config config; + bool non_ll; }; struct dw_edma_irq { @@ -126,6 +127,8 @@ struct dw_edma_core_ops { void (*start)(struct dw_edma_chunk *chunk, bool first); void (*ch_config)(struct dw_edma_chan *chan); void (*debugfs_on)(struct dw_edma *dw); + void (*ack_emulated_irq)(struct dw_edma *dw); + resource_size_t (*db_offset)(struct dw_edma *dw); }; struct dw_edma_sg { @@ -206,4 +209,19 @@ void dw_edma_core_debugfs_on(struct dw_edma *dw) dw->core->debugfs_on(dw); } +static inline int dw_edma_core_ack_emulated_irq(struct dw_edma *dw) +{ + if (!dw->core->ack_emulated_irq) + return -EOPNOTSUPP; + + dw->core->ack_emulated_irq(dw); + return 0; +} + +static inline resource_size_t +dw_edma_core_db_offset(struct dw_edma *dw) +{ + return dw->core->db_offset(dw); +} + #endif /* _DW_EDMA_CORE_H */ diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 83230acaa597..0b30ce138503 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -14,14 +14,35 @@ #include <linux/pci-epf.h> #include <linux/msi.h> #include <linux/bitfield.h> +#include <linux/sizes.h> #include "dw-edma-core.h" -#define DW_PCIE_VSEC_DMA_ID 0x6 -#define DW_PCIE_VSEC_DMA_BAR GENMASK(10, 8) -#define DW_PCIE_VSEC_DMA_MAP GENMASK(2, 0) -#define DW_PCIE_VSEC_DMA_WR_CH GENMASK(9, 0) -#define DW_PCIE_VSEC_DMA_RD_CH GENMASK(25, 16) +/* Synopsys */ +#define DW_PCIE_SYNOPSYS_VSEC_DMA_ID 0x6 +#define DW_PCIE_SYNOPSYS_VSEC_DMA_BAR GENMASK(10, 8) +#define DW_PCIE_SYNOPSYS_VSEC_DMA_MAP GENMASK(2, 0) +#define DW_PCIE_SYNOPSYS_VSEC_DMA_WR_CH GENMASK(9, 0) +#define DW_PCIE_SYNOPSYS_VSEC_DMA_RD_CH GENMASK(25, 16) + +/* AMD MDB (Xilinx) specific defines */ +#define PCI_DEVICE_ID_XILINX_B054 0xb054 + +#define DW_PCIE_XILINX_MDB_VSEC_DMA_ID 0x6 +#define DW_PCIE_XILINX_MDB_VSEC_ID 0x20 +#define DW_PCIE_XILINX_MDB_VSEC_DMA_BAR GENMASK(10, 8) +#define DW_PCIE_XILINX_MDB_VSEC_DMA_MAP GENMASK(2, 0) +#define DW_PCIE_XILINX_MDB_VSEC_DMA_WR_CH GENMASK(9, 0) +#define DW_PCIE_XILINX_MDB_VSEC_DMA_RD_CH GENMASK(25, 16) + +#define DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_HIGH 0xc +#define DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_LOW 0x8 +#define DW_PCIE_XILINX_MDB_INVALID_ADDR (~0ULL) + +#define DW_PCIE_XILINX_MDB_LL_OFF_GAP 0x200000 +#define DW_PCIE_XILINX_MDB_LL_SIZE 0x800 +#define DW_PCIE_XILINX_MDB_DT_OFF_GAP 0x100000 +#define DW_PCIE_XILINX_MDB_DT_SIZE 0x800 #define DW_BLOCK(a, b, c) \ { \ @@ -50,6 +71,7 @@ struct dw_edma_pcie_data { u8 irqs; u16 wr_ch_cnt; u16 rd_ch_cnt; + u64 devmem_phys_off; }; static const struct dw_edma_pcie_data snps_edda_data = { @@ -90,6 +112,64 @@ static const struct dw_edma_pcie_data snps_edda_data = { .rd_ch_cnt = 2, }; +static const struct dw_edma_pcie_data xilinx_mdb_data = { + /* MDB registers location */ + .rg.bar = BAR_0, + .rg.off = SZ_4K, /* 4 Kbytes */ + .rg.sz = SZ_8K, /* 8 Kbytes */ + + /* Other */ + .mf = EDMA_MF_HDMA_NATIVE, + .irqs = 1, + .wr_ch_cnt = 8, + .rd_ch_cnt = 8, +}; + +static void dw_edma_set_chan_region_offset(struct dw_edma_pcie_data *pdata, + enum pci_barno bar, off_t start_off, + off_t ll_off_gap, size_t ll_size, + off_t dt_off_gap, size_t dt_size) +{ + u16 wr_ch = pdata->wr_ch_cnt; + u16 rd_ch = pdata->rd_ch_cnt; + off_t off; + u16 i; + + off = start_off; + + /* Write channel LL region */ + for (i = 0; i < wr_ch; i++) { + pdata->ll_wr[i].bar = bar; + pdata->ll_wr[i].off = off; + pdata->ll_wr[i].sz = ll_size; + off += ll_off_gap; + } + + /* Read channel LL region */ + for (i = 0; i < rd_ch; i++) { + pdata->ll_rd[i].bar = bar; + pdata->ll_rd[i].off = off; + pdata->ll_rd[i].sz = ll_size; + off += ll_off_gap; + } + + /* Write channel data region */ + for (i = 0; i < wr_ch; i++) { + pdata->dt_wr[i].bar = bar; + pdata->dt_wr[i].off = off; + pdata->dt_wr[i].sz = dt_size; + off += dt_off_gap; + } + + /* Read channel data region */ + for (i = 0; i < rd_ch; i++) { + pdata->dt_rd[i].bar = bar; + pdata->dt_rd[i].off = off; + pdata->dt_rd[i].sz = dt_size; + off += dt_off_gap; + } +} + static int dw_edma_pcie_irq_vector(struct device *dev, unsigned int nr) { return pci_irq_vector(to_pci_dev(dev), nr); @@ -114,15 +194,15 @@ static const struct dw_edma_plat_ops dw_edma_pcie_plat_ops = { .pci_address = dw_edma_pcie_address, }; -static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, - struct dw_edma_pcie_data *pdata) +static void dw_edma_pcie_get_synopsys_dma_data(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata) { u32 val, map; u16 vsec; u64 off; vsec = pci_find_vsec_capability(pdev, PCI_VENDOR_ID_SYNOPSYS, - DW_PCIE_VSEC_DMA_ID); + DW_PCIE_SYNOPSYS_VSEC_DMA_ID); if (!vsec) return; @@ -131,9 +211,9 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, PCI_VNDR_HEADER_LEN(val) != 0x18) return; - pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability DMA\n"); + pci_dbg(pdev, "Detected Synopsys PCIe Vendor-Specific Extended Capability DMA\n"); pci_read_config_dword(pdev, vsec + 0x8, &val); - map = FIELD_GET(DW_PCIE_VSEC_DMA_MAP, val); + map = FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_MAP, val); if (map != EDMA_MF_EDMA_LEGACY && map != EDMA_MF_EDMA_UNROLL && map != EDMA_MF_HDMA_COMPAT && @@ -141,13 +221,55 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, return; pdata->mf = map; - pdata->rg.bar = FIELD_GET(DW_PCIE_VSEC_DMA_BAR, val); + pdata->rg.bar = FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_BAR, val); pci_read_config_dword(pdev, vsec + 0xc, &val); pdata->wr_ch_cnt = min_t(u16, pdata->wr_ch_cnt, - FIELD_GET(DW_PCIE_VSEC_DMA_WR_CH, val)); + FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_WR_CH, val)); pdata->rd_ch_cnt = min_t(u16, pdata->rd_ch_cnt, - FIELD_GET(DW_PCIE_VSEC_DMA_RD_CH, val)); + FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_RD_CH, val)); + + pci_read_config_dword(pdev, vsec + 0x14, &val); + off = val; + pci_read_config_dword(pdev, vsec + 0x10, &val); + off <<= 32; + off |= val; + pdata->rg.off = off; +} + +static void dw_edma_pcie_get_xilinx_dma_data(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata) +{ + u32 val, map; + u16 vsec; + u64 off; + + pdata->devmem_phys_off = DW_PCIE_XILINX_MDB_INVALID_ADDR; + + vsec = pci_find_vsec_capability(pdev, PCI_VENDOR_ID_XILINX, + DW_PCIE_XILINX_MDB_VSEC_DMA_ID); + if (!vsec) + return; + + pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, &val); + if (PCI_VNDR_HEADER_REV(val) != 0x00 || + PCI_VNDR_HEADER_LEN(val) != 0x18) + return; + + pci_dbg(pdev, "Detected Xilinx PCIe Vendor-Specific Extended Capability DMA\n"); + pci_read_config_dword(pdev, vsec + 0x8, &val); + map = FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_MAP, val); + if (map != EDMA_MF_HDMA_NATIVE) + return; + + pdata->mf = map; + pdata->rg.bar = FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_BAR, val); + + pci_read_config_dword(pdev, vsec + 0xc, &val); + pdata->wr_ch_cnt = min(pdata->wr_ch_cnt, + FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_WR_CH, val)); + pdata->rd_ch_cnt = min(pdata->rd_ch_cnt, + FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_RD_CH, val)); pci_read_config_dword(pdev, vsec + 0x14, &val); off = val; @@ -155,6 +277,31 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, off <<= 32; off |= val; pdata->rg.off = off; + + vsec = pci_find_vsec_capability(pdev, PCI_VENDOR_ID_XILINX, + DW_PCIE_XILINX_MDB_VSEC_ID); + if (!vsec) + return; + + pci_read_config_dword(pdev, + vsec + DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_HIGH, + &val); + off = val; + pci_read_config_dword(pdev, + vsec + DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_LOW, + &val); + off <<= 32; + off |= val; + pdata->devmem_phys_off = off; +} + +static u64 dw_edma_get_phys_addr(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata, + enum pci_barno bar) +{ + if (pdev->vendor == PCI_VENDOR_ID_XILINX) + return pdata->devmem_phys_off; + return pci_bus_address(pdev, bar); } static int dw_edma_pcie_probe(struct pci_dev *pdev, @@ -165,6 +312,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, struct dw_edma_chip *chip; int err, nr_irqs; int i, mask; + bool non_ll = false; struct dw_edma_pcie_data *vsec_data __free(kfree) = kmalloc_obj(*vsec_data); @@ -184,7 +332,32 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, * Tries to find if exists a PCIe Vendor-Specific Extended Capability * for the DMA, if one exists, then reconfigures it. */ - dw_edma_pcie_get_vsec_dma_data(pdev, vsec_data); + dw_edma_pcie_get_synopsys_dma_data(pdev, vsec_data); + + if (pdev->vendor == PCI_VENDOR_ID_XILINX) { + dw_edma_pcie_get_xilinx_dma_data(pdev, vsec_data); + + /* + * There is no valid address found for the LL memory + * space on the device side. In the absence of LL base + * address use the non-LL mode or simple mode supported by + * the HDMA IP. + */ + if (vsec_data->devmem_phys_off == DW_PCIE_XILINX_MDB_INVALID_ADDR) + non_ll = true; + + /* + * Configure the channel LL and data blocks if number of + * channels enabled in VSEC capability are more than the + * channels configured in xilinx_mdb_data. + */ + if (!non_ll) + dw_edma_set_chan_region_offset(vsec_data, BAR_2, 0, + DW_PCIE_XILINX_MDB_LL_OFF_GAP, + DW_PCIE_XILINX_MDB_LL_SIZE, + DW_PCIE_XILINX_MDB_DT_OFF_GAP, + DW_PCIE_XILINX_MDB_DT_SIZE); + } /* Mapping PCI BAR regions */ mask = BIT(vsec_data->rg.bar); @@ -231,6 +404,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->mf = vsec_data->mf; chip->nr_irqs = nr_irqs; chip->ops = &dw_edma_pcie_plat_ops; + chip->cfg_non_ll = non_ll; chip->ll_wr_cnt = vsec_data->wr_ch_cnt; chip->ll_rd_cnt = vsec_data->rd_ch_cnt; @@ -239,7 +413,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, if (!chip->reg_base) return -ENOMEM; - for (i = 0; i < chip->ll_wr_cnt; i++) { + for (i = 0; i < chip->ll_wr_cnt && !non_ll; i++) { struct dw_edma_region *ll_region = &chip->ll_region_wr[i]; struct dw_edma_region *dt_region = &chip->dt_region_wr[i]; struct dw_edma_block *ll_block = &vsec_data->ll_wr[i]; @@ -250,7 +424,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; ll_region->vaddr.io += ll_block->off; - ll_region->paddr = pci_bus_address(pdev, ll_block->bar); + ll_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data, + ll_block->bar); ll_region->paddr += ll_block->off; ll_region->sz = ll_block->sz; @@ -259,12 +434,13 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; dt_region->vaddr.io += dt_block->off; - dt_region->paddr = pci_bus_address(pdev, dt_block->bar); + dt_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data, + dt_block->bar); dt_region->paddr += dt_block->off; dt_region->sz = dt_block->sz; } - for (i = 0; i < chip->ll_rd_cnt; i++) { + for (i = 0; i < chip->ll_rd_cnt && !non_ll; i++) { struct dw_edma_region *ll_region = &chip->ll_region_rd[i]; struct dw_edma_region *dt_region = &chip->dt_region_rd[i]; struct dw_edma_block *ll_block = &vsec_data->ll_rd[i]; @@ -275,7 +451,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; ll_region->vaddr.io += ll_block->off; - ll_region->paddr = pci_bus_address(pdev, ll_block->bar); + ll_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data, + ll_block->bar); ll_region->paddr += ll_block->off; ll_region->sz = ll_block->sz; @@ -284,7 +461,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; dt_region->vaddr.io += dt_block->off; - dt_region->paddr = pci_bus_address(pdev, dt_block->bar); + dt_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data, + dt_block->bar); dt_region->paddr += dt_block->off; dt_region->sz = dt_block->sz; } @@ -367,6 +545,8 @@ static void dw_edma_pcie_remove(struct pci_dev *pdev) static const struct pci_device_id dw_edma_pcie_id_table[] = { { PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_data) }, + { PCI_VDEVICE(XILINX, PCI_DEVICE_ID_XILINX_B054), + (kernel_ulong_t)&xilinx_mdb_data }, { } }; MODULE_DEVICE_TABLE(pci, dw_edma_pcie_id_table); diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index b75fdaffad9a..69e8279adec8 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -509,6 +509,25 @@ static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw) dw_edma_v0_debugfs_on(dw); } +static void dw_edma_v0_core_ack_emulated_irq(struct dw_edma *dw) +{ + /* + * Interrupt emulation may assert the IRQ without setting + * DONE/ABORT status bits. A zero write to INT_CLEAR deasserts the + * emulated IRQ, while being a no-op for real interrupts. + */ + SET_BOTH_32(dw, int_clear, 0); +} + +static resource_size_t dw_edma_v0_core_db_offset(struct dw_edma *dw) +{ + /* + * rd_int_status is chosen arbitrarily, but wr_int_status would be + * equally suitable. + */ + return offsetof(struct dw_edma_v0_regs, rd_int_status); +} + static const struct dw_edma_core_ops dw_edma_v0_core = { .off = dw_edma_v0_core_off, .ch_count = dw_edma_v0_core_ch_count, @@ -517,6 +536,8 @@ static const struct dw_edma_core_ops dw_edma_v0_core = { .start = dw_edma_v0_core_start, .ch_config = dw_edma_v0_core_ch_config, .debugfs_on = dw_edma_v0_core_debugfs_on, + .ack_emulated_irq = dw_edma_v0_core_ack_emulated_irq, + .db_offset = dw_edma_v0_core_db_offset, }; void dw_edma_v0_core_register(struct dw_edma *dw) diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index ce8f7254bab2..632abb8b481c 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -225,7 +225,7 @@ static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk) readl(chunk->ll_region.vaddr.io); } -static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) +static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first) { struct dw_edma_chan *chan = chunk->chan; struct dw_edma *dw = chan->dw; @@ -263,6 +263,68 @@ static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); } +static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chunk *chunk) +{ + struct dw_edma_chan *chan = chunk->chan; + struct dw_edma *dw = chan->dw; + struct dw_edma_burst *child; + u32 val; + + child = list_first_entry_or_null(&chunk->burst->list, + struct dw_edma_burst, list); + if (!child) + return; + + SET_CH_32(dw, chan->dir, chan->id, ch_en, HDMA_V0_CH_EN); + + /* Source address */ + SET_CH_32(dw, chan->dir, chan->id, sar.lsb, + lower_32_bits(child->sar)); + SET_CH_32(dw, chan->dir, chan->id, sar.msb, + upper_32_bits(child->sar)); + + /* Destination address */ + SET_CH_32(dw, chan->dir, chan->id, dar.lsb, + lower_32_bits(child->dar)); + SET_CH_32(dw, chan->dir, chan->id, dar.msb, + upper_32_bits(child->dar)); + + /* Transfer size */ + SET_CH_32(dw, chan->dir, chan->id, transfer_size, child->sz); + + /* Interrupt setup */ + val = GET_CH_32(dw, chan->dir, chan->id, int_setup) | + HDMA_V0_STOP_INT_MASK | + HDMA_V0_ABORT_INT_MASK | + HDMA_V0_LOCAL_STOP_INT_EN | + HDMA_V0_LOCAL_ABORT_INT_EN; + + if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) { + val |= HDMA_V0_REMOTE_STOP_INT_EN | + HDMA_V0_REMOTE_ABORT_INT_EN; + } + + SET_CH_32(dw, chan->dir, chan->id, int_setup, val); + + /* Channel control setup */ + val = GET_CH_32(dw, chan->dir, chan->id, control1); + val &= ~HDMA_V0_LINKLIST_EN; + SET_CH_32(dw, chan->dir, chan->id, control1, val); + + SET_CH_32(dw, chan->dir, chan->id, doorbell, + HDMA_V0_DOORBELL_START); +} + +static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) +{ + struct dw_edma_chan *chan = chunk->chan; + + if (chan->non_ll) + dw_hdma_v0_core_non_ll_start(chunk); + else + dw_hdma_v0_core_ll_start(chunk, first); +} + static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan) { struct dw_edma *dw = chan->dw; @@ -283,6 +345,12 @@ static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw) dw_hdma_v0_debugfs_on(dw); } +static resource_size_t dw_hdma_v0_core_db_offset(struct dw_edma *dw) +{ + /* Implement once the correct offset is known. */ + return ~0; +} + static const struct dw_edma_core_ops dw_hdma_v0_core = { .off = dw_hdma_v0_core_off, .ch_count = dw_hdma_v0_core_ch_count, @@ -291,6 +359,7 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = { .start = dw_hdma_v0_core_start, .ch_config = dw_hdma_v0_core_ch_config, .debugfs_on = dw_hdma_v0_core_debugfs_on, + .db_offset = dw_hdma_v0_core_db_offset, }; void dw_hdma_v0_core_register(struct dw_edma *dw) diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h index eab5fd7177e5..7759ba9b4850 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h +++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h @@ -12,6 +12,7 @@ #include <linux/dmaengine.h> #define HDMA_V0_MAX_NR_CH 8 +#define HDMA_V0_CH_EN BIT(0) #define HDMA_V0_LOCAL_ABORT_INT_EN BIT(6) #define HDMA_V0_REMOTE_ABORT_INT_EN BIT(5) #define HDMA_V0_LOCAL_STOP_INT_EN BIT(4) diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index 6a38738e56e2..bb7531c456df 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -905,8 +905,7 @@ void fsl_edma_free_chan_resources(struct dma_chan *chan) fsl_chan->is_sw = false; fsl_chan->srcid = 0; fsl_chan->is_remote = false; - if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_CHCLK) - clk_disable_unprepare(fsl_chan->clk); + clk_disable_unprepare(fsl_chan->clk); } void fsl_edma_cleanup_vchan(struct dma_device *dmadev) diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index b596baa0a182..36155ab1602a 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -705,16 +705,14 @@ static int fsl_edma_probe(struct platform_device *pdev) int ret, i; drvdata = device_get_match_data(&pdev->dev); - if (!drvdata) { - dev_err(&pdev->dev, "unable to find driver data\n"); - return -EINVAL; - } + if (!drvdata) + return dev_err_probe(&pdev->dev, -EINVAL, + "unable to find driver data\n"); ret = of_property_read_u32(np, "dma-channels", &chans); - if (ret) { - dev_err(&pdev->dev, "Can't get dma-channels.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't get dma-channels.\n"); fsl_edma = devm_kzalloc(&pdev->dev, struct_size(fsl_edma, chans, chans), GFP_KERNEL); @@ -738,10 +736,10 @@ static int fsl_edma_probe(struct platform_device *pdev) if (drvdata->flags & FSL_EDMA_DRV_HAS_DMACLK) { fsl_edma->dmaclk = devm_clk_get_enabled(&pdev->dev, "dma"); - if (IS_ERR(fsl_edma->dmaclk)) { - dev_err(&pdev->dev, "Missing DMA block clock.\n"); - return PTR_ERR(fsl_edma->dmaclk); - } + if (IS_ERR(fsl_edma->dmaclk)) + return dev_err_probe(&pdev->dev, + PTR_ERR(fsl_edma->dmaclk), + "Missing DMA block clock.\n"); } ret = of_property_read_variable_u32_array(np, "dma-channel-mask", chan_mask, 1, 2); @@ -765,11 +763,10 @@ static int fsl_edma_probe(struct platform_device *pdev) sprintf(clkname, "dmamux%d", i); fsl_edma->muxclk[i] = devm_clk_get_enabled(&pdev->dev, clkname); - if (IS_ERR(fsl_edma->muxclk[i])) { - dev_err(&pdev->dev, "Missing DMAMUX block clock.\n"); - /* on error: disable all previously enabled clks */ - return PTR_ERR(fsl_edma->muxclk[i]); - } + if (IS_ERR(fsl_edma->muxclk[i])) + return dev_err_probe(&pdev->dev, + PTR_ERR(fsl_edma->muxclk[i]), + "Missing DMAMUX block clock.\n"); } fsl_edma->big_endian = of_property_read_bool(np, "big-endian"); @@ -878,22 +875,17 @@ static int fsl_edma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, fsl_edma); - ret = dma_async_device_register(&fsl_edma->dma_dev); - if (ret) { - dev_err(&pdev->dev, - "Can't register Freescale eDMA engine. (%d)\n", ret); - return ret; - } + ret = dmaenginem_async_device_register(&fsl_edma->dma_dev); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register Freescale eDMA engine.\n"); - ret = of_dma_controller_register(np, + ret = devm_of_dma_controller_register(&pdev->dev, np, drvdata->dmamuxs ? fsl_edma_xlate : fsl_edma3_xlate, fsl_edma); - if (ret) { - dev_err(&pdev->dev, - "Can't register Freescale eDMA of_dma. (%d)\n", ret); - dma_async_device_unregister(&fsl_edma->dma_dev); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register Freescale eDMA of_dma.\n"); /* enable round robin arbitration */ if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) @@ -904,12 +896,9 @@ static int fsl_edma_probe(struct platform_device *pdev) static void fsl_edma_remove(struct platform_device *pdev) { - struct device_node *np = pdev->dev.of_node; struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev); fsl_edma_irq_exit(pdev, fsl_edma); - of_dma_controller_free(np); - dma_async_device_unregister(&fsl_edma->dma_dev); fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); } diff --git a/drivers/dma/fsl-qdma.c b/drivers/dma/fsl-qdma.c index 0bbff9df362f..df843fad0ece 100644 --- a/drivers/dma/fsl-qdma.c +++ b/drivers/dma/fsl-qdma.c @@ -1127,22 +1127,19 @@ static int fsl_qdma_probe(struct platform_device *pdev) struct device_node *np = pdev->dev.of_node; ret = of_property_read_u32(np, "dma-channels", &chans); - if (ret) { - dev_err(&pdev->dev, "Can't get dma-channels.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't get dma-channels.\n"); ret = of_property_read_u32(np, "block-offset", &blk_off); - if (ret) { - dev_err(&pdev->dev, "Can't get block-offset.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't get block-offset.\n"); ret = of_property_read_u32(np, "block-number", &blk_num); - if (ret) { - dev_err(&pdev->dev, "Can't get block-number.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't get block-number.\n"); blk_num = min_t(int, blk_num, num_online_cpus()); @@ -1167,10 +1164,8 @@ static int fsl_qdma_probe(struct platform_device *pdev) return -ENOMEM; ret = of_property_read_u32(np, "fsl,dma-queues", &queues); - if (ret) { - dev_err(&pdev->dev, "Can't get queues.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "Can't get queues.\n"); fsl_qdma->desc_allocated = 0; fsl_qdma->n_chans = chans; @@ -1231,28 +1226,24 @@ static int fsl_qdma_probe(struct platform_device *pdev) fsl_qdma->dma_dev.device_terminate_all = fsl_qdma_terminate_all; ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(40)); - if (ret) { - dev_err(&pdev->dev, "dma_set_mask failure.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "dma_set_mask failure.\n"); platform_set_drvdata(pdev, fsl_qdma); ret = fsl_qdma_reg_init(fsl_qdma); - if (ret) { - dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't Initialize the qDMA engine.\n"); ret = fsl_qdma_irq_init(pdev, fsl_qdma); if (ret) return ret; ret = dma_async_device_register(&fsl_qdma->dma_dev); - if (ret) { - dev_err(&pdev->dev, "Can't register NXP Layerscape qDMA engine.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register NXP Layerscape qDMA engine.\n"); return 0; } diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 4c8196d78001..3d527883776b 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -2265,34 +2265,24 @@ static int sdma_probe(struct platform_device *pdev) if (IS_ERR(sdma->regs)) return PTR_ERR(sdma->regs); - sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); + sdma->clk_ipg = devm_clk_get_prepared(&pdev->dev, "ipg"); if (IS_ERR(sdma->clk_ipg)) return PTR_ERR(sdma->clk_ipg); - sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb"); + sdma->clk_ahb = devm_clk_get_prepared(&pdev->dev, "ahb"); if (IS_ERR(sdma->clk_ahb)) return PTR_ERR(sdma->clk_ahb); - ret = clk_prepare(sdma->clk_ipg); - if (ret) - return ret; - - ret = clk_prepare(sdma->clk_ahb); - if (ret) - goto err_clk; - ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, dev_name(&pdev->dev), sdma); if (ret) - goto err_irq; + return ret; sdma->irq = irq; sdma->script_addrs = kzalloc_obj(*sdma->script_addrs); - if (!sdma->script_addrs) { - ret = -ENOMEM; - goto err_irq; - } + if (!sdma->script_addrs) + return -ENOMEM; /* initially no scripts available */ saddr_arr = (s32 *)sdma->script_addrs; @@ -2333,11 +2323,11 @@ static int sdma_probe(struct platform_device *pdev) ret = sdma_init(sdma); if (ret) - goto err_init; + return ret; ret = sdma_event_remap(sdma); if (ret) - goto err_init; + return ret; if (sdma->drvdata->script_addrs) sdma_add_scripts(sdma, sdma->drvdata->script_addrs); @@ -2363,18 +2353,16 @@ static int sdma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, sdma); - ret = dma_async_device_register(&sdma->dma_device); - if (ret) { - dev_err(&pdev->dev, "unable to register\n"); - goto err_init; - } + ret = dmaenginem_async_device_register(&sdma->dma_device); + if (ret) + return dev_err_probe(&pdev->dev, ret, "unable to register\n"); if (np) { - ret = of_dma_controller_register(np, sdma_xlate, sdma); - if (ret) { - dev_err(&pdev->dev, "failed to register controller\n"); - goto err_register; - } + ret = devm_of_dma_controller_register(&pdev->dev, np, + sdma_xlate, sdma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to register controller\n"); spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus"); ret = of_address_to_resource(spba_bus, 0, &spba_res); @@ -2401,16 +2389,6 @@ static int sdma_probe(struct platform_device *pdev) } return 0; - -err_register: - dma_async_device_unregister(&sdma->dma_device); -err_init: - kfree(sdma->script_addrs); -err_irq: - clk_unprepare(sdma->clk_ahb); -err_clk: - clk_unprepare(sdma->clk_ipg); - return ret; } static void sdma_remove(struct platform_device *pdev) @@ -2419,10 +2397,6 @@ static void sdma_remove(struct platform_device *pdev) int i; devm_free_irq(&pdev->dev, sdma->irq, sdma); - dma_async_device_unregister(&sdma->dma_device); - kfree(sdma->script_addrs); - clk_unprepare(sdma->clk_ahb); - clk_unprepare(sdma->clk_ipg); /* Kill the tasklet */ for (i = 0; i < MAX_DMA_CHANNELS; i++) { struct sdma_channel *sdmac = &sdma->channel[i]; diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 12a4a4860a74..e8a880f338c6 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -140,12 +140,6 @@ struct ioatdma_chan { int prev_intr_coalesce; }; -struct ioat_sysfs_entry { - struct attribute attr; - ssize_t (*show)(struct dma_chan *, char *); - ssize_t (*store)(struct dma_chan *, const char *, size_t); -}; - /** * struct ioat_sed_ent - wrapper around super extended hardware descriptor * @hw: hardware SED @@ -195,11 +189,8 @@ struct ioat_ring_ent { struct ioat_sed_ent *sed; }; -extern const struct sysfs_ops ioat_sysfs_ops; -extern struct ioat_sysfs_entry ioat_version_attr; -extern struct ioat_sysfs_entry ioat_cap_attr; extern int ioat_pending_level; -extern struct kobj_type ioat_ktype; +extern const struct kobj_type ioat_ktype; extern struct kmem_cache *ioat_cache; extern struct kmem_cache *ioat_sed_cache; @@ -402,7 +393,7 @@ void ioat_issue_pending(struct dma_chan *chan); /* IOAT Init functions */ bool is_bwd_ioat(struct pci_dev *pdev); struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); -void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type); +void ioat_kobject_add(struct ioatdma_device *ioat_dma, const struct kobj_type *type); void ioat_kobject_del(struct ioatdma_device *ioat_dma); int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma); void ioat_stop(struct ioatdma_chan *ioat_chan); diff --git a/drivers/dma/ioat/sysfs.c b/drivers/dma/ioat/sysfs.c index 168adf28c5b1..e796ddb5383f 100644 --- a/drivers/dma/ioat/sysfs.c +++ b/drivers/dma/ioat/sysfs.c @@ -14,6 +14,12 @@ #include "../dmaengine.h" +struct ioat_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct dma_chan *, char *); + ssize_t (*store)(struct dma_chan *, const char *, size_t); +}; + static ssize_t cap_show(struct dma_chan *c, char *page) { struct dma_device *dma = c->device; @@ -26,7 +32,7 @@ static ssize_t cap_show(struct dma_chan *c, char *page) dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : ""); } -struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap); +static const struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap); static ssize_t version_show(struct dma_chan *c, char *page) { @@ -36,15 +42,15 @@ static ssize_t version_show(struct dma_chan *c, char *page) return sprintf(page, "%d.%d\n", ioat_dma->version >> 4, ioat_dma->version & 0xf); } -struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version); +static const struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version); static ssize_t ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page) { - struct ioat_sysfs_entry *entry; + const struct ioat_sysfs_entry *entry; struct ioatdma_chan *ioat_chan; - entry = container_of(attr, struct ioat_sysfs_entry, attr); + entry = container_of_const(attr, struct ioat_sysfs_entry, attr); ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); if (!entry->show) @@ -56,10 +62,10 @@ static ssize_t ioat_attr_store(struct kobject *kobj, struct attribute *attr, const char *page, size_t count) { - struct ioat_sysfs_entry *entry; + const struct ioat_sysfs_entry *entry; struct ioatdma_chan *ioat_chan; - entry = container_of(attr, struct ioat_sysfs_entry, attr); + entry = container_of_const(attr, struct ioat_sysfs_entry, attr); ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); if (!entry->store) @@ -67,12 +73,12 @@ const char *page, size_t count) return entry->store(&ioat_chan->dma_chan, page, count); } -const struct sysfs_ops ioat_sysfs_ops = { +static const struct sysfs_ops ioat_sysfs_ops = { .show = ioat_attr_show, .store = ioat_attr_store, }; -void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type) +void ioat_kobject_add(struct ioatdma_device *ioat_dma, const struct kobj_type *type) { struct dma_device *dma = &ioat_dma->dma_dev; struct dma_chan *c; @@ -114,7 +120,7 @@ static ssize_t ring_size_show(struct dma_chan *c, char *page) return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1); } -static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); +static const struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); static ssize_t ring_active_show(struct dma_chan *c, char *page) { @@ -123,7 +129,7 @@ static ssize_t ring_active_show(struct dma_chan *c, char *page) /* ...taken outside the lock, no need to be precise */ return sprintf(page, "%d\n", ioat_ring_active(ioat_chan)); } -static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); +static const struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); static ssize_t intr_coalesce_show(struct dma_chan *c, char *page) { @@ -148,9 +154,9 @@ size_t count) return count; } -static struct ioat_sysfs_entry intr_coalesce_attr = __ATTR_RW(intr_coalesce); +static const struct ioat_sysfs_entry intr_coalesce_attr = __ATTR_RW(intr_coalesce); -static struct attribute *ioat_attrs[] = { +static const struct attribute *const ioat_attrs[] = { &ring_size_attr.attr, &ring_active_attr.attr, &ioat_cap_attr.attr, @@ -160,7 +166,7 @@ static struct attribute *ioat_attrs[] = { }; ATTRIBUTE_GROUPS(ioat); -struct kobj_type ioat_ktype = { +const struct kobj_type ioat_ktype = { .sysfs_ops = &ioat_sysfs_ops, .default_groups = ioat_groups, }; diff --git a/drivers/dma/loongson/Kconfig b/drivers/dma/loongson/Kconfig new file mode 100644 index 000000000000..c4e62dce5d4f --- /dev/null +++ b/drivers/dma/loongson/Kconfig @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Loongson DMA controllers drivers +# +if MACH_LOONGSON32 || MACH_LOONGSON64 || COMPILE_TEST + +config LOONGSON1_APB_DMA + tristate "Loongson1 APB DMA support" + depends on MACH_LOONGSON32 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + This selects support for the APB DMA controller in Loongson1 SoCs, + which is required by Loongson1 NAND and audio support. + +config LOONGSON2_APB_DMA + tristate "Loongson2 APB DMA support" + depends on MACH_LOONGSON64 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the Loongson2 APB DMA controller driver. The + DMA controller is having single DMA channel which can be + configured for different peripherals like audio, nand, sdio + etc which is in APB bus. + + This DMA controller transfers data from memory to peripheral fifo. + It does not support memory to memory data transfer. + +config LOONGSON2_APB_CMC_DMA + tristate "Loongson2 Chain Multi-Channel DMA support" + depends on MACH_LOONGSON64 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the Loongson Chain Multi-Channel DMA controller driver. + It is discovered on the Loongson-2K chip (Loongson-2K0300/Loongson-2K3000), + which has 4/8 channels internally, enabling bidirectional data transfer + between devices and memory. + +endif diff --git a/drivers/dma/loongson/Makefile b/drivers/dma/loongson/Makefile new file mode 100644 index 000000000000..48c19781e729 --- /dev/null +++ b/drivers/dma/loongson/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_LOONGSON1_APB_DMA) += loongson1-apb-dma.o +obj-$(CONFIG_LOONGSON2_APB_DMA) += loongson2-apb-dma.o +obj-$(CONFIG_LOONGSON2_APB_CMC_DMA) += loongson2-apb-cmc-dma.o diff --git a/drivers/dma/loongson1-apb-dma.c b/drivers/dma/loongson/loongson1-apb-dma.c index 2e347aba9af8..89786cbd20ab 100644 --- a/drivers/dma/loongson1-apb-dma.c +++ b/drivers/dma/loongson/loongson1-apb-dma.c @@ -16,8 +16,8 @@ #include <linux/platform_device.h> #include <linux/slab.h> -#include "dmaengine.h" -#include "virt-dma.h" +#include "../dmaengine.h" +#include "../virt-dma.h" /* Loongson-1 DMA Control Register */ #define LS1X_DMA_CTRL 0x0 diff --git a/drivers/dma/loongson/loongson2-apb-cmc-dma.c b/drivers/dma/loongson/loongson2-apb-cmc-dma.c new file mode 100644 index 000000000000..1c9a542edc85 --- /dev/null +++ b/drivers/dma/loongson/loongson2-apb-cmc-dma.c @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Loongson-2 Chain Multi-Channel DMA Controller driver + * + * Copyright (C) 2024-2026 Loongson Technology Corporation Limited + */ + +#include <linux/acpi.h> +#include <linux/acpi_dma.h> +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "../dmaengine.h" +#include "../virt-dma.h" + +#define LOONGSON2_CMCDMA_ISR 0x0 /* DMA Interrupt Status Register */ +#define LOONGSON2_CMCDMA_IFCR 0x4 /* DMA Interrupt Flag Clear Register */ +#define LOONGSON2_CMCDMA_CCR 0x8 /* DMA Channel Configuration Register */ +#define LOONGSON2_CMCDMA_CNDTR 0xc /* DMA Channel Transmit Count Register */ +#define LOONGSON2_CMCDMA_CPAR 0x10 /* DMA Channel Peripheral Address Register */ +#define LOONGSON2_CMCDMA_CMAR 0x14 /* DMA Channel Memory Address Register */ + +/* Bitfields of DMA interrupt status register */ +#define LOONGSON2_CMCDMA_TCI BIT(1) /* Transfer Complete Interrupt */ +#define LOONGSON2_CMCDMA_HTI BIT(2) /* Half Transfer Interrupt */ +#define LOONGSON2_CMCDMA_TEI BIT(3) /* Transfer Error Interrupt */ + +#define LOONGSON2_CMCDMA_MASKI \ + (LOONGSON2_CMCDMA_TCI | LOONGSON2_CMCDMA_HTI | LOONGSON2_CMCDMA_TEI) + +/* Bitfields of DMA channel x Configuration Register */ +#define LOONGSON2_CMCDMA_CCR_EN BIT(0) /* Stream Enable */ +#define LOONGSON2_CMCDMA_CCR_TCIE BIT(1) /* Transfer Complete Interrupt Enable */ +#define LOONGSON2_CMCDMA_CCR_HTIE BIT(2) /* Half Transfer Complete Interrupt Enable */ +#define LOONGSON2_CMCDMA_CCR_TEIE BIT(3) /* Transfer Error Interrupt Enable */ +#define LOONGSON2_CMCDMA_CCR_DIR BIT(4) /* Data Transfer Direction */ +#define LOONGSON2_CMCDMA_CCR_CIRC BIT(5) /* Circular mode */ +#define LOONGSON2_CMCDMA_CCR_PINC BIT(6) /* Peripheral increment mode */ +#define LOONGSON2_CMCDMA_CCR_MINC BIT(7) /* Memory increment mode */ +#define LOONGSON2_CMCDMA_CCR_PSIZE_MASK GENMASK(9, 8) +#define LOONGSON2_CMCDMA_CCR_MSIZE_MASK GENMASK(11, 10) +#define LOONGSON2_CMCDMA_CCR_PL_MASK GENMASK(13, 12) +#define LOONGSON2_CMCDMA_CCR_M2M BIT(14) + +#define LOONGSON2_CMCDMA_CCR_CFG_MASK \ + (LOONGSON2_CMCDMA_CCR_PINC | LOONGSON2_CMCDMA_CCR_MINC | LOONGSON2_CMCDMA_CCR_PL_MASK) + +#define LOONGSON2_CMCDMA_CCR_IRQ_MASK \ + (LOONGSON2_CMCDMA_CCR_TCIE | LOONGSON2_CMCDMA_CCR_HTIE | LOONGSON2_CMCDMA_CCR_TEIE) + +#define LOONGSON2_CMCDMA_STREAM_MASK \ + (LOONGSON2_CMCDMA_CCR_CFG_MASK | LOONGSON2_CMCDMA_CCR_IRQ_MASK) + +#define LOONGSON2_CMCDMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) + +#define LOONSON2_CMCDMA_MAX_DATA_ITEMS SZ_64K + +struct loongson2_cmc_dma_chan_reg { + u32 ccr; + u32 cndtr; + u32 cpar; + u32 cmar; +}; + +struct loongson2_cmc_dma_sg_req { + u32 len; + struct loongson2_cmc_dma_chan_reg chan_reg; +}; + +struct loongson2_cmc_dma_desc { + struct virt_dma_desc vdesc; + bool cyclic; + u32 num_sgs; + struct loongson2_cmc_dma_sg_req sg_req[] __counted_by(num_sgs); +}; + +struct loongson2_cmc_dma_chan { + struct virt_dma_chan vchan; + struct dma_slave_config dma_sconfig; + struct loongson2_cmc_dma_desc *desc; + u32 id; + u32 irq; + u32 next_sg; + struct loongson2_cmc_dma_chan_reg chan_reg; +}; + +struct loongson2_cmc_dma_dev { + struct dma_device ddev; + struct clk *dma_clk; + void __iomem *base; + u32 nr_channels; + u32 chan_reg_offset; + struct loongson2_cmc_dma_chan chan[] __counted_by(nr_channels); +}; + +struct loongson2_cmc_dma_config { + u32 max_channels; + u32 chan_reg_offset; +}; + +static const struct loongson2_cmc_dma_config ls2k0300_cmc_dma_config = { + .max_channels = 8, + .chan_reg_offset = 0x14, +}; + +static const struct loongson2_cmc_dma_config ls2k3000_cmc_dma_config = { + .max_channels = 4, + .chan_reg_offset = 0x18, +}; + +static struct loongson2_cmc_dma_dev *lmdma_get_dev(struct loongson2_cmc_dma_chan *lchan) +{ + return container_of(lchan->vchan.chan.device, struct loongson2_cmc_dma_dev, ddev); +} + +static struct loongson2_cmc_dma_chan *to_lmdma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct loongson2_cmc_dma_chan, vchan.chan); +} + +static struct loongson2_cmc_dma_desc *to_lmdma_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct loongson2_cmc_dma_desc, vdesc); +} + +static struct device *chan2dev(struct loongson2_cmc_dma_chan *lchan) +{ + return &lchan->vchan.chan.dev->device; +} + +static u32 loongson2_cmc_dma_read(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id) +{ + return readl(lddev->base + (reg + lddev->chan_reg_offset * id)); +} + +static void loongson2_cmc_dma_write(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id, u32 val) +{ + writel(val, lddev->base + (reg + lddev->chan_reg_offset * id)); +} + +static int loongson2_cmc_dma_get_width(enum dma_slave_buswidth width) +{ + switch (width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + case DMA_SLAVE_BUSWIDTH_2_BYTES: + case DMA_SLAVE_BUSWIDTH_4_BYTES: + return ffs(width) - 1; + default: + return -EINVAL; + } +} + +static int loongson2_cmc_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *config) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + + memcpy(&lchan->dma_sconfig, config, sizeof(*config)); + + return 0; +} + +static void loongson2_cmc_dma_irq_clear(struct loongson2_cmc_dma_chan *lchan, u32 flags) +{ + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + u32 ifcr; + + ifcr = flags << (4 * lchan->id); + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_IFCR, 0, ifcr); +} + +static void loongson2_cmc_dma_stop(struct loongson2_cmc_dma_chan *lchan) +{ + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + u32 ccr; + + ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id); + ccr &= ~(LOONGSON2_CMCDMA_CCR_IRQ_MASK | LOONGSON2_CMCDMA_CCR_EN); + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, ccr); + + loongson2_cmc_dma_irq_clear(lchan, LOONGSON2_CMCDMA_MASKI); +} + +static int loongson2_cmc_dma_terminate_all(struct dma_chan *chan) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + + LIST_HEAD(head); + + scoped_guard(spinlock_irqsave, &lchan->vchan.lock) { + if (lchan->desc) { + vchan_terminate_vdesc(&lchan->desc->vdesc); + loongson2_cmc_dma_stop(lchan); + lchan->desc = NULL; + } + vchan_get_all_descriptors(&lchan->vchan, &head); + } + + vchan_dma_desc_free_list(&lchan->vchan, &head); + + return 0; +} + +static void loongson2_cmc_dma_synchronize(struct dma_chan *chan) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + + vchan_synchronize(&lchan->vchan); +} + +static void loongson2_cmc_dma_start_transfer(struct loongson2_cmc_dma_chan *lchan) +{ + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + struct loongson2_cmc_dma_sg_req *sg_req; + struct loongson2_cmc_dma_chan_reg *reg; + struct virt_dma_desc *vdesc; + + loongson2_cmc_dma_stop(lchan); + + if (!lchan->desc) { + vdesc = vchan_next_desc(&lchan->vchan); + if (!vdesc) + return; + + list_del(&vdesc->node); + lchan->desc = to_lmdma_desc(vdesc); + lchan->next_sg = 0; + } + + if (lchan->next_sg == lchan->desc->num_sgs) + lchan->next_sg = 0; + + sg_req = &lchan->desc->sg_req[lchan->next_sg]; + reg = &sg_req->chan_reg; + + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, reg->ccr); + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CNDTR, lchan->id, reg->cndtr); + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CPAR, lchan->id, reg->cpar); + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CMAR, lchan->id, reg->cmar); + + lchan->next_sg++; + + /* Start DMA */ + reg->ccr |= LOONGSON2_CMCDMA_CCR_EN; + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, reg->ccr); +} + +static void loongson2_cmc_dma_configure_next_sg(struct loongson2_cmc_dma_chan *lchan) +{ + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + struct loongson2_cmc_dma_sg_req *sg_req; + u32 ccr, id = lchan->id; + + if (lchan->next_sg == lchan->desc->num_sgs) + lchan->next_sg = 0; + + /* Stop to update mem addr */ + ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, id); + ccr &= ~LOONGSON2_CMCDMA_CCR_EN; + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, id, ccr); + + sg_req = &lchan->desc->sg_req[lchan->next_sg]; + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CMAR, id, sg_req->chan_reg.cmar); + + /* Start transition */ + ccr |= LOONGSON2_CMCDMA_CCR_EN; + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, id, ccr); +} + +static void loongson2_cmc_dma_handle_chan_done(struct loongson2_cmc_dma_chan *lchan) +{ + if (!lchan->desc) + return; + + if (lchan->desc->cyclic) { + vchan_cyclic_callback(&lchan->desc->vdesc); + /* LOONGSON2_CMCDMA_CCR_CIRC mode don't need update register */ + if (lchan->desc->num_sgs == 1) + return; + loongson2_cmc_dma_configure_next_sg(lchan); + lchan->next_sg++; + } else { + if (lchan->next_sg == lchan->desc->num_sgs) { + vchan_cookie_complete(&lchan->desc->vdesc); + lchan->desc = NULL; + } + loongson2_cmc_dma_start_transfer(lchan); + } +} + +static irqreturn_t loongson2_cmc_dma_chan_irq(int irq, void *devid) +{ + struct loongson2_cmc_dma_chan *lchan = devid; + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + struct device *dev = chan2dev(lchan); + u32 ists, status, ccr; + + scoped_guard(spinlock, &lchan->vchan.lock) { + ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id); + ists = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_ISR, 0); + status = (ists >> (4 * lchan->id)) & LOONGSON2_CMCDMA_MASKI; + + loongson2_cmc_dma_irq_clear(lchan, status); + + if (status & LOONGSON2_CMCDMA_TCI) { + loongson2_cmc_dma_handle_chan_done(lchan); + status &= ~LOONGSON2_CMCDMA_TCI; + } + + if (status & LOONGSON2_CMCDMA_HTI) + status &= ~LOONGSON2_CMCDMA_HTI; + + if (status & LOONGSON2_CMCDMA_TEI) { + dev_err(dev, "DMA Transform Error.\n"); + if (!(ccr & LOONGSON2_CMCDMA_CCR_EN)) + dev_err(dev, "Channel disabled by HW.\n"); + } + } + + return IRQ_HANDLED; +} + +static void loongson2_cmc_dma_issue_pending(struct dma_chan *chan) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + + guard(spinlock_irqsave)(&lchan->vchan.lock); + + if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) { + dev_dbg(chan2dev(lchan), "vchan %pK: issued\n", &lchan->vchan); + loongson2_cmc_dma_start_transfer(lchan); + } +} + +static int loongson2_cmc_dma_set_xfer_param(struct loongson2_cmc_dma_chan *lchan, + enum dma_transfer_direction direction, + enum dma_slave_buswidth *buswidth, u32 buf_len) +{ + struct dma_slave_config sconfig = lchan->dma_sconfig; + struct device *dev = chan2dev(lchan); + int dev_width; + u32 ccr; + + switch (direction) { + case DMA_MEM_TO_DEV: + dev_width = loongson2_cmc_dma_get_width(sconfig.dst_addr_width); + if (dev_width < 0) { + dev_err(dev, "DMA_MEM_TO_DEV bus width not supported\n"); + return dev_width; + } + lchan->chan_reg.cpar = sconfig.dst_addr; + ccr = LOONGSON2_CMCDMA_CCR_DIR; + *buswidth = sconfig.dst_addr_width; + break; + case DMA_DEV_TO_MEM: + dev_width = loongson2_cmc_dma_get_width(sconfig.src_addr_width); + if (dev_width < 0) { + dev_err(dev, "DMA_DEV_TO_MEM bus width not supported\n"); + return dev_width; + } + lchan->chan_reg.cpar = sconfig.src_addr; + ccr = LOONGSON2_CMCDMA_CCR_MINC; + *buswidth = sconfig.src_addr_width; + break; + default: + return -EINVAL; + } + + ccr |= FIELD_PREP(LOONGSON2_CMCDMA_CCR_PSIZE_MASK, dev_width) | + FIELD_PREP(LOONGSON2_CMCDMA_CCR_MSIZE_MASK, dev_width); + + /* Set DMA control register */ + lchan->chan_reg.ccr &= ~(LOONGSON2_CMCDMA_CCR_PSIZE_MASK | LOONGSON2_CMCDMA_CCR_MSIZE_MASK); + lchan->chan_reg.ccr |= ccr; + + return 0; +} + +static struct dma_async_tx_descriptor * +loongson2_cmc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, u32 sg_len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + struct loongson2_cmc_dma_desc *desc; + enum dma_slave_buswidth buswidth; + struct scatterlist *sg; + u32 num_items, i; + int ret; + + desc = kzalloc_flex(*desc, sg_req, sg_len, GFP_NOWAIT); + if (!desc) + return ERR_PTR(-ENOMEM); + + for_each_sg(sgl, sg, sg_len, i) { + ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, sg_dma_len(sg)); + if (ret) + return ERR_PTR(ret); + + num_items = DIV_ROUND_UP(sg_dma_len(sg), buswidth); + if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) { + dev_err(chan2dev(lchan), "Number of items not supported\n"); + kfree(desc); + return ERR_PTR(-EINVAL); + } + + desc->sg_req[i].len = sg_dma_len(sg); + desc->sg_req[i].chan_reg.ccr = lchan->chan_reg.ccr; + desc->sg_req[i].chan_reg.cpar = lchan->chan_reg.cpar; + desc->sg_req[i].chan_reg.cmar = sg_dma_address(sg); + desc->sg_req[i].chan_reg.cndtr = num_items; + } + + desc->num_sgs = sg_len; + desc->cyclic = false; + + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); +} + +static struct dma_async_tx_descriptor * +loongson2_cmc_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + struct loongson2_cmc_dma_desc *desc; + enum dma_slave_buswidth buswidth; + u32 num_periods, num_items, i; + int ret; + + if (unlikely(buf_len % period_len)) + return ERR_PTR(-EINVAL); + + ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, period_len); + if (ret) + return ERR_PTR(ret); + + num_items = DIV_ROUND_UP(period_len, buswidth); + if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) { + dev_err(chan2dev(lchan), "Number of items not supported\n"); + return ERR_PTR(-EINVAL); + } + + /* Enable Circular mode */ + if (buf_len == period_len) + lchan->chan_reg.ccr |= LOONGSON2_CMCDMA_CCR_CIRC; + + num_periods = DIV_ROUND_UP(buf_len, period_len); + desc = kzalloc_flex(*desc, sg_req, num_periods, GFP_NOWAIT); + if (!desc) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < num_periods; i++) { + desc->sg_req[i].len = period_len; + desc->sg_req[i].chan_reg.ccr = lchan->chan_reg.ccr; + desc->sg_req[i].chan_reg.cpar = lchan->chan_reg.cpar; + desc->sg_req[i].chan_reg.cmar = buf_addr; + desc->sg_req[i].chan_reg.cndtr = num_items; + buf_addr += period_len; + } + + desc->num_sgs = num_periods; + desc->cyclic = true; + + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); +} + +static size_t loongson2_cmc_dma_desc_residue(struct loongson2_cmc_dma_chan *lchan, + struct loongson2_cmc_dma_desc *desc, u32 next_sg) +{ + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + u32 residue, width, ndtr, ccr, i; + + ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id); + width = FIELD_GET(LOONGSON2_CMCDMA_CCR_PSIZE_MASK, ccr); + + ndtr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CNDTR, lchan->id); + residue = ndtr << width; + + if (lchan->desc->cyclic && next_sg == 0) + return residue; + + for (i = next_sg; i < desc->num_sgs; i++) + residue += desc->sg_req[i].len; + + return residue; +} + +static enum dma_status loongson2_cmc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + struct virt_dma_desc *vdesc; + enum dma_status status; + + status = dma_cookie_status(chan, cookie, state); + if (status == DMA_COMPLETE || !state) + return status; + + scoped_guard(spinlock_irqsave, &lchan->vchan.lock) { + vdesc = vchan_find_desc(&lchan->vchan, cookie); + if (lchan->desc && cookie == lchan->desc->vdesc.tx.cookie) + state->residue = loongson2_cmc_dma_desc_residue(lchan, lchan->desc, + lchan->next_sg); + else if (vdesc) + state->residue = loongson2_cmc_dma_desc_residue(lchan, + to_lmdma_desc(vdesc), 0); + } + + return status; +} + +static void loongson2_cmc_dma_free_chan_resources(struct dma_chan *chan) +{ + vchan_free_chan_resources(to_virt_chan(chan)); +} + +static void loongson2_cmc_dma_desc_free(struct virt_dma_desc *vdesc) +{ + kfree(to_lmdma_desc(vdesc)); +} + +static bool loongson2_cmc_dma_acpi_filter(struct dma_chan *chan, void *param) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + struct acpi_dma_spec *dma_spec = param; + + memset(&lchan->chan_reg, 0, sizeof(struct loongson2_cmc_dma_chan_reg)); + lchan->chan_reg.ccr = dma_spec->chan_id & LOONGSON2_CMCDMA_STREAM_MASK; + + return true; +} + +static int loongson2_cmc_dma_acpi_controller_register(struct loongson2_cmc_dma_dev *lddev) +{ + struct device *dev = lddev->ddev.dev; + struct acpi_dma_filter_info *info; + + if (!is_acpi_node(dev_fwnode(dev))) + return 0; + + info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + dma_cap_zero(info->dma_cap); + info->dma_cap = lddev->ddev.cap_mask; + info->filter_fn = loongson2_cmc_dma_acpi_filter; + + return devm_acpi_dma_controller_register(dev, acpi_dma_simple_xlate, info); +} + +static struct dma_chan *loongson2_cmc_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct loongson2_cmc_dma_dev *lddev = ofdma->of_dma_data; + struct device *dev = lddev->ddev.dev; + struct loongson2_cmc_dma_chan *lchan; + struct dma_chan *chan; + + if (dma_spec->args_count < 2) + return ERR_PTR(-EINVAL); + + if (dma_spec->args[0] >= lddev->nr_channels) { + dev_err(dev, "Invalid channel id.\n"); + return ERR_PTR(-EINVAL); + } + + lchan = &lddev->chan[dma_spec->args[0]]; + chan = dma_get_slave_channel(&lchan->vchan.chan); + if (!chan) { + dev_err(dev, "No more channels available.\n"); + return ERR_PTR(-EINVAL); + } + + memset(&lchan->chan_reg, 0, sizeof(struct loongson2_cmc_dma_chan_reg)); + lchan->chan_reg.ccr = dma_spec->args[1] & LOONGSON2_CMCDMA_STREAM_MASK; + + return chan; +} + +static int loongson2_cmc_dma_of_controller_register(struct loongson2_cmc_dma_dev *lddev) +{ + struct device *dev = lddev->ddev.dev; + + if (!is_of_node(dev_fwnode(dev))) + return 0; + + return of_dma_controller_register(dev->of_node, loongson2_cmc_dma_of_xlate, lddev); +} + +static int loongson2_cmc_dma_probe(struct platform_device *pdev) +{ + const struct loongson2_cmc_dma_config *config; + struct loongson2_cmc_dma_chan *lchan; + struct loongson2_cmc_dma_dev *lddev; + struct device *dev = &pdev->dev; + struct dma_device *ddev; + u32 nr_chans, i; + int ret; + + config = (const struct loongson2_cmc_dma_config *)device_get_match_data(dev); + if (!config) + return -EINVAL; + + ret = device_property_read_u32(dev, "dma-channels", &nr_chans); + if (ret || nr_chans > config->max_channels) { + dev_err(dev, "missing or invalid dma-channels property\n"); + nr_chans = config->max_channels; + } + + lddev = devm_kzalloc(dev, struct_size(lddev, chan, nr_chans), GFP_KERNEL); + if (!lddev) + return -ENOMEM; + + lddev->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(lddev->base)) + return PTR_ERR(lddev->base); + + platform_set_drvdata(pdev, lddev); + lddev->nr_channels = nr_chans; + lddev->chan_reg_offset = config->chan_reg_offset; + + lddev->dma_clk = devm_clk_get_optional_enabled(dev, NULL); + if (IS_ERR(lddev->dma_clk)) + return dev_err_probe(dev, PTR_ERR(lddev->dma_clk), "Failed to get dma clock\n"); + + ddev = &lddev->ddev; + ddev->dev = dev; + + dma_cap_zero(ddev->cap_mask); + dma_cap_set(DMA_SLAVE, ddev->cap_mask); + dma_cap_set(DMA_PRIVATE, ddev->cap_mask); + dma_cap_set(DMA_CYCLIC, ddev->cap_mask); + + ddev->device_free_chan_resources = loongson2_cmc_dma_free_chan_resources; + ddev->device_config = loongson2_cmc_dma_slave_config; + ddev->device_prep_slave_sg = loongson2_cmc_dma_prep_slave_sg; + ddev->device_prep_dma_cyclic = loongson2_cmc_dma_prep_dma_cyclic; + ddev->device_issue_pending = loongson2_cmc_dma_issue_pending; + ddev->device_synchronize = loongson2_cmc_dma_synchronize; + ddev->device_tx_status = loongson2_cmc_dma_tx_status; + ddev->device_terminate_all = loongson2_cmc_dma_terminate_all; + + ddev->max_sg_burst = LOONSON2_CMCDMA_MAX_DATA_ITEMS; + ddev->src_addr_widths = LOONGSON2_CMCDMA_BUSWIDTHS; + ddev->dst_addr_widths = LOONGSON2_CMCDMA_BUSWIDTHS; + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + INIT_LIST_HEAD(&ddev->channels); + + for (i = 0; i < nr_chans; i++) { + lchan = &lddev->chan[i]; + + lchan->id = i; + lchan->vchan.desc_free = loongson2_cmc_dma_desc_free; + vchan_init(&lchan->vchan, ddev); + } + + ret = dmaenginem_async_device_register(ddev); + if (ret) + return dev_err_probe(dev, ret, "Failed to register DMA engine device.\n"); + + for (i = 0; i < nr_chans; i++) { + lchan = &lddev->chan[i]; + + lchan->irq = platform_get_irq(pdev, i); + if (lchan->irq < 0) + return lchan->irq; + + ret = devm_request_irq(dev, lchan->irq, loongson2_cmc_dma_chan_irq, IRQF_SHARED, + dev_name(chan2dev(lchan)), lchan); + if (ret) + return ret; + } + + ret = loongson2_cmc_dma_acpi_controller_register(lddev); + if (ret) + return dev_err_probe(dev, ret, "Failed to register dma controller with ACPI.\n"); + + ret = loongson2_cmc_dma_of_controller_register(lddev); + if (ret) + return dev_err_probe(dev, ret, "Failed to register dma controller with FDT.\n"); + + dev_info(dev, "Loongson-2 Multi-Channel DMA Controller registered successfully.\n"); + + return 0; +} + +static void loongson2_cmc_dma_remove(struct platform_device *pdev) +{ + of_dma_controller_free(pdev->dev.of_node); +} + +static const struct of_device_id loongson2_cmc_dma_of_match[] = { + { .compatible = "loongson,ls2k0300-dma", .data = &ls2k0300_cmc_dma_config }, + { .compatible = "loongson,ls2k3000-dma", .data = &ls2k3000_cmc_dma_config }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, loongson2_cmc_dma_of_match); + +static const struct acpi_device_id loongson2_cmc_dma_acpi_match[] = { + { "LOON0014", .driver_data = (kernel_ulong_t)&ls2k3000_cmc_dma_config }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(acpi, loongson2_cmc_dma_acpi_match); + +static struct platform_driver loongson2_cmc_dma_driver = { + .driver = { + .name = "loongson2-apb-cmc-dma", + .of_match_table = loongson2_cmc_dma_of_match, + .acpi_match_table = loongson2_cmc_dma_acpi_match, + }, + .probe = loongson2_cmc_dma_probe, + .remove = loongson2_cmc_dma_remove, +}; +module_platform_driver(loongson2_cmc_dma_driver); + +MODULE_DESCRIPTION("Loongson-2 Chain Multi-Channel DMA Controller driver"); +MODULE_AUTHOR("Loongson Technology Corporation Limited"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/loongson2-apb-dma.c b/drivers/dma/loongson/loongson2-apb-dma.c index b981475e6779..aceb069e71fc 100644 --- a/drivers/dma/loongson2-apb-dma.c +++ b/drivers/dma/loongson/loongson2-apb-dma.c @@ -17,8 +17,8 @@ #include <linux/platform_device.h> #include <linux/slab.h> -#include "dmaengine.h" -#include "virt-dma.h" +#include "../dmaengine.h" +#include "../virt-dma.h" /* Global Configuration Register */ #define LDMA_ORDER_ERG 0x0 @@ -461,12 +461,11 @@ static int ls2x_dma_slave_config(struct dma_chan *chan, static void ls2x_dma_issue_pending(struct dma_chan *chan) { struct ls2x_dma_chan *lchan = to_ldma_chan(chan); - unsigned long flags; - spin_lock_irqsave(&lchan->vchan.lock, flags); + guard(spinlock_irqsave)(&lchan->vchan.lock); + if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) ls2x_dma_start_transfer(lchan); - spin_unlock_irqrestore(&lchan->vchan.lock, flags); } /* @@ -478,19 +477,18 @@ static void ls2x_dma_issue_pending(struct dma_chan *chan) static int ls2x_dma_terminate_all(struct dma_chan *chan) { struct ls2x_dma_chan *lchan = to_ldma_chan(chan); - unsigned long flags; LIST_HEAD(head); - spin_lock_irqsave(&lchan->vchan.lock, flags); - /* Setting stop cmd */ - ls2x_dma_write_cmd(lchan, LDMA_STOP); - if (lchan->desc) { - vchan_terminate_vdesc(&lchan->desc->vdesc); - lchan->desc = NULL; - } + scoped_guard(spinlock_irqsave, &lchan->vchan.lock) { + /* Setting stop cmd */ + ls2x_dma_write_cmd(lchan, LDMA_STOP); + if (lchan->desc) { + vchan_terminate_vdesc(&lchan->desc->vdesc); + lchan->desc = NULL; + } - vchan_get_all_descriptors(&lchan->vchan, &head); - spin_unlock_irqrestore(&lchan->vchan.lock, flags); + vchan_get_all_descriptors(&lchan->vchan, &head); + } vchan_dma_desc_free_list(&lchan->vchan, &head); return 0; @@ -511,14 +509,13 @@ static void ls2x_dma_synchronize(struct dma_chan *chan) static int ls2x_dma_pause(struct dma_chan *chan) { struct ls2x_dma_chan *lchan = to_ldma_chan(chan); - unsigned long flags; - spin_lock_irqsave(&lchan->vchan.lock, flags); + guard(spinlock_irqsave)(&lchan->vchan.lock); + if (lchan->desc && lchan->desc->status == DMA_IN_PROGRESS) { ls2x_dma_write_cmd(lchan, LDMA_STOP); lchan->desc->status = DMA_PAUSED; } - spin_unlock_irqrestore(&lchan->vchan.lock, flags); return 0; } @@ -526,14 +523,13 @@ static int ls2x_dma_pause(struct dma_chan *chan) static int ls2x_dma_resume(struct dma_chan *chan) { struct ls2x_dma_chan *lchan = to_ldma_chan(chan); - unsigned long flags; - spin_lock_irqsave(&lchan->vchan.lock, flags); + guard(spinlock_irqsave)(&lchan->vchan.lock); + if (lchan->desc && lchan->desc->status == DMA_PAUSED) { lchan->desc->status = DMA_IN_PROGRESS; ls2x_dma_write_cmd(lchan, LDMA_START); } - spin_unlock_irqrestore(&lchan->vchan.lock, flags); return 0; } @@ -550,22 +546,22 @@ static irqreturn_t ls2x_dma_isr(int irq, void *dev_id) struct ls2x_dma_chan *lchan = dev_id; struct ls2x_dma_desc *desc; - spin_lock(&lchan->vchan.lock); - desc = lchan->desc; - if (desc) { - if (desc->cyclic) { - vchan_cyclic_callback(&desc->vdesc); - } else { - desc->status = DMA_COMPLETE; - vchan_cookie_complete(&desc->vdesc); - ls2x_dma_start_transfer(lchan); + scoped_guard(spinlock, &lchan->vchan.lock) { + desc = lchan->desc; + if (desc) { + if (desc->cyclic) { + vchan_cyclic_callback(&desc->vdesc); + } else { + desc->status = DMA_COMPLETE; + vchan_cookie_complete(&desc->vdesc); + ls2x_dma_start_transfer(lchan); + } + + /* ls2x_dma_start_transfer() updates lchan->desc */ + if (!lchan->desc) + ls2x_dma_write_cmd(lchan, LDMA_STOP); } - - /* ls2x_dma_start_transfer() updates lchan->desc */ - if (!lchan->desc) - ls2x_dma_write_cmd(lchan, LDMA_STOP); } - spin_unlock(&lchan->vchan.lock); return IRQ_HANDLED; } @@ -616,17 +612,13 @@ static int ls2x_dma_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(priv->regs), "devm_platform_ioremap_resource failed.\n"); - priv->dma_clk = devm_clk_get(&pdev->dev, NULL); + priv->dma_clk = devm_clk_get_enabled(dev, NULL); if (IS_ERR(priv->dma_clk)) - return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "devm_clk_get failed.\n"); - - ret = clk_prepare_enable(priv->dma_clk); - if (ret) - return dev_err_probe(dev, ret, "clk_prepare_enable failed.\n"); + return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "Couldn't start the clock.\n"); ret = ls2x_dma_chan_init(pdev, priv); if (ret) - goto disable_clk; + return ret; ddev = &priv->ddev; ddev->dev = dev; @@ -650,25 +642,18 @@ static int ls2x_dma_probe(struct platform_device *pdev) ddev->dst_addr_widths = LDMA_SLAVE_BUSWIDTHS; ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); - ret = dma_async_device_register(&priv->ddev); + ret = dmaenginem_async_device_register(&priv->ddev); if (ret < 0) - goto disable_clk; + return dev_err_probe(dev, ret, "Failed to register DMA engine device.\n"); ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, priv); if (ret < 0) - goto unregister_dmac; + return dev_err_probe(dev, ret, "Failed to register dma controller.\n"); platform_set_drvdata(pdev, priv); dev_info(dev, "Loongson LS2X APB DMA driver registered successfully.\n"); return 0; - -unregister_dmac: - dma_async_device_unregister(&priv->ddev); -disable_clk: - clk_disable_unprepare(priv->dma_clk); - - return ret; } /* @@ -677,11 +662,7 @@ disable_clk: */ static void ls2x_dma_remove(struct platform_device *pdev) { - struct ls2x_dma_priv *priv = platform_get_drvdata(pdev); - of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&priv->ddev); - clk_disable_unprepare(priv->dma_clk); } static const struct of_device_id ls2x_dma_of_match_table[] = { diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index cfb9962417ef..7acb3d29dad3 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -744,20 +744,19 @@ static int mxs_dma_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; const struct mxs_dma_type *dma_type; + struct device *dev = &pdev->dev; struct mxs_dma_engine *mxs_dma; int ret, i; - mxs_dma = devm_kzalloc(&pdev->dev, sizeof(*mxs_dma), GFP_KERNEL); + mxs_dma = devm_kzalloc(dev, sizeof(*mxs_dma), GFP_KERNEL); if (!mxs_dma) return -ENOMEM; ret = of_property_read_u32(np, "dma-channels", &mxs_dma->nr_channels); - if (ret) { - dev_err(&pdev->dev, "failed to read dma-channels\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "failed to read dma-channels\n"); - dma_type = (struct mxs_dma_type *)of_device_get_match_data(&pdev->dev); + dma_type = (struct mxs_dma_type *)of_device_get_match_data(dev); mxs_dma->type = dma_type->type; mxs_dma->dev_id = dma_type->id; @@ -765,7 +764,7 @@ static int mxs_dma_probe(struct platform_device *pdev) if (IS_ERR(mxs_dma->base)) return PTR_ERR(mxs_dma->base); - mxs_dma->clk = devm_clk_get(&pdev->dev, NULL); + mxs_dma->clk = devm_clk_get(dev, NULL); if (IS_ERR(mxs_dma->clk)) return PTR_ERR(mxs_dma->clk); @@ -795,10 +794,10 @@ static int mxs_dma_probe(struct platform_device *pdev) return ret; mxs_dma->pdev = pdev; - mxs_dma->dma_device.dev = &pdev->dev; + mxs_dma->dma_device.dev = dev; /* mxs_dma gets 65535 bytes maximum sg size */ - dma_set_max_seg_size(mxs_dma->dma_device.dev, MAX_XFER_BYTES); + dma_set_max_seg_size(dev, MAX_XFER_BYTES); mxs_dma->dma_device.device_alloc_chan_resources = mxs_dma_alloc_chan_resources; mxs_dma->dma_device.device_free_chan_resources = mxs_dma_free_chan_resources; @@ -815,18 +814,15 @@ static int mxs_dma_probe(struct platform_device *pdev) mxs_dma->dma_device.device_issue_pending = mxs_dma_enable_chan; ret = dmaenginem_async_device_register(&mxs_dma->dma_device); - if (ret) { - dev_err(mxs_dma->dma_device.dev, "unable to register\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "unable to register\n"); - ret = of_dma_controller_register(np, mxs_dma_xlate, mxs_dma); - if (ret) { - dev_err(mxs_dma->dma_device.dev, - "failed to register controller\n"); - } + ret = devm_of_dma_controller_register(dev, np, mxs_dma_xlate, mxs_dma); + if (ret) + return dev_err_probe(dev, ret, + "failed to register controller\n"); - dev_info(mxs_dma->dma_device.dev, "initialized\n"); + dev_info(dev, "initialized\n"); return 0; } @@ -840,3 +836,6 @@ static struct platform_driver mxs_dma_driver = { }; builtin_platform_driver(mxs_dma_driver); + +MODULE_DESCRIPTION("MXS DMA driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c index f30bdf69c740..625ff29024de 100644 --- a/drivers/dma/sh/rz-dmac.c +++ b/drivers/dma/sh/rz-dmac.c @@ -16,6 +16,7 @@ #include <linux/interrupt.h> #include <linux/iopoll.h> #include <linux/irqchip/irq-renesas-rzv2h.h> +#include <linux/irqchip/irq-renesas-rzt2h.h> #include <linux/list.h> #include <linux/module.h> #include <linux/of.h> @@ -95,9 +96,16 @@ struct rz_dmac_icu { u8 dmac_index; }; +struct rz_dmac_info { + void (*icu_register_dma_req)(struct platform_device *icu_dev, + u8 dmac_index, u8 dmac_channel, u16 req_no); + u16 default_dma_req_no; +}; + struct rz_dmac { struct dma_device engine; struct rz_dmac_icu icu; + const struct rz_dmac_info *info; struct device *dev; struct reset_control *rstc; void __iomem *base; @@ -106,8 +114,6 @@ struct rz_dmac { unsigned int n_channels; struct rz_dmac_chan *channels; - bool has_icu; - DECLARE_BITMAP(modules, 1024); }; @@ -118,10 +124,12 @@ struct rz_dmac { * Registers */ +#define CRTB 0x0020 #define CHSTAT 0x0024 #define CHCTRL 0x0028 #define CHCFG 0x002c #define NXLA 0x0038 +#define CRLA 0x003c #define DCTRL 0x0000 @@ -132,10 +140,12 @@ struct rz_dmac { #define CHANNEL_8_15_COMMON_BASE 0x0700 #define CHSTAT_ER BIT(4) +#define CHSTAT_SUS BIT(3) #define CHSTAT_EN BIT(0) #define CHCTRL_CLRINTMSK BIT(17) #define CHCTRL_CLRSUS BIT(9) +#define CHCTRL_SETSUS BIT(8) #define CHCTRL_CLRTC BIT(6) #define CHCTRL_CLREND BIT(5) #define CHCTRL_CLRRQ BIT(4) @@ -266,15 +276,12 @@ static void rz_dmac_enable_hw(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); - unsigned long flags; u32 nxla; u32 chctrl; u32 chstat; dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); - local_irq_save(flags); - rz_dmac_lmdesc_recycle(channel); nxla = channel->lmdesc.base_dma + @@ -289,8 +296,6 @@ static void rz_dmac_enable_hw(struct rz_dmac_chan *channel) rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1); } - - local_irq_restore(flags); } static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) @@ -316,6 +321,16 @@ static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) rz_dmac_ext_writel(dmac, dmars32, dmars_offset); } +static void rz_dmac_set_dma_req_no(struct rz_dmac *dmac, unsigned int index, + int req_no) +{ + if (dmac->info->icu_register_dma_req) + dmac->info->icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, + index, req_no); + else + rz_dmac_set_dmars_register(dmac, index, req_no); +} + static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; @@ -333,13 +348,7 @@ static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) lmdesc->chext = 0; lmdesc->header = HEADER_LV; - if (dmac->has_icu) { - rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, - channel->index, - RZV2H_ICU_DMAC_REQ_NO_DEFAULT); - } else { - rz_dmac_set_dmars_register(dmac, channel->index, 0); - } + rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no); channel->chcfg = chcfg; channel->chctrl = CHCTRL_STG | CHCTRL_SETEN; @@ -390,12 +399,7 @@ static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) channel->lmdesc.tail = lmdesc; - if (dmac->has_icu) { - rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, - channel->index, channel->mid_rid); - } else { - rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid); - } + rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid); channel->chctrl = CHCTRL_SETEN; } @@ -460,15 +464,12 @@ static void rz_dmac_free_chan_resources(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); - struct rz_lmdesc *lmdesc = channel->lmdesc.base; struct rz_dmac_desc *desc, *_desc; unsigned long flags; - unsigned int i; spin_lock_irqsave(&channel->vc.lock, flags); - for (i = 0; i < DMAC_NR_LMDESC; i++) - lmdesc[i].header = 0; + rz_lmdesc_setup(channel, channel->lmdesc.base); rz_dmac_disable_hw(channel); list_splice_tail_init(&channel->ld_active, &channel->ld_free); @@ -560,15 +561,12 @@ rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, static int rz_dmac_terminate_all(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); - struct rz_lmdesc *lmdesc = channel->lmdesc.base; unsigned long flags; - unsigned int i; LIST_HEAD(head); spin_lock_irqsave(&channel->vc.lock, flags); rz_dmac_disable_hw(channel); - for (i = 0; i < DMAC_NR_LMDESC; i++) - lmdesc[i].header = 0; + rz_lmdesc_setup(channel, channel->lmdesc.base); list_splice_tail_init(&channel->ld_active, &channel->ld_free); list_splice_tail_init(&channel->ld_queue, &channel->ld_free); @@ -679,13 +677,185 @@ static void rz_dmac_device_synchronize(struct dma_chan *chan) if (ret < 0) dev_warn(dmac->dev, "DMA Timeout"); - if (dmac->has_icu) { - rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, - channel->index, - RZV2H_ICU_DMAC_REQ_NO_DEFAULT); - } else { - rz_dmac_set_dmars_register(dmac, channel->index, 0); + rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no); +} + +static struct rz_lmdesc * +rz_dmac_get_next_lmdesc(struct rz_lmdesc *base, struct rz_lmdesc *lmdesc) +{ + struct rz_lmdesc *next = ++lmdesc; + + if (next >= base + DMAC_NR_LMDESC) + next = base; + + return next; +} + +static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel, u32 crla) +{ + struct rz_lmdesc *lmdesc = channel->lmdesc.head; + struct dma_chan *chan = &channel->vc.chan; + struct rz_dmac *dmac = to_rz_dmac(chan->device); + u32 residue = 0, i = 0; + + while (lmdesc->nxla != crla) { + lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); + if (++i >= DMAC_NR_LMDESC) + return 0; + } + + /* Calculate residue from next lmdesc to end of virtual desc */ + while (lmdesc->chcfg & CHCFG_DEM) { + residue += lmdesc->tb; + lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); + } + + dev_dbg(dmac->dev, "%s: VD residue is %u\n", __func__, residue); + + return residue; +} + +static u32 rz_dmac_chan_get_residue(struct rz_dmac_chan *channel, + dma_cookie_t cookie) +{ + struct rz_dmac_desc *current_desc, *desc; + enum dma_status status; + u32 crla, crtb, i; + + /* Get current processing virtual descriptor */ + current_desc = list_first_entry(&channel->ld_active, + struct rz_dmac_desc, node); + if (!current_desc) + return 0; + + /* + * If the cookie corresponds to a descriptor that has been completed + * there is no residue. The same check has already been performed by the + * caller but without holding the channel lock, so the descriptor could + * now be complete. + */ + status = dma_cookie_status(&channel->vc.chan, cookie, NULL); + if (status == DMA_COMPLETE) + return 0; + + /* + * If the cookie doesn't correspond to the currently processing virtual + * descriptor then the descriptor hasn't been processed yet, and the + * residue is equal to the full descriptor size. Also, a client driver + * is possible to call this function before rz_dmac_irq_handler_thread() + * runs. In this case, the running descriptor will be the next + * descriptor, and will appear in the done list. So, if the argument + * cookie matches the done list's cookie, we can assume the residue is + * zero. + */ + if (cookie != current_desc->vd.tx.cookie) { + list_for_each_entry(desc, &channel->ld_free, node) { + if (cookie == desc->vd.tx.cookie) + return 0; + } + + list_for_each_entry(desc, &channel->ld_queue, node) { + if (cookie == desc->vd.tx.cookie) + return desc->len; + } + + list_for_each_entry(desc, &channel->ld_active, node) { + if (cookie == desc->vd.tx.cookie) + return desc->len; + } + + /* + * No descriptor found for the cookie, there's thus no residue. + * This shouldn't happen if the calling driver passes a correct + * cookie value. + */ + WARN(1, "No descriptor for cookie!"); + return 0; + } + + /* + * We need to read two registers. Make sure the hardware does not move + * to next lmdesc while reading the current lmdesc. Trying it 3 times + * should be enough: initial read, retry, retry for the paranoid. + */ + for (i = 0; i < 3; i++) { + crla = rz_dmac_ch_readl(channel, CRLA, 1); + crtb = rz_dmac_ch_readl(channel, CRTB, 1); + /* Still the same? */ + if (crla == rz_dmac_ch_readl(channel, CRLA, 1)) + break; + } + + WARN_ONCE(i >= 3, "residue might not be continuous!"); + + /* + * Calculate number of bytes transferred in processing virtual descriptor. + * One virtual descriptor can have many lmdesc. + */ + return crtb + rz_dmac_calculate_residue_bytes_in_vd(channel, crla); +} + +static enum dma_status rz_dmac_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + enum dma_status status; + u32 residue; + + status = dma_cookie_status(chan, cookie, txstate); + if (status == DMA_COMPLETE || !txstate) + return status; + + scoped_guard(spinlock_irqsave, &channel->vc.lock) { + u32 val; + + residue = rz_dmac_chan_get_residue(channel, cookie); + + val = rz_dmac_ch_readl(channel, CHSTAT, 1); + if (val & CHSTAT_SUS) + status = DMA_PAUSED; } + + /* if there's no residue and no paused, the cookie is complete */ + if (!residue && status != DMA_PAUSED) + return DMA_COMPLETE; + + dma_set_residue(txstate, residue); + + return status; +} + +static int rz_dmac_device_pause(struct dma_chan *chan) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + u32 val; + + guard(spinlock_irqsave)(&channel->vc.lock); + + val = rz_dmac_ch_readl(channel, CHSTAT, 1); + if (!(val & CHSTAT_EN)) + return 0; + + rz_dmac_ch_writel(channel, CHCTRL_SETSUS, CHCTRL, 1); + return read_poll_timeout_atomic(rz_dmac_ch_readl, val, + (val & CHSTAT_SUS), 1, 1024, + false, channel, CHSTAT, 1); +} + +static int rz_dmac_device_resume(struct dma_chan *chan) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + u32 val; + + guard(spinlock_irqsave)(&channel->vc.lock); + + /* Do not check CHSTAT_SUS but rely on HW capabilities. */ + + rz_dmac_ch_writel(channel, CHCTRL_CLRSUS, CHCTRL, 1); + return read_poll_timeout_atomic(rz_dmac_ch_readl, val, + !(val & CHSTAT_SUS), 1, 1024, + false, channel, CHSTAT, 1); } /* @@ -697,7 +867,7 @@ static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); - u32 chstat, chctrl; + u32 chstat; chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); if (chstat & CHSTAT_ER) { @@ -706,13 +876,14 @@ static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) scoped_guard(spinlock_irqsave, &channel->vc.lock) rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); - goto done; + return; } - chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1); - rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1); -done: - return; + /* + * No need to lock. This just clears the END interrupt. Writing + * zeros to CHCTRL is just ignored by HW. + */ + rz_dmac_ch_writel(channel, CHCTRL_CLREND, CHCTRL, 1); } static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id) @@ -876,14 +1047,13 @@ static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac) uint32_t dmac_index; int ret; - ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args); - if (ret == -ENOENT) + if (!dmac->info->icu_register_dma_req) return 0; + + ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args); if (ret) return ret; - dmac->has_icu = true; - dmac->icu.pdev = of_find_device_by_node(args.np); of_node_put(args.np); if (!dmac->icu.pdev) { @@ -938,6 +1108,7 @@ static int rz_dmac_probe(struct platform_device *pdev) if (!dmac) return -ENOMEM; + dmac->info = device_get_match_data(&pdev->dev); dmac->dev = &pdev->dev; platform_set_drvdata(pdev, dmac); @@ -955,23 +1126,22 @@ static int rz_dmac_probe(struct platform_device *pdev) if (IS_ERR(dmac->base)) return PTR_ERR(dmac->base); - if (!dmac->has_icu) { + if (!dmac->info->icu_register_dma_req) { dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(dmac->ext_base)) return PTR_ERR(dmac->ext_base); } /* Register interrupt handler for error */ - irq = platform_get_irq_byname(pdev, irqname); - if (irq < 0) - return irq; - - ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, - irqname, NULL); - if (ret) { - dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", - irq, ret); - return ret; + irq = platform_get_irq_byname_optional(pdev, irqname); + if (irq > 0) { + ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, + irqname, NULL); + if (ret) { + dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", + irq, ret); + return ret; + } } /* Initialize the channels. */ @@ -1009,6 +1179,7 @@ static int rz_dmac_probe(struct platform_device *pdev) engine = &dmac->engine; dma_cap_set(DMA_SLAVE, engine->cap_mask); dma_cap_set(DMA_MEMCPY, engine->cap_mask); + engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); @@ -1016,13 +1187,15 @@ static int rz_dmac_probe(struct platform_device *pdev) engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources; engine->device_free_chan_resources = rz_dmac_free_chan_resources; - engine->device_tx_status = dma_cookie_status; + engine->device_tx_status = rz_dmac_tx_status; engine->device_prep_slave_sg = rz_dmac_prep_slave_sg; engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy; engine->device_config = rz_dmac_config; engine->device_terminate_all = rz_dmac_terminate_all; engine->device_issue_pending = rz_dmac_issue_pending; engine->device_synchronize = rz_dmac_device_synchronize; + engine->device_pause = rz_dmac_device_pause; + engine->device_resume = rz_dmac_device_resume; engine->copy_align = DMAENGINE_ALIGN_1_BYTE; dma_set_max_seg_size(engine->dev, U32_MAX); @@ -1076,9 +1249,24 @@ static void rz_dmac_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); } +static const struct rz_dmac_info rz_dmac_v2h_info = { + .icu_register_dma_req = rzv2h_icu_register_dma_req, + .default_dma_req_no = RZV2H_ICU_DMAC_REQ_NO_DEFAULT, +}; + +static const struct rz_dmac_info rz_dmac_t2h_info = { + .icu_register_dma_req = rzt2h_icu_register_dma_req, + .default_dma_req_no = RZT2H_ICU_DMAC_REQ_NO_DEFAULT, +}; + +static const struct rz_dmac_info rz_dmac_generic_info = { + .default_dma_req_no = 0, +}; + static const struct of_device_id of_rz_dmac_match[] = { - { .compatible = "renesas,r9a09g057-dmac", }, - { .compatible = "renesas,rz-dmac", }, + { .compatible = "renesas,r9a09g057-dmac", .data = &rz_dmac_v2h_info }, + { .compatible = "renesas,r9a09g077-dmac", .data = &rz_dmac_t2h_info }, + { .compatible = "renesas,rz-dmac", .data = &rz_dmac_generic_info }, { /* Sentinel */ } }; MODULE_DEVICE_TABLE(of, of_rz_dmac_match); diff --git a/drivers/dma/switchtec_dma.c b/drivers/dma/switchtec_dma.c new file mode 100644 index 000000000000..3ef928640615 --- /dev/null +++ b/drivers/dma/switchtec_dma.c @@ -0,0 +1,1437 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip Switchtec(tm) DMA Controller Driver + * Copyright (c) 2025, Kelvin Cao <kelvin.cao@microchip.com> + * Copyright (c) 2025, Microchip Corporation + */ + +#include <linux/bitfield.h> +#include <linux/circ_buf.h> +#include <linux/dmaengine.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/delay.h> +#include <linux/iopoll.h> + +#include "dmaengine.h" + +MODULE_DESCRIPTION("Switchtec PCIe Switch DMA Engine"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Kelvin Cao"); + +#define SWITCHTEC_DMAC_CHAN_CTRL_OFFSET 0x1000 +#define SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET 0x160000 + +#define SWITCHTEC_DMA_CHAN_HW_REGS_SIZE 0x1000 +#define SWITCHTEC_DMA_CHAN_FW_REGS_SIZE 0x80 + +#define SWITCHTEC_REG_CAP 0x80 +#define SWITCHTEC_REG_CHAN_CNT 0x84 +#define SWITCHTEC_REG_TAG_LIMIT 0x90 +#define SWITCHTEC_REG_CHAN_STS_VEC 0x94 +#define SWITCHTEC_REG_SE_BUF_CNT 0x98 +#define SWITCHTEC_REG_SE_BUF_BASE 0x9a + +#define SWITCHTEC_DESC_MAX_SIZE 0x100000 + +#define SWITCHTEC_CHAN_CTRL_PAUSE BIT(0) +#define SWITCHTEC_CHAN_CTRL_HALT BIT(1) +#define SWITCHTEC_CHAN_CTRL_RESET BIT(2) +#define SWITCHTEC_CHAN_CTRL_ERR_PAUSE BIT(3) + +#define SWITCHTEC_CHAN_STS_PAUSED BIT(9) +#define SWITCHTEC_CHAN_STS_HALTED BIT(10) +#define SWITCHTEC_CHAN_STS_PAUSED_MASK GENMASK(29, 13) + +#define SWITCHTEC_INVALID_HFID 0xffff + +#define SWITCHTEC_DMA_SQ_SIZE SZ_32K +#define SWITCHTEC_DMA_CQ_SIZE SZ_32K + +#define SWITCHTEC_DMA_RING_SIZE SZ_32K + +static const char * const channel_status_str[] = { + [13] = "received a VDM with length error status", + [14] = "received a VDM or Cpl with Unsupported Request error status", + [15] = "received a VDM or Cpl with Completion Abort error status", + [16] = "received a VDM with ECRC error status", + [17] = "received a VDM with EP error status", + [18] = "received a VDM with Reserved Cpl error status", + [19] = "received only part of split SE CplD", + [20] = "the ISP_DMAC detected a Completion Time Out", + [21] = "received a Cpl with Unsupported Request status", + [22] = "received a Cpl with Completion Abort status", + [23] = "received a Cpl with a reserved status", + [24] = "received a TLP with ECRC error status in its metadata", + [25] = "received a TLP with the EP bit set in the header", + [26] = "the ISP_DMAC tried to process a SE with an invalid Connection ID", + [27] = "the ISP_DMAC tried to process a SE with an invalid Remote Host interrupt", + [28] = "a reserved opcode was detected in an SE", + [29] = "received a SE Cpl with error status", +}; + +struct chan_hw_regs { + u16 cq_head; + u16 rsvd1; + u16 sq_tail; + u16 rsvd2; + u8 ctrl; + u8 rsvd3[3]; + u16 status; + u16 rsvd4; +}; + +#define PERF_BURST_SCALE_MASK GENMASK_U32(3, 2) +#define PERF_MRRS_MASK GENMASK_U32(6, 4) +#define PERF_INTERVAL_MASK GENMASK_U32(10, 8) +#define PERF_BURST_SIZE_MASK GENMASK_U32(14, 12) +#define PERF_ARB_WEIGHT_MASK GENMASK_U32(31, 24) + +#define SE_BUF_BASE_MASK GENMASK_U32(10, 2) +#define SE_BUF_LEN_MASK GENMASK_U32(20, 12) +#define SE_THRESH_MASK GENMASK_U32(31, 23) + +#define SWITCHTEC_CHAN_ENABLE BIT(1) + +struct chan_fw_regs { + u32 valid_en_se; + u32 cq_base_lo; + u32 cq_base_hi; + u16 cq_size; + u16 rsvd1; + u32 sq_base_lo; + u32 sq_base_hi; + u16 sq_size; + u16 rsvd2; + u32 int_vec; + u32 perf_cfg; + u32 rsvd3; + u32 perf_latency_selector; + u32 perf_fetched_se_cnt_lo; + u32 perf_fetched_se_cnt_hi; + u32 perf_byte_cnt_lo; + u32 perf_byte_cnt_hi; + u32 rsvd4; + u16 perf_se_pending; + u16 perf_se_buf_empty; + u32 perf_chan_idle; + u32 perf_lat_max; + u32 perf_lat_min; + u32 perf_lat_last; + u16 sq_current; + u16 sq_phase; + u16 cq_current; + u16 cq_phase; +}; + +struct switchtec_dma_chan { + struct switchtec_dma_dev *swdma_dev; + struct dma_chan dma_chan; + struct chan_hw_regs __iomem *mmio_chan_hw; + struct chan_fw_regs __iomem *mmio_chan_fw; + + /* Serialize hardware control register access */ + spinlock_t hw_ctrl_lock; + + struct tasklet_struct desc_task; + + /* Serialize descriptor preparation */ + spinlock_t submit_lock; + bool ring_active; + int cid; + + /* Serialize completion processing */ + spinlock_t complete_lock; + bool comp_ring_active; + + /* channel index and irq */ + int index; + int irq; + + /* + * In driver context, head is advanced by producer while + * tail is advanced by consumer. + */ + + /* the head and tail for both desc_ring and hw_sq */ + int head; + int tail; + int phase_tag; + struct switchtec_dma_hw_se_desc *hw_sq; + dma_addr_t dma_addr_sq; + + /* the tail for hw_cq */ + int cq_tail; + struct switchtec_dma_hw_ce *hw_cq; + dma_addr_t dma_addr_cq; + + struct list_head list; + + struct switchtec_dma_desc *desc_ring[SWITCHTEC_DMA_RING_SIZE]; +}; + +struct switchtec_dma_dev { + struct dma_device dma_dev; + struct pci_dev __rcu *pdev; + void __iomem *bar; + + struct switchtec_dma_chan **swdma_chans; + int chan_cnt; + int chan_status_irq; +}; + +enum chan_op { + ENABLE_CHAN, + DISABLE_CHAN, +}; + +enum switchtec_dma_opcode { + SWITCHTEC_DMA_OPC_MEMCPY = 0, + SWITCHTEC_DMA_OPC_RDIMM = 0x1, + SWITCHTEC_DMA_OPC_WRIMM = 0x2, + SWITCHTEC_DMA_OPC_RHI = 0x6, + SWITCHTEC_DMA_OPC_NOP = 0x7, +}; + +struct switchtec_dma_hw_se_desc { + u8 opc; + u8 ctrl; + __le16 tlp_setting; + __le16 rsvd1; + __le16 cid; + __le32 byte_cnt; + __le32 addr_lo; /* SADDR_LO/WIADDR_LO */ + __le32 addr_hi; /* SADDR_HI/WIADDR_HI */ + __le32 daddr_lo; + __le32 daddr_hi; + __le16 dfid; + __le16 sfid; +}; + +#define SWITCHTEC_SE_DFM BIT(5) +#define SWITCHTEC_SE_LIOF BIT(6) +#define SWITCHTEC_SE_BRR BIT(7) +#define SWITCHTEC_SE_CID_MASK GENMASK(15, 0) + +#define SWITCHTEC_CE_SC_LEN_ERR BIT(0) +#define SWITCHTEC_CE_SC_UR BIT(1) +#define SWITCHTEC_CE_SC_CA BIT(2) +#define SWITCHTEC_CE_SC_RSVD_CPL BIT(3) +#define SWITCHTEC_CE_SC_ECRC_ERR BIT(4) +#define SWITCHTEC_CE_SC_EP_SET BIT(5) +#define SWITCHTEC_CE_SC_D_RD_CTO BIT(8) +#define SWITCHTEC_CE_SC_D_RIMM_UR BIT(9) +#define SWITCHTEC_CE_SC_D_RIMM_CA BIT(10) +#define SWITCHTEC_CE_SC_D_RIMM_RSVD_CPL BIT(11) +#define SWITCHTEC_CE_SC_D_ECRC BIT(12) +#define SWITCHTEC_CE_SC_D_EP_SET BIT(13) +#define SWITCHTEC_CE_SC_D_BAD_CONNID BIT(14) +#define SWITCHTEC_CE_SC_D_BAD_RHI_ADDR BIT(15) +#define SWITCHTEC_CE_SC_D_INVD_CMD BIT(16) +#define SWITCHTEC_CE_SC_MASK GENMASK(16, 0) + +struct switchtec_dma_hw_ce { + __le32 rdimm_cpl_dw0; + __le32 rdimm_cpl_dw1; + __le32 rsvd1; + __le32 cpl_byte_cnt; + __le16 sq_head; + __le16 rsvd2; + __le32 rsvd3; + __le32 sts_code; + __le16 cid; + __le16 phase_tag; +}; + +struct switchtec_dma_desc { + struct dma_async_tx_descriptor txd; + struct switchtec_dma_hw_se_desc *hw; + u32 orig_size; + bool completed; +}; + +static int wait_for_chan_status(struct chan_hw_regs __iomem *chan_hw, u32 mask, + bool set) +{ + u32 status; + + return readl_poll_timeout_atomic(&chan_hw->status, status, + (set && (status & mask)) || + (!set && !(status & mask)), + 10, 100 * USEC_PER_MSEC); +} + +static int halt_channel(struct switchtec_dma_chan *swdma_chan) +{ + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + int ret; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + ret = -ENODEV; + goto unlock_and_exit; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + writeb(SWITCHTEC_CHAN_CTRL_HALT, &chan_hw->ctrl); + ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, true); + spin_unlock(&swdma_chan->hw_ctrl_lock); + +unlock_and_exit: + rcu_read_unlock(); + return ret; +} + +static int unhalt_channel(struct switchtec_dma_chan *swdma_chan) +{ + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + u8 ctrl; + int ret; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + ret = -ENODEV; + goto unlock_and_exit; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + ctrl = readb(&chan_hw->ctrl); + ctrl &= ~SWITCHTEC_CHAN_CTRL_HALT; + writeb(ctrl, &chan_hw->ctrl); + ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, false); + spin_unlock(&swdma_chan->hw_ctrl_lock); + +unlock_and_exit: + rcu_read_unlock(); + return ret; +} + +static void flush_pci_write(struct chan_hw_regs __iomem *chan_hw) +{ + readl(&chan_hw->cq_head); +} + +static int reset_channel(struct switchtec_dma_chan *swdma_chan) +{ + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + rcu_read_unlock(); + return -ENODEV; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + writel(SWITCHTEC_CHAN_CTRL_RESET | SWITCHTEC_CHAN_CTRL_ERR_PAUSE, + &chan_hw->ctrl); + flush_pci_write(chan_hw); + + udelay(1000); + + writel(SWITCHTEC_CHAN_CTRL_ERR_PAUSE, &chan_hw->ctrl); + spin_unlock(&swdma_chan->hw_ctrl_lock); + flush_pci_write(chan_hw); + + rcu_read_unlock(); + return 0; +} + +static int pause_reset_channel(struct switchtec_dma_chan *swdma_chan) +{ + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + rcu_read_unlock(); + return -ENODEV; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl); + spin_unlock(&swdma_chan->hw_ctrl_lock); + + flush_pci_write(chan_hw); + + rcu_read_unlock(); + + /* wait 60ms to ensure no pending CEs */ + mdelay(60); + + return reset_channel(swdma_chan); +} + +static int channel_op(struct switchtec_dma_chan *swdma_chan, int op) +{ + struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw; + struct pci_dev *pdev; + u32 valid_en_se; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + rcu_read_unlock(); + return -ENODEV; + } + + valid_en_se = readl(&chan_fw->valid_en_se); + if (op == ENABLE_CHAN) + valid_en_se |= SWITCHTEC_CHAN_ENABLE; + else + valid_en_se &= ~SWITCHTEC_CHAN_ENABLE; + + writel(valid_en_se, &chan_fw->valid_en_se); + + rcu_read_unlock(); + return 0; +} + +static int enable_channel(struct switchtec_dma_chan *swdma_chan) +{ + return channel_op(swdma_chan, ENABLE_CHAN); +} + +static int disable_channel(struct switchtec_dma_chan *swdma_chan) +{ + return channel_op(swdma_chan, DISABLE_CHAN); +} + +static void +switchtec_dma_cleanup_completed(struct switchtec_dma_chan *swdma_chan) +{ + struct device *chan_dev = &swdma_chan->dma_chan.dev->device; + struct switchtec_dma_desc *desc; + struct switchtec_dma_hw_ce *ce; + struct dmaengine_result res; + int tail, cid, se_idx, i; + __le16 phase_tag; + u32 sts_code; + __le32 *p; + + do { + spin_lock_bh(&swdma_chan->complete_lock); + if (!swdma_chan->comp_ring_active) { + spin_unlock_bh(&swdma_chan->complete_lock); + break; + } + + ce = &swdma_chan->hw_cq[swdma_chan->cq_tail]; + /* + * phase_tag is updated by hardware, ensure the value is + * not from the cache + */ + phase_tag = smp_load_acquire(&ce->phase_tag); + if (le16_to_cpu(phase_tag) == swdma_chan->phase_tag) { + spin_unlock_bh(&swdma_chan->complete_lock); + break; + } + + cid = le16_to_cpu(ce->cid); + se_idx = cid & (SWITCHTEC_DMA_SQ_SIZE - 1); + desc = swdma_chan->desc_ring[se_idx]; + + tail = swdma_chan->tail; + + res.residue = desc->orig_size - le32_to_cpu(ce->cpl_byte_cnt); + + sts_code = le32_to_cpu(ce->sts_code); + + if (!(sts_code & SWITCHTEC_CE_SC_MASK)) { + res.result = DMA_TRANS_NOERROR; + } else { + if (sts_code & SWITCHTEC_CE_SC_D_RD_CTO) + res.result = DMA_TRANS_READ_FAILED; + else + res.result = DMA_TRANS_WRITE_FAILED; + + dev_err(chan_dev, "CID 0x%04x failed, SC 0x%08x\n", cid, + (u32)(sts_code & SWITCHTEC_CE_SC_MASK)); + + p = (__le32 *)ce; + for (i = 0; i < sizeof(*ce) / 4; i++) { + dev_err(chan_dev, "CE DW%d: 0x%08x\n", i, + le32_to_cpu(*p)); + p++; + } + } + + desc->completed = true; + + swdma_chan->cq_tail++; + swdma_chan->cq_tail &= SWITCHTEC_DMA_CQ_SIZE - 1; + + rcu_read_lock(); + if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) { + rcu_read_unlock(); + spin_unlock_bh(&swdma_chan->complete_lock); + return; + } + writew(swdma_chan->cq_tail, &swdma_chan->mmio_chan_hw->cq_head); + rcu_read_unlock(); + + if (swdma_chan->cq_tail == 0) + swdma_chan->phase_tag = !swdma_chan->phase_tag; + + /* Out of order CE */ + if (se_idx != tail) { + spin_unlock_bh(&swdma_chan->complete_lock); + continue; + } + + do { + dma_cookie_complete(&desc->txd); + dma_descriptor_unmap(&desc->txd); + dmaengine_desc_get_callback_invoke(&desc->txd, &res); + desc->txd.callback = NULL; + desc->txd.callback_result = NULL; + desc->completed = false; + + tail++; + tail &= SWITCHTEC_DMA_SQ_SIZE - 1; + + /* + * Ensure the desc updates are visible before updating + * the tail index + */ + smp_store_release(&swdma_chan->tail, tail); + desc = swdma_chan->desc_ring[swdma_chan->tail]; + if (!desc->completed) + break; + } while (CIRC_CNT(READ_ONCE(swdma_chan->head), swdma_chan->tail, + SWITCHTEC_DMA_SQ_SIZE)); + + spin_unlock_bh(&swdma_chan->complete_lock); + } while (1); +} + +static void +switchtec_dma_abort_desc(struct switchtec_dma_chan *swdma_chan, int force) +{ + struct switchtec_dma_desc *desc; + struct dmaengine_result res; + + if (!force) + switchtec_dma_cleanup_completed(swdma_chan); + + spin_lock_bh(&swdma_chan->complete_lock); + + while (CIRC_CNT(swdma_chan->head, swdma_chan->tail, + SWITCHTEC_DMA_SQ_SIZE) >= 1) { + desc = swdma_chan->desc_ring[swdma_chan->tail]; + + res.residue = desc->orig_size; + res.result = DMA_TRANS_ABORTED; + + dma_cookie_complete(&desc->txd); + dma_descriptor_unmap(&desc->txd); + if (!force) + dmaengine_desc_get_callback_invoke(&desc->txd, &res); + desc->txd.callback = NULL; + desc->txd.callback_result = NULL; + + swdma_chan->tail++; + swdma_chan->tail &= SWITCHTEC_DMA_SQ_SIZE - 1; + } + + spin_unlock_bh(&swdma_chan->complete_lock); +} + +static void switchtec_dma_chan_stop(struct switchtec_dma_chan *swdma_chan) +{ + int rc; + + rc = halt_channel(swdma_chan); + if (rc) + return; + + rcu_read_lock(); + if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) { + rcu_read_unlock(); + return; + } + + writel(0, &swdma_chan->mmio_chan_fw->sq_base_lo); + writel(0, &swdma_chan->mmio_chan_fw->sq_base_hi); + writel(0, &swdma_chan->mmio_chan_fw->cq_base_lo); + writel(0, &swdma_chan->mmio_chan_fw->cq_base_hi); + + rcu_read_unlock(); +} + +static int switchtec_dma_terminate_all(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + + spin_lock_bh(&swdma_chan->complete_lock); + swdma_chan->comp_ring_active = false; + spin_unlock_bh(&swdma_chan->complete_lock); + + return pause_reset_channel(swdma_chan); +} + +static void switchtec_dma_synchronize(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + + int rc; + + switchtec_dma_abort_desc(swdma_chan, 1); + + rc = enable_channel(swdma_chan); + if (rc) + return; + + rc = reset_channel(swdma_chan); + if (rc) + return; + + rc = unhalt_channel(swdma_chan); + if (rc) + return; + + spin_lock_bh(&swdma_chan->submit_lock); + swdma_chan->head = 0; + spin_unlock_bh(&swdma_chan->submit_lock); + + spin_lock_bh(&swdma_chan->complete_lock); + swdma_chan->comp_ring_active = true; + swdma_chan->phase_tag = 0; + swdma_chan->tail = 0; + swdma_chan->cq_tail = 0; + swdma_chan->cid = 0; + dma_cookie_init(chan); + spin_unlock_bh(&swdma_chan->complete_lock); +} + +static struct dma_async_tx_descriptor * +switchtec_dma_prep_desc(struct dma_chan *c, u16 dst_fid, dma_addr_t dma_dst, + u16 src_fid, dma_addr_t dma_src, u64 data, + size_t len, unsigned long flags) + __acquires(swdma_chan->submit_lock) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(c, struct switchtec_dma_chan, dma_chan); + struct switchtec_dma_desc *desc; + int head, tail; + + spin_lock_bh(&swdma_chan->submit_lock); + + if (!swdma_chan->ring_active) + goto err_unlock; + + tail = READ_ONCE(swdma_chan->tail); + head = swdma_chan->head; + + if (!CIRC_SPACE(head, tail, SWITCHTEC_DMA_RING_SIZE)) + goto err_unlock; + + desc = swdma_chan->desc_ring[head]; + + if (src_fid != SWITCHTEC_INVALID_HFID && + dst_fid != SWITCHTEC_INVALID_HFID) + desc->hw->ctrl |= SWITCHTEC_SE_DFM; + + if (flags & DMA_PREP_INTERRUPT) + desc->hw->ctrl |= SWITCHTEC_SE_LIOF; + + if (flags & DMA_PREP_FENCE) + desc->hw->ctrl |= SWITCHTEC_SE_BRR; + + desc->txd.flags = flags; + + desc->completed = false; + desc->hw->opc = SWITCHTEC_DMA_OPC_MEMCPY; + desc->hw->addr_lo = cpu_to_le32(lower_32_bits(dma_src)); + desc->hw->addr_hi = cpu_to_le32(upper_32_bits(dma_src)); + desc->hw->daddr_lo = cpu_to_le32(lower_32_bits(dma_dst)); + desc->hw->daddr_hi = cpu_to_le32(upper_32_bits(dma_dst)); + desc->hw->byte_cnt = cpu_to_le32(len); + desc->hw->tlp_setting = 0; + desc->hw->dfid = cpu_to_le16(dst_fid); + desc->hw->sfid = cpu_to_le16(src_fid); + swdma_chan->cid &= SWITCHTEC_SE_CID_MASK; + desc->hw->cid = cpu_to_le16(swdma_chan->cid++); + desc->orig_size = len; + + /* return with the lock held, it will be released in tx_submit */ + + return &desc->txd; + +err_unlock: + /* + * Keep sparse happy by restoring an even lock count on + * this lock. + */ + __acquire(swdma_chan->submit_lock); + + spin_unlock_bh(&swdma_chan->submit_lock); + return NULL; +} + +static struct dma_async_tx_descriptor * +switchtec_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dst, + dma_addr_t dma_src, size_t len, unsigned long flags) + __acquires(swdma_chan->submit_lock) +{ + if (len > SWITCHTEC_DESC_MAX_SIZE) { + /* + * Keep sparse happy by restoring an even lock count on + * this lock. + */ + __acquire(swdma_chan->submit_lock); + return NULL; + } + + return switchtec_dma_prep_desc(c, SWITCHTEC_INVALID_HFID, dma_dst, + SWITCHTEC_INVALID_HFID, dma_src, 0, len, + flags); +} + +static dma_cookie_t +switchtec_dma_tx_submit(struct dma_async_tx_descriptor *desc) + __releases(swdma_chan->submit_lock) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(desc->chan, struct switchtec_dma_chan, dma_chan); + dma_cookie_t cookie; + int head; + + head = swdma_chan->head + 1; + head &= SWITCHTEC_DMA_RING_SIZE - 1; + + /* + * Ensure the desc updates are visible before updating the head index + */ + smp_store_release(&swdma_chan->head, head); + + cookie = dma_cookie_assign(desc); + + spin_unlock_bh(&swdma_chan->submit_lock); + + return cookie; +} + +static enum dma_status switchtec_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + enum dma_status ret; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + /* + * For jobs where the interrupts are disabled, this is the only place + * to process the completions returned by the hardware. Callers that + * disable interrupts must call tx_status() to determine when a job + * is done, so it is safe to process completions here. If a job has + * interrupts enabled, then the completions will normally be processed + * in the tasklet that is triggered by the interrupt and tx_status() + * does not need to be called. + */ + switchtec_dma_cleanup_completed(swdma_chan); + + return dma_cookie_status(chan, cookie, txstate); +} + +static void switchtec_dma_issue_pending(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; + + /* + * The sq_tail register is actually for the head of the + * submisssion queue. Chip has the opposite define of head/tail + * to the Linux kernel. + */ + + rcu_read_lock(); + if (!rcu_dereference(swdma_dev->pdev)) { + rcu_read_unlock(); + return; + } + + spin_lock_bh(&swdma_chan->submit_lock); + writew(swdma_chan->head, &swdma_chan->mmio_chan_hw->sq_tail); + spin_unlock_bh(&swdma_chan->submit_lock); + + rcu_read_unlock(); +} + +static int switchtec_dma_pause(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + int ret; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + ret = -ENODEV; + goto unlock_and_exit; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl); + ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_PAUSED, true); + spin_unlock(&swdma_chan->hw_ctrl_lock); + +unlock_and_exit: + rcu_read_unlock(); + return ret; +} + +static int switchtec_dma_resume(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + int ret; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + ret = -ENODEV; + goto unlock_and_exit; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + writeb(0, &chan_hw->ctrl); + ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_PAUSED, false); + spin_unlock(&swdma_chan->hw_ctrl_lock); + +unlock_and_exit: + rcu_read_unlock(); + return ret; +} + +static void switchtec_dma_desc_task(unsigned long data) +{ + struct switchtec_dma_chan *swdma_chan = (void *)data; + + switchtec_dma_cleanup_completed(swdma_chan); +} + +static irqreturn_t switchtec_dma_isr(int irq, void *chan) +{ + struct switchtec_dma_chan *swdma_chan = chan; + + if (swdma_chan->comp_ring_active) + tasklet_schedule(&swdma_chan->desc_task); + + return IRQ_HANDLED; +} + +static irqreturn_t switchtec_dma_chan_status_isr(int irq, void *dma) +{ + struct switchtec_dma_dev *swdma_dev = dma; + struct dma_device *dma_dev = &swdma_dev->dma_dev; + struct switchtec_dma_chan *swdma_chan; + struct chan_hw_regs __iomem *chan_hw; + struct device *chan_dev; + struct dma_chan *chan; + u32 chan_status; + int bit; + + list_for_each_entry(chan, &dma_dev->channels, device_node) { + swdma_chan = container_of(chan, struct switchtec_dma_chan, + dma_chan); + chan_dev = &swdma_chan->dma_chan.dev->device; + chan_hw = swdma_chan->mmio_chan_hw; + + rcu_read_lock(); + if (!rcu_dereference(swdma_dev->pdev)) { + rcu_read_unlock(); + goto out; + } + + chan_status = readl(&chan_hw->status); + chan_status &= SWITCHTEC_CHAN_STS_PAUSED_MASK; + rcu_read_unlock(); + + bit = ffs(chan_status); + if (!bit) + dev_dbg(chan_dev, "No pause bit set.\n"); + else + dev_err(chan_dev, "Paused, %s\n", + channel_status_str[bit - 1]); + } + +out: + return IRQ_HANDLED; +} + +static void switchtec_dma_free_desc(struct switchtec_dma_chan *swdma_chan) +{ + struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; + size_t size; + int i; + + size = SWITCHTEC_DMA_SQ_SIZE * sizeof(*swdma_chan->hw_sq); + if (swdma_chan->hw_sq) + dma_free_coherent(swdma_dev->dma_dev.dev, size, + swdma_chan->hw_sq, swdma_chan->dma_addr_sq); + + size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq); + if (swdma_chan->hw_cq) + dma_free_coherent(swdma_dev->dma_dev.dev, size, + swdma_chan->hw_cq, swdma_chan->dma_addr_cq); + + for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++) + kfree(swdma_chan->desc_ring[i]); +} + +static int switchtec_dma_alloc_desc(struct switchtec_dma_chan *swdma_chan) +{ + struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; + struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw; + struct switchtec_dma_desc *desc; + struct pci_dev *pdev; + size_t size; + int rc, i; + + swdma_chan->head = 0; + swdma_chan->tail = 0; + swdma_chan->cq_tail = 0; + + size = SWITCHTEC_DMA_SQ_SIZE * sizeof(*swdma_chan->hw_sq); + swdma_chan->hw_sq = dma_alloc_coherent(swdma_dev->dma_dev.dev, size, + &swdma_chan->dma_addr_sq, + GFP_NOWAIT); + if (!swdma_chan->hw_sq) { + rc = -ENOMEM; + goto free_and_exit; + } + + size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq); + swdma_chan->hw_cq = dma_alloc_coherent(swdma_dev->dma_dev.dev, size, + &swdma_chan->dma_addr_cq, + GFP_NOWAIT); + if (!swdma_chan->hw_cq) { + rc = -ENOMEM; + goto free_and_exit; + } + + /* reset host phase tag */ + swdma_chan->phase_tag = 0; + + for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++) { + desc = kzalloc_obj(*desc, GFP_NOWAIT); + if (!desc) { + rc = -ENOMEM; + goto free_and_exit; + } + + dma_async_tx_descriptor_init(&desc->txd, &swdma_chan->dma_chan); + desc->txd.tx_submit = switchtec_dma_tx_submit; + desc->hw = &swdma_chan->hw_sq[i]; + desc->completed = true; + + swdma_chan->desc_ring[i] = desc; + } + + rcu_read_lock(); + pdev = rcu_dereference(swdma_dev->pdev); + if (!pdev) { + rcu_read_unlock(); + rc = -ENODEV; + goto free_and_exit; + } + + /* set sq/cq */ + writel(lower_32_bits(swdma_chan->dma_addr_sq), &chan_fw->sq_base_lo); + writel(upper_32_bits(swdma_chan->dma_addr_sq), &chan_fw->sq_base_hi); + writel(lower_32_bits(swdma_chan->dma_addr_cq), &chan_fw->cq_base_lo); + writel(upper_32_bits(swdma_chan->dma_addr_cq), &chan_fw->cq_base_hi); + + writew(SWITCHTEC_DMA_SQ_SIZE, &swdma_chan->mmio_chan_fw->sq_size); + writew(SWITCHTEC_DMA_CQ_SIZE, &swdma_chan->mmio_chan_fw->cq_size); + + rcu_read_unlock(); + return 0; + +free_and_exit: + switchtec_dma_free_desc(swdma_chan); + return rc; +} + +static int switchtec_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; + u32 perf_cfg; + int rc; + + rc = switchtec_dma_alloc_desc(swdma_chan); + if (rc) + return rc; + + rc = enable_channel(swdma_chan); + if (rc) + return rc; + + rc = reset_channel(swdma_chan); + if (rc) + return rc; + + rc = unhalt_channel(swdma_chan); + if (rc) + return rc; + + swdma_chan->ring_active = true; + swdma_chan->comp_ring_active = true; + swdma_chan->cid = 0; + + dma_cookie_init(chan); + + rcu_read_lock(); + if (!rcu_dereference(swdma_dev->pdev)) { + rcu_read_unlock(); + return -ENODEV; + } + + perf_cfg = readl(&swdma_chan->mmio_chan_fw->perf_cfg); + rcu_read_unlock(); + + dev_dbg(&chan->dev->device, "Burst Size: 0x%x\n", + FIELD_GET(PERF_BURST_SIZE_MASK, perf_cfg)); + + dev_dbg(&chan->dev->device, "Burst Scale: 0x%x\n", + FIELD_GET(PERF_BURST_SCALE_MASK, perf_cfg)); + + dev_dbg(&chan->dev->device, "Interval: 0x%x\n", + FIELD_GET(PERF_INTERVAL_MASK, perf_cfg)); + + dev_dbg(&chan->dev->device, "Arb Weight: 0x%x\n", + FIELD_GET(PERF_ARB_WEIGHT_MASK, perf_cfg)); + + dev_dbg(&chan->dev->device, "MRRS: 0x%x\n", + FIELD_GET(PERF_MRRS_MASK, perf_cfg)); + + return SWITCHTEC_DMA_SQ_SIZE; +} + +static void switchtec_dma_free_chan_resources(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + + spin_lock_bh(&swdma_chan->submit_lock); + swdma_chan->ring_active = false; + spin_unlock_bh(&swdma_chan->submit_lock); + + spin_lock_bh(&swdma_chan->complete_lock); + swdma_chan->comp_ring_active = false; + spin_unlock_bh(&swdma_chan->complete_lock); + + switchtec_dma_chan_stop(swdma_chan); + switchtec_dma_abort_desc(swdma_chan, 0); + switchtec_dma_free_desc(swdma_chan); + + disable_channel(swdma_chan); +} + +static int switchtec_dma_chan_init(struct switchtec_dma_dev *swdma_dev, + struct pci_dev *pdev, int i) +{ + struct dma_device *dma = &swdma_dev->dma_dev; + struct switchtec_dma_chan *swdma_chan; + u32 valid_en_se, thresh; + int se_buf_len, irq, rc; + struct dma_chan *chan; + + swdma_chan = kzalloc_obj(*swdma_chan, GFP_KERNEL); + if (!swdma_chan) + return -ENOMEM; + + swdma_chan->phase_tag = 0; + swdma_chan->index = i; + swdma_chan->swdma_dev = swdma_dev; + + spin_lock_init(&swdma_chan->hw_ctrl_lock); + spin_lock_init(&swdma_chan->submit_lock); + spin_lock_init(&swdma_chan->complete_lock); + tasklet_init(&swdma_chan->desc_task, switchtec_dma_desc_task, + (unsigned long)swdma_chan); + + swdma_chan->mmio_chan_fw = + swdma_dev->bar + SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET + + i * SWITCHTEC_DMA_CHAN_FW_REGS_SIZE; + swdma_chan->mmio_chan_hw = + swdma_dev->bar + SWITCHTEC_DMAC_CHAN_CTRL_OFFSET + + i * SWITCHTEC_DMA_CHAN_HW_REGS_SIZE; + + swdma_dev->swdma_chans[i] = swdma_chan; + + rc = pause_reset_channel(swdma_chan); + if (rc) + goto free_and_exit; + + /* init perf tuner */ + writel(FIELD_PREP(PERF_BURST_SCALE_MASK, 1) | + FIELD_PREP(PERF_MRRS_MASK, 3) | + FIELD_PREP(PERF_BURST_SIZE_MASK, 6) | + FIELD_PREP(PERF_ARB_WEIGHT_MASK, 1), + &swdma_chan->mmio_chan_fw->perf_cfg); + + valid_en_se = readl(&swdma_chan->mmio_chan_fw->valid_en_se); + + dev_dbg(&pdev->dev, "Channel %d: SE buffer base %d\n", i, + FIELD_GET(SE_BUF_BASE_MASK, valid_en_se)); + + se_buf_len = FIELD_GET(SE_BUF_LEN_MASK, valid_en_se); + dev_dbg(&pdev->dev, "Channel %d: SE buffer count %d\n", i, se_buf_len); + + thresh = se_buf_len / 2; + valid_en_se |= FIELD_GET(SE_THRESH_MASK, thresh); + writel(valid_en_se, &swdma_chan->mmio_chan_fw->valid_en_se); + + /* request irqs */ + irq = readl(&swdma_chan->mmio_chan_fw->int_vec); + dev_dbg(&pdev->dev, "Channel %d: CE irq vector %d\n", i, irq); + + rc = pci_request_irq(pdev, irq, switchtec_dma_isr, NULL, swdma_chan, + KBUILD_MODNAME); + if (rc) + goto free_and_exit; + + swdma_chan->irq = irq; + + chan = &swdma_chan->dma_chan; + chan->device = dma; + dma_cookie_init(chan); + + list_add_tail(&chan->device_node, &dma->channels); + + return 0; + +free_and_exit: + kfree(swdma_chan); + return rc; +} + +static int switchtec_dma_chan_free(struct pci_dev *pdev, + struct switchtec_dma_chan *swdma_chan) +{ + spin_lock_bh(&swdma_chan->submit_lock); + swdma_chan->ring_active = false; + spin_unlock_bh(&swdma_chan->submit_lock); + + spin_lock_bh(&swdma_chan->complete_lock); + swdma_chan->comp_ring_active = false; + spin_unlock_bh(&swdma_chan->complete_lock); + + pci_free_irq(pdev, swdma_chan->irq, swdma_chan); + tasklet_kill(&swdma_chan->desc_task); + + switchtec_dma_chan_stop(swdma_chan); + + return 0; +} + +static int switchtec_dma_chans_release(struct pci_dev *pdev, + struct switchtec_dma_dev *swdma_dev) +{ + int i; + + for (i = 0; i < swdma_dev->chan_cnt; i++) + switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]); + + return 0; +} + +static int switchtec_dma_chans_enumerate(struct switchtec_dma_dev *swdma_dev, + struct pci_dev *pdev, int chan_cnt) +{ + struct dma_device *dma = &swdma_dev->dma_dev; + int base, cnt, rc, i; + + swdma_dev->swdma_chans = kcalloc(chan_cnt, sizeof(*swdma_dev->swdma_chans), + GFP_KERNEL); + + if (!swdma_dev->swdma_chans) + return -ENOMEM; + + base = readw(swdma_dev->bar + SWITCHTEC_REG_SE_BUF_BASE); + cnt = readw(swdma_dev->bar + SWITCHTEC_REG_SE_BUF_CNT); + + dev_dbg(&pdev->dev, "EP SE buffer base %d\n", base); + dev_dbg(&pdev->dev, "EP SE buffer count %d\n", cnt); + + INIT_LIST_HEAD(&dma->channels); + + for (i = 0; i < chan_cnt; i++) { + rc = switchtec_dma_chan_init(swdma_dev, pdev, i); + if (rc) { + dev_err(&pdev->dev, "Channel %d: init channel failed\n", + i); + chan_cnt = i; + goto err_exit; + } + } + + return chan_cnt; + +err_exit: + for (i = 0; i < chan_cnt; i++) + switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]); + + kfree(swdma_dev->swdma_chans); + + return rc; +} + +static void switchtec_dma_release(struct dma_device *dma_dev) +{ + struct switchtec_dma_dev *swdma_dev = + container_of(dma_dev, struct switchtec_dma_dev, dma_dev); + int i; + + for (i = 0; i < swdma_dev->chan_cnt; i++) + kfree(swdma_dev->swdma_chans[i]); + + kfree(swdma_dev->swdma_chans); + + put_device(dma_dev->dev); + kfree(swdma_dev); +} + +static int switchtec_dma_create(struct pci_dev *pdev) +{ + struct switchtec_dma_dev *swdma_dev; + int chan_cnt, nr_vecs, irq, rc; + struct dma_device *dma; + struct dma_chan *chan; + + /* + * Create the switchtec dma device + */ + swdma_dev = kzalloc_obj(*swdma_dev, GFP_KERNEL); + if (!swdma_dev) + return -ENOMEM; + + swdma_dev->bar = ioremap(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); + + RCU_INIT_POINTER(swdma_dev->pdev, pdev); + + nr_vecs = pci_msix_vec_count(pdev); + rc = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX); + if (rc < 0) + goto err_exit; + + irq = readw(swdma_dev->bar + SWITCHTEC_REG_CHAN_STS_VEC); + pci_dbg(pdev, "Channel pause irq vector %d\n", irq); + + rc = pci_request_irq(pdev, irq, NULL, switchtec_dma_chan_status_isr, + swdma_dev, KBUILD_MODNAME); + if (rc) + goto err_exit; + + swdma_dev->chan_status_irq = irq; + + chan_cnt = readl(swdma_dev->bar + SWITCHTEC_REG_CHAN_CNT); + if (!chan_cnt) { + pci_err(pdev, "No channel configured.\n"); + rc = -ENXIO; + goto err_exit; + } + + chan_cnt = switchtec_dma_chans_enumerate(swdma_dev, pdev, chan_cnt); + if (chan_cnt < 0) { + pci_err(pdev, "Failed to enumerate dma channels: %d\n", + chan_cnt); + rc = -ENXIO; + goto err_exit; + } + + swdma_dev->chan_cnt = chan_cnt; + + dma = &swdma_dev->dma_dev; + dma->copy_align = DMAENGINE_ALIGN_8_BYTES; + dma_cap_set(DMA_MEMCPY, dma->cap_mask); + dma_cap_set(DMA_PRIVATE, dma->cap_mask); + dma->dev = get_device(&pdev->dev); + + dma->device_alloc_chan_resources = switchtec_dma_alloc_chan_resources; + dma->device_free_chan_resources = switchtec_dma_free_chan_resources; + dma->device_prep_dma_memcpy = switchtec_dma_prep_memcpy; + dma->device_tx_status = switchtec_dma_tx_status; + dma->device_issue_pending = switchtec_dma_issue_pending; + dma->device_pause = switchtec_dma_pause; + dma->device_resume = switchtec_dma_resume; + dma->device_terminate_all = switchtec_dma_terminate_all; + dma->device_synchronize = switchtec_dma_synchronize; + dma->device_release = switchtec_dma_release; + + rc = dma_async_device_register(dma); + if (rc) { + pci_err(pdev, "Failed to register dma device: %d\n", rc); + goto err_chans_release_exit; + } + + pci_dbg(pdev, "Channel count: %d\n", chan_cnt); + + list_for_each_entry(chan, &dma->channels, device_node) + pci_dbg(pdev, "%s\n", dma_chan_name(chan)); + + pci_set_drvdata(pdev, swdma_dev); + + return 0; + +err_chans_release_exit: + switchtec_dma_chans_release(pdev, swdma_dev); + +err_exit: + if (swdma_dev->chan_status_irq) + free_irq(swdma_dev->chan_status_irq, swdma_dev); + + iounmap(swdma_dev->bar); + kfree(swdma_dev); + return rc; +} + +static int switchtec_dma_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + int rc; + + rc = pci_enable_device(pdev); + if (rc) + return rc; + + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + + rc = pci_request_mem_regions(pdev, KBUILD_MODNAME); + if (rc) + goto err_disable; + + pci_set_master(pdev); + + rc = switchtec_dma_create(pdev); + if (rc) + goto err_free; + + return 0; + +err_free: + pci_free_irq_vectors(pdev); + pci_release_mem_regions(pdev); + +err_disable: + pci_disable_device(pdev); + + return rc; +} + +static void switchtec_dma_remove(struct pci_dev *pdev) +{ + struct switchtec_dma_dev *swdma_dev = pci_get_drvdata(pdev); + + switchtec_dma_chans_release(pdev, swdma_dev); + + rcu_assign_pointer(swdma_dev->pdev, NULL); + synchronize_rcu(); + + pci_free_irq(pdev, swdma_dev->chan_status_irq, swdma_dev); + + pci_free_irq_vectors(pdev); + + dma_async_device_unregister(&swdma_dev->dma_dev); + + iounmap(swdma_dev->bar); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +/* + * Also use the class code to identify the devices, as some of the + * device IDs are also used for other devices with other classes by + * Microsemi. + */ +#define SW_ID(vendor_id, device_id) \ + { \ + .vendor = vendor_id, \ + .device = device_id, \ + .subvendor = PCI_ANY_ID, \ + .subdevice = PCI_ANY_ID, \ + .class = PCI_CLASS_SYSTEM_OTHER << 8, \ + .class_mask = 0xffffffff, \ + } + +static const struct pci_device_id switchtec_dma_pci_tbl[] = { + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4000), /* PFX 100XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4084), /* PFX 84XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4068), /* PFX 68XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4052), /* PFX 52XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4036), /* PFX 36XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4028), /* PFX 28XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4100), /* PSX 100XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4184), /* PSX 84XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4168), /* PSX 68XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4152), /* PSX 52XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4136), /* PSX 36XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4128), /* PSX 28XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4352), /* PFXA 52XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4336), /* PFXA 36XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4328), /* PFXA 28XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4452), /* PSXA 52XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4436), /* PSXA 36XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4428), /* PSXA 28XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5000), /* PFX 100XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5084), /* PFX 84XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5068), /* PFX 68XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5052), /* PFX 52XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5036), /* PFX 36XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5028), /* PFX 28XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5100), /* PSX 100XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5184), /* PSX 84XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5168), /* PSX 68XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5152), /* PSX 52XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5136), /* PSX 36XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5128), /* PSX 28XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5300), /* PFXA 100XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5384), /* PFXA 84XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5368), /* PFXA 68XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5352), /* PFXA 52XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5336), /* PFXA 36XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5328), /* PFXA 28XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5400), /* PSXA 100XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5484), /* PSXA 84XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5468), /* PSXA 68XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5452), /* PSXA 52XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5436), /* PSXA 36XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5428), /* PSXA 28XG5 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1001), /* PCI1001 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1002), /* PCI1002 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1003), /* PCI1003 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1004), /* PCI1004 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1005), /* PCI1005 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1006), /* PCI1006 16XG4 */ + {0} +}; +MODULE_DEVICE_TABLE(pci, switchtec_dma_pci_tbl); + +static struct pci_driver switchtec_dma_pci_driver = { + .name = KBUILD_MODNAME, + .id_table = switchtec_dma_pci_tbl, + .probe = switchtec_dma_probe, + .remove = switchtec_dma_remove, +}; +module_pci_driver(switchtec_dma_pci_driver); diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 782a55edc55b..90a22a730cc9 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -61,6 +61,8 @@ struct xdma_desc_block { * @dir: Transferring direction of the channel * @cfg: Transferring config of the channel * @irq: IRQ assigned to the channel + * @last_interrupt: task for comppleting last interrupt + * @stop_requested: stop request flag */ struct xdma_chan { struct virt_dma_chan vchan; diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index e3a18ee42aa2..404235c17353 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -3194,7 +3194,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) = axivdma_clk_init; struct device_node *node = pdev->dev.of_node; struct xilinx_dma_device *xdev; - struct device_node *child, *np = pdev->dev.of_node; + struct device_node *np = pdev->dev.of_node; u32 num_frames, addr_width = XILINX_DMA_DFAULT_ADDRWIDTH, len_width; int i, err; @@ -3334,12 +3334,10 @@ static int xilinx_dma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, xdev); /* Initialize the channels */ - for_each_child_of_node(node, child) { + for_each_child_of_node_scoped(node, child) { err = xilinx_dma_child_probe(xdev, child); - if (err < 0) { - of_node_put(child); + if (err < 0) goto error; - } } if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { |