diff options
Diffstat (limited to 'drivers/dma')
70 files changed, 5583 insertions, 1532 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 66cda7cc9f7a..ae6a682c9f76 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -376,29 +376,6 @@ config K3_DMA Support the DMA engine for Hisilicon K3 platform devices. -config LOONGSON1_APB_DMA - tristate "Loongson1 APB DMA support" - depends on MACH_LOONGSON32 || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - This selects support for the APB DMA controller in Loongson1 SoCs, - which is required by Loongson1 NAND and audio support. - -config LOONGSON2_APB_DMA - tristate "Loongson2 APB DMA support" - depends on LOONGARCH || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support for the Loongson2 APB DMA controller driver. The - DMA controller is having single DMA channel which can be - configured for different peripherals like audio, nand, sdio - etc which is in APB bus. - - This DMA controller transfers data from memory to peripheral fifo. - It does not support memory to memory data transfer. - config LPC18XX_DMAMUX bool "NXP LPC18xx/43xx DMA MUX for PL080" depends on ARCH_LPC18XX || COMPILE_TEST @@ -505,7 +482,7 @@ config MV_XOR_V2 platforms. config MXS_DMA - bool "MXS DMA support" + tristate "MXS DMA support" depends on ARCH_MXS || ARCH_MXC || COMPILE_TEST select STMP_DEVICE select DMA_ENGINE @@ -610,6 +587,15 @@ config SPRD_DMA help Enable support for the on-chip DMA controller on Spreadtrum platform. +config SWITCHTEC_DMA + tristate "Switchtec PSX/PFX Switch DMA Engine Support" + depends on PCI + select DMA_ENGINE + help + Some Switchtec PSX/PFX PCIe Switches support additional DMA engines. + These are exposed via an extra function on the switch's upstream + port. + config TXX9_DMAC tristate "Toshiba TXx9 SoC DMA support" depends on MACH_TX49XX @@ -774,6 +760,8 @@ source "drivers/dma/fsl-dpaa2-qdma/Kconfig" source "drivers/dma/lgm/Kconfig" +source "drivers/dma/loongson/Kconfig" + source "drivers/dma/stm32/Kconfig" # clients diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index a54d7688392b..14aa086629d5 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -49,8 +49,6 @@ obj-$(CONFIG_INTEL_IDMA64) += idma64.o obj-$(CONFIG_INTEL_IOATDMA) += ioat/ obj-y += idxd/ obj-$(CONFIG_K3_DMA) += k3dma.o -obj-$(CONFIG_LOONGSON1_APB_DMA) += loongson1-apb-dma.o -obj-$(CONFIG_LOONGSON2_APB_DMA) += loongson2-apb-dma.o obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o obj-$(CONFIG_LPC32XX_DMAMUX) += lpc32xx-dmamux.o obj-$(CONFIG_MILBEAUT_HDMAC) += milbeaut-hdmac.o @@ -74,6 +72,7 @@ obj-$(CONFIG_SF_PDMA) += sf-pdma/ obj-$(CONFIG_SOPHGO_CV1800B_DMAMUX) += cv1800b-dmamux.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o obj-$(CONFIG_SPRD_DMA) += sprd-dma.o +obj-$(CONFIG_SWITCHTEC_DMA) += switchtec_dma.o obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o obj-$(CONFIG_TEGRA186_GPC_DMA) += tegra186-gpc-dma.o obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o @@ -87,6 +86,7 @@ obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/ obj-$(CONFIG_INTEL_LDMA) += lgm/ obj-y += amd/ +obj-y += loongson/ obj-y += mediatek/ obj-y += qcom/ obj-y += stm32/ diff --git a/drivers/dma/altera-msgdma.c b/drivers/dma/altera-msgdma.c index b46999c81df0..e23e5b441a24 100644 --- a/drivers/dma/altera-msgdma.c +++ b/drivers/dma/altera-msgdma.c @@ -496,6 +496,11 @@ static void msgdma_copy_one(struct msgdma_device *mdev, { void __iomem *hw_desc = mdev->desc; + /* Ensure control is the last field — required for correct FIFO flush ordering */ + static_assert(offsetof(struct msgdma_extended_desc, control) == + sizeof(struct msgdma_extended_desc) - sizeof(u32), + "control must be the last field in msgdma_extended_desc"); + /* * Check if the DESC FIFO it not full. If its full, we need to wait * for at least one entry to become free again @@ -504,17 +509,18 @@ static void msgdma_copy_one(struct msgdma_device *mdev, MSGDMA_CSR_STAT_DESC_BUF_FULL) mdelay(1); + /* Ensure control is the last field — required for correct FIFO flush ordering */ + static_assert(offsetof(struct msgdma_extended_desc, control) == + sizeof(struct msgdma_extended_desc) - sizeof(u32), + "control must be the last field in msgdma_extended_desc"); + /* - * The descriptor needs to get copied into the descriptor FIFO - * of the DMA controller. The descriptor will get flushed to the - * FIFO, once the last word (control word) is written. Since we - * are not 100% sure that memcpy() writes all word in the "correct" - * order (address from low to high) on all architectures, we make - * sure this control word is written last by single coding it and - * adding some write-barriers here. + * Copy the descriptor into the descriptor FIFO of the DMA controller, + * excluding the control word. The FIFO is flushed and the descriptor + * becomes valid once the control word is written last. */ - memcpy((void __force *)hw_desc, &desc->hw_desc, - sizeof(desc->hw_desc) - sizeof(u32)); + memcpy_toio(hw_desc, &desc->hw_desc, + offsetof(struct msgdma_extended_desc, control)); /* Write control word last to flush this descriptor into the FIFO */ mdev->idle = false; diff --git a/drivers/dma/amd/qdma/qdma.c b/drivers/dma/amd/qdma/qdma.c index f5a02c6ed348..3e61e8c4356a 100644 --- a/drivers/dma/amd/qdma/qdma.c +++ b/drivers/dma/amd/qdma/qdma.c @@ -9,7 +9,6 @@ #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include <linux/module.h> -#include <linux/mod_devicetable.h> #include <linux/platform_device.h> #include <linux/platform_data/amd_qdma.h> #include <linux/regmap.h> diff --git a/drivers/dma/arm-dma350.c b/drivers/dma/arm-dma350.c index 84220fa83029..09403aca8bb0 100644 --- a/drivers/dma/arm-dma350.c +++ b/drivers/dma/arm-dma350.c @@ -13,6 +13,11 @@ #include "dmaengine.h" #include "virt-dma.h" +#define DMANSECCTRL 0x200 + +#define NSEC_CTRL 0x0c +#define INTREN_ANYCHINTR_EN BIT(0) + #define DMAINFO 0x0f00 #define DMA_BUILDCFG0 0xb0 @@ -582,6 +587,10 @@ static int d350_probe(struct platform_device *pdev) dmac->dma.device_issue_pending = d350_issue_pending; INIT_LIST_HEAD(&dmac->dma.channels); + reg = readl_relaxed(base + DMANSECCTRL + NSEC_CTRL); + writel_relaxed(reg | INTREN_ANYCHINTR_EN, + base + DMANSECCTRL + NSEC_CTRL); + /* Would be nice to have per-channel caps for this... */ memset = true; for (int i = 0; i < nchan; i++) { diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c index ed037fa883f6..0de03611252e 100644 --- a/drivers/dma/bcm-sba-raid.c +++ b/drivers/dma/bcm-sba-raid.c @@ -40,6 +40,7 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/raid/pq.h> +#include <linux/raid/pq_tables.h> #include "dmaengine.h" diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index eb65872c5d5c..d47ff27e1408 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -13,6 +13,7 @@ #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> +#include <linux/dmapool.h> #include <linux/err.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -134,6 +135,7 @@ struct axi_dmac_desc { struct axi_dmac_chan *chan; bool cyclic; + bool cyclic_eot; bool have_partial_xfer; unsigned int num_submitted; @@ -146,6 +148,7 @@ struct axi_dmac_chan { struct virt_dma_chan vchan; struct axi_dmac_desc *next_desc; + void *pool; struct list_head active_descs; enum dma_transfer_direction direction; @@ -162,14 +165,13 @@ struct axi_dmac_chan { bool hw_cyclic; bool hw_2d; bool hw_sg; + bool hw_cyclic_hotfix; }; struct axi_dmac { void __iomem *base; int irq; - struct clk *clk; - struct dma_device dma_dev; struct axi_dmac_chan chan; }; @@ -227,29 +229,94 @@ static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr) return true; } +static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) +{ + return list_first_entry_or_null(&chan->active_descs, + struct axi_dmac_desc, vdesc.node); +} + +static struct axi_dmac_desc *axi_dmac_get_next_desc(struct axi_dmac *dmac, + struct axi_dmac_chan *chan) +{ + struct axi_dmac_desc *active = axi_dmac_active_desc(chan); + struct virt_dma_desc *vdesc; + struct axi_dmac_desc *desc; + unsigned int val; + + /* + * Just play safe and ignore any SOF if we have an active cyclic transfer + * flagged to end. We'll start it as soon as the current cyclic one ends. + */ + if (active && active->cyclic_eot) + return NULL; + + /* + * It means a SW cyclic transfer is in place so we should just return + * the same descriptor. SW cyclic transfer termination is handled + * in axi_dmac_transfer_done(). + */ + if (chan->next_desc) + return chan->next_desc; + + vdesc = vchan_next_desc(&chan->vchan); + if (!vdesc) + return NULL; + + if (active && active->cyclic && !(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) { + struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev; + + dev_warn(dev, "Discarding non EOT transfer after cyclic\n"); + list_del(&vdesc->node); + return NULL; + } + + list_move_tail(&vdesc->node, &chan->active_descs); + desc = to_axi_dmac_desc(vdesc); + chan->next_desc = desc; + + if (!active || !active->cyclic) + return desc; + + active->cyclic_eot = true; + + if (chan->hw_sg) { + unsigned long flags = AXI_DMAC_HW_FLAG_IRQ | AXI_DMAC_HW_FLAG_LAST; + /* + * Let's then stop the current cyclic transfer by making sure we + * get an EOT interrupt and to open the cyclic loop by marking + * the last segment. + */ + active->sg[active->num_sgs - 1].hw->flags = flags; + return NULL; + } + + /* + * Clear the cyclic bit if there's no Scatter-Gather HW so that we get + * at the end of the transfer. + */ + val = axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS); + val &= ~AXI_DMAC_FLAG_CYCLIC; + axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, val); + + return NULL; +} + static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) { struct axi_dmac *dmac = chan_to_axi_dmac(chan); - struct virt_dma_desc *vdesc; struct axi_dmac_desc *desc; struct axi_dmac_sg *sg; unsigned int flags = 0; unsigned int val; + desc = axi_dmac_get_next_desc(dmac, chan); + if (!desc) + return; + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); if (val) /* Queue is full, wait for the next SOT IRQ */ return; - desc = chan->next_desc; - - if (!desc) { - vdesc = vchan_next_desc(&chan->vchan); - if (!vdesc) - return; - list_move_tail(&vdesc->node, &chan->active_descs); - desc = to_axi_dmac_desc(vdesc); - chan->next_desc = desc; - } sg = &desc->sg[desc->num_submitted]; /* Already queued in cyclic mode. Wait for it to finish */ @@ -291,10 +358,12 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) * call, enable hw cyclic mode to avoid unnecessary interrupts. */ if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) { - if (chan->hw_sg) + if (chan->hw_sg) { desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ; - else if (desc->num_sgs == 1) + } else if (desc->num_sgs == 1) { + chan->next_desc = NULL; flags |= AXI_DMAC_FLAG_CYCLIC; + } } if (chan->hw_partial_xfer) @@ -312,12 +381,6 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); } -static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) -{ - return list_first_entry_or_null(&chan->active_descs, - struct axi_dmac_desc, vdesc.node); -} - static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan, struct axi_dmac_sg *sg) { @@ -398,6 +461,61 @@ static void axi_dmac_compute_residue(struct axi_dmac_chan *chan, } } +static bool axi_dmac_handle_cyclic_eot(struct axi_dmac_chan *chan, + struct axi_dmac_desc *active) +{ + struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev; + struct virt_dma_desc *vdesc; + + /* wrap around */ + active->num_completed = 0; + + if (active->cyclic_eot) { + /* + * It means an HW cyclic transfer was marked to stop. And we + * know we have something to schedule, so start the next + * transfer now the cyclic one is done. + */ + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + + if (chan->hw_cyclic_hotfix) { + struct axi_dmac *dmac = chan_to_axi_dmac(chan); + /* + * In older IP cores, ending a cyclic transfer by clearing + * the CYCLIC flag does not guarantee a graceful end. + * It can happen that some data (of the next frame) is + * already prefetched and will be wrongly visible in the + * next transfer. To workaround this, we need to reenable + * the core so everything is flushed. Newer cores handles + * this correctly and do not require this "hotfix". The + * SG IP also does not require this. + */ + dev_dbg(dev, "HW cyclic hotfix\n"); + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0); + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); + } + + return true; + } + + vdesc = vchan_next_desc(&chan->vchan); + if (!vdesc) + return false; + if (!(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) { + dev_warn(dev, "Discarding non EOT transfer after cyclic\n"); + list_del(&vdesc->node); + return false; + } + + /* then let's end the cyclic transfer */ + chan->next_desc = NULL; + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + + return true; +} + static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, unsigned int completed_transfers) { @@ -416,6 +534,7 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, if (chan->hw_sg) { if (active->cyclic) { vchan_cyclic_callback(&active->vdesc); + start_next = axi_dmac_handle_cyclic_eot(chan, active); } else { list_del(&active->vdesc.node); vchan_cookie_complete(&active->vdesc); @@ -445,7 +564,8 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, if (active->num_completed == active->num_sgs || sg->partial_len) { if (active->cyclic) { - active->num_completed = 0; /* wrap around */ + /* keep start_next as is, if already true... */ + start_next |= axi_dmac_handle_cyclic_eot(chan, active); } else { list_del(&active->vdesc.node); vchan_cookie_complete(&active->vdesc); @@ -530,11 +650,17 @@ static void axi_dmac_issue_pending(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); } +static void axi_dmac_free_desc(struct axi_dmac_desc *desc) +{ + for (unsigned int i = 0; i < desc->num_sgs; i++) + dma_pool_free(desc->chan->pool, desc->sg[i].hw, desc->sg[i].hw_phys); + + kfree(desc); +} + static struct axi_dmac_desc * axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) { - struct axi_dmac *dmac = chan_to_axi_dmac(chan); - struct device *dev = dmac->dma_dev.dev; struct axi_dmac_hw_desc *hws; struct axi_dmac_desc *desc; dma_addr_t hw_phys; @@ -546,22 +672,22 @@ axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) desc->num_sgs = num_sgs; desc->chan = chan; - hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)), - &hw_phys, GFP_ATOMIC); - if (!hws) { - kfree(desc); - return NULL; - } - for (i = 0; i < num_sgs; i++) { - desc->sg[i].hw = &hws[i]; - desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws); + hws = dma_pool_zalloc(chan->pool, GFP_NOWAIT, &hw_phys); + if (!hws) { + desc->num_sgs = i; + axi_dmac_free_desc(desc); + return NULL; + } + + desc->sg[i].hw = hws; + desc->sg[i].hw_phys = hw_phys; - hws[i].id = AXI_DMAC_SG_UNUSED; - hws[i].flags = 0; + hws->id = AXI_DMAC_SG_UNUSED; /* Link hardware descriptors */ - hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws); + if (i) + desc->sg[i - 1].hw->next_sg_addr = hw_phys; } /* The last hardware descriptor will trigger an interrupt */ @@ -570,18 +696,6 @@ axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) return desc; } -static void axi_dmac_free_desc(struct axi_dmac_desc *desc) -{ - struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan); - struct device *dev = dmac->dma_dev.dev; - struct axi_dmac_hw_desc *hw = desc->sg[0].hw; - dma_addr_t hw_phys = desc->sg[0].hw_phys; - - dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)), - hw, hw_phys); - kfree(desc); -} - static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, enum dma_transfer_direction direction, dma_addr_t addr, unsigned int num_periods, unsigned int period_len, @@ -649,7 +763,7 @@ axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs, for (i = 0; i < nb; i++) { if (!axi_dmac_check_addr(chan, vecs[i].addr) || !axi_dmac_check_len(chan, vecs[i].len)) { - kfree(desc); + axi_dmac_free_desc(desc); return NULL; } @@ -657,7 +771,12 @@ axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs, vecs[i].len, dsg); } - desc->cyclic = false; + desc->cyclic = flags & DMA_PREP_REPEAT; + if (desc->cyclic) { + /* Chain the last descriptor to the first, and remove its "last" flag */ + desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST; + desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys; + } return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); } @@ -810,9 +929,26 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved( return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); } +static int axi_dmac_alloc_chan_resources(struct dma_chan *c) +{ + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + struct device *dev = c->device->dev; + + chan->pool = dma_pool_create(dev_name(dev), dev, + sizeof(struct axi_dmac_hw_desc), + __alignof__(struct axi_dmac_hw_desc), 0); + if (!chan->pool) + return -ENOMEM; + + return 0; +} + static void axi_dmac_free_chan_resources(struct dma_chan *c) { + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + vchan_free_chan_resources(to_virt_chan(c)); + dma_pool_destroy(chan->pool); } static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) @@ -1053,6 +1189,9 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) chan->length_align_mask = chan->address_align_mask; } + if (version < ADI_AXI_PCORE_VER(4, 6, 0) && !chan->hw_sg) + chan->hw_cyclic_hotfix = true; + return 0; } @@ -1070,6 +1209,7 @@ static int axi_dmac_probe(struct platform_device *pdev) { struct dma_device *dma_dev; struct axi_dmac *dmac; + struct clk *clk; struct regmap *regmap; unsigned int version; u32 irq_mask = 0; @@ -1089,9 +1229,9 @@ static int axi_dmac_probe(struct platform_device *pdev) if (IS_ERR(dmac->base)) return PTR_ERR(dmac->base); - dmac->clk = devm_clk_get_enabled(&pdev->dev, NULL); - if (IS_ERR(dmac->clk)) - return PTR_ERR(dmac->clk); + clk = devm_clk_get_enabled(&pdev->dev, NULL); + if (IS_ERR(clk)) + return PTR_ERR(clk); version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION); @@ -1111,6 +1251,7 @@ static int axi_dmac_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask); + dma_dev->device_alloc_chan_resources = axi_dmac_alloc_chan_resources; dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources; dma_dev->device_tx_status = dma_cookie_status; dma_dev->device_issue_pending = axi_dmac_issue_pending; diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 27a8980b03dd..c71763047126 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -31,29 +31,29 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/platform_device.h> -#include <linux/dma-mapping.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> +#include <linux/acpi.h> +#include <linux/acpi_dma.h> #include <linux/device.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/hardirq.h> -#include <linux/spinlock.h> +#include <linux/idr.h> +#include <linux/init.h> +#include <linux/jiffies.h> +#include <linux/mempool.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/numa.h> #include <linux/of.h> -#include <linux/property.h> +#include <linux/of_dma.h> #include <linux/percpu.h> -#include <linux/rcupdate.h> -#include <linux/mutex.h> -#include <linux/jiffies.h> +#include <linux/platform_device.h> +#include <linux/property.h> #include <linux/rculist.h> -#include <linux/idr.h> +#include <linux/rcupdate.h> #include <linux/slab.h> -#include <linux/acpi.h> -#include <linux/acpi_dma.h> -#include <linux/of_dma.h> -#include <linux/mempool.h> -#include <linux/numa.h> +#include <linux/spinlock.h> #include "dmaengine.h" @@ -428,11 +428,18 @@ static void dma_device_release(struct kref *ref) list_del_rcu(&device->global_node); dma_channel_rebalance(); + synchronize_rcu(); if (device->device_release) device->device_release(device); } +static int __must_check dma_device_get(struct dma_device *device) +{ + lockdep_assert_held(&dma_list_mutex); + return kref_get_unless_zero(&device->ref); +} + static void dma_device_put(struct dma_device *device) { lockdep_assert_held(&dma_list_mutex); @@ -460,8 +467,7 @@ static int dma_chan_get(struct dma_chan *chan) if (!try_module_get(owner)) return -ENODEV; - ret = kref_get_unless_zero(&chan->device->ref); - if (!ret) { + if (!dma_device_get(chan->device)) { ret = -ENODEV; goto module_put_out; } @@ -495,10 +501,13 @@ module_put_out: */ static void dma_chan_put(struct dma_chan *chan) { + struct module *owner; + /* This channel is not in use, bail out */ if (!chan->client_count) return; + owner = dma_chan_to_owner(chan); chan->client_count--; /* This channel is not in use anymore, free it */ @@ -515,8 +524,10 @@ static void dma_chan_put(struct dma_chan *chan) chan->route_data = NULL; } - dma_device_put(chan->device); - module_put(dma_chan_to_owner(chan)); + /* This channel is not in use anymore, drop the device ref */ + if (!chan->client_count) + dma_device_put(chan->device); + module_put(owner); } enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie) @@ -765,7 +776,7 @@ struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, mutex_lock(&dma_list_mutex); list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { /* Finds a DMA controller with matching device node */ - if (np && device->dev->of_node && np != device->dev->of_node) + if (np && !device_match_of_node(device->dev, np)) continue; chan = find_candidate(device, mask, fn, fn_param); @@ -814,10 +825,15 @@ static const struct dma_slave_map *dma_filter_match(struct dma_device *device, */ struct dma_chan *dma_request_chan(struct device *dev, const char *name) { - struct fwnode_handle *fwnode = dev_fwnode(dev); + struct fwnode_handle *fwnode; struct dma_device *d, *_d; struct dma_chan *chan = NULL; + if (WARN_ON(!dev || !name)) + return ERR_PTR(-EINVAL); + + fwnode = dev_fwnode(dev); + if (is_of_node(fwnode)) chan = of_dma_request_slave_channel(to_of_node(fwnode), name); else if (is_acpi_device_node(fwnode)) @@ -905,11 +921,12 @@ void dma_release_channel(struct dma_chan *chan) mutex_lock(&dma_list_mutex); WARN_ONCE(chan->client_count != 1, "chan reference count %d != 1\n", chan->client_count); - dma_chan_put(chan); /* drop PRIVATE cap enabled by __dma_request_channel() */ if (--chan->device->privatecnt == 0) dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask); + dma_chan_put(chan); + if (chan->slave) { sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME); sysfs_remove_link(&chan->slave->kobj, chan->name); @@ -943,12 +960,14 @@ static void dmaenginem_release_channel(void *chan) struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name) { - struct dma_chan *chan = dma_request_chan(dev, name); - int ret = 0; + struct dma_chan *chan; + int ret; - if (!IS_ERR(chan)) - ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan); + chan = dma_request_chan(dev, name); + if (IS_ERR(chan)) + return chan; + ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan); if (ret) return ERR_PTR(ret); @@ -1097,6 +1116,8 @@ static int __dma_async_device_channel_register(struct dma_device *device, chan->dev->device.parent = device->dev; chan->dev->chan = chan; chan->dev->dev_id = device->dev_id; + spin_lock_init(&chan->lock); + if (!name) dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id); else diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index df38681a1ff4..2ae3469397f3 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -137,7 +137,7 @@ struct dmatest_params { * @did_init: module has been initialized completely * @last_error: test has faced configuration issues */ -static struct dmatest_info { +struct dmatest_info { /* Test parameters */ struct dmatest_params params; @@ -147,7 +147,9 @@ static struct dmatest_info { int last_error; struct mutex lock; bool did_init; -} test_info = { +}; + +static struct dmatest_info test_info = { .channels = LIST_HEAD_INIT(test_info.channels), .lock = __MUTEX_INITIALIZER(test_info.lock), }; diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index 5d74bc29cf89..eebed2474210 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -50,6 +50,7 @@ #define AXI_DMA_FLAG_HAS_APB_REGS BIT(0) #define AXI_DMA_FLAG_HAS_RESETS BIT(1) #define AXI_DMA_FLAG_USE_CFG2 BIT(2) +#define AXI_DMA_FLAG_ARG0_AS_CHAN BIT(3) static inline void axi_dma_iowrite32(struct axi_dma_chip *chip, u32 reg, u32 val) @@ -342,8 +343,8 @@ static void axi_desc_put(struct axi_dma_desc *desc) kfree(desc); atomic_sub(descs_put, &chan->descs_allocated); dev_vdbg(chan2dev(chan), "%s: %d descs put, %d still allocated\n", - axi_chan_name(chan), descs_put, - atomic_read(&chan->descs_allocated)); + axi_chan_name(chan), descs_put, + atomic_read(&chan->descs_allocated)); } static void vchan_desc_put(struct virt_dma_desc *vdesc) @@ -353,7 +354,7 @@ static void vchan_desc_put(struct virt_dma_desc *vdesc) static enum dma_status dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, - struct dma_tx_state *txstate) + struct dma_tx_state *txstate) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); struct virt_dma_desc *vdesc; @@ -419,6 +420,7 @@ static void dw_axi_dma_set_byte_halfword(struct axi_dma_chan *chan, bool set) iowrite32(val, chan->chip->apb_regs + offset); } + /* Called in chan locked context */ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, struct axi_dma_desc *first) @@ -435,8 +437,6 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, return; } - axi_dma_enable(chan->chip); - config.dst_multblk_type = DWAXIDMAC_MBLK_TYPE_LL; config.src_multblk_type = DWAXIDMAC_MBLK_TYPE_LL; config.tt_fc = DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC; @@ -491,7 +491,7 @@ static void axi_chan_start_first_queued(struct axi_dma_chan *chan) desc = vd_to_axi_desc(vd); dev_vdbg(chan2dev(chan), "%s: started %u\n", axi_chan_name(chan), - vd->tx.cookie); + vd->tx.cookie); axi_chan_block_xfer_start(chan, desc); } @@ -516,11 +516,17 @@ static void dw_axi_dma_synchronize(struct dma_chan *dchan) static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + int ret; + + ret = pm_runtime_resume_and_get(chan->chip->dev); + if (ret < 0) + return ret; /* ASSERT: channel is idle */ if (axi_chan_is_hw_enable(chan)) { dev_err(chan2dev(chan), "%s is non-idle!\n", axi_chan_name(chan)); + pm_runtime_put(chan->chip->dev); return -EBUSY; } @@ -531,12 +537,11 @@ static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) 64, 0); if (!chan->desc_pool) { dev_err(chan2dev(chan), "No memory for descriptors\n"); + pm_runtime_put(chan->chip->dev); return -ENOMEM; } dev_vdbg(dchan2dev(dchan), "%s: allocating\n", axi_chan_name(chan)); - pm_runtime_get(chan->chip->dev); - return 0; } @@ -592,8 +597,6 @@ static void dw_axi_dma_set_hw_channel(struct axi_dma_chan *chan, bool set) (chan->id * DMA_APB_HS_SEL_BIT_SIZE)); reg_value |= (val << (chan->id * DMA_APB_HS_SEL_BIT_SIZE)); lo_hi_writeq(reg_value, chip->apb_regs + DMAC_APB_HW_HS_SEL_0); - - return; } /* @@ -1123,7 +1126,7 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) hw_desc = &desc->hw_desc[i]; if (hw_desc->llp == llp) { axi_chan_irq_clear(chan, hw_desc->lli->status_lo); - hw_desc->lli->ctl_hi |= CH_CTL_H_LLI_VALID; + hw_desc->lli->ctl_hi |= cpu_to_le32(CH_CTL_H_LLI_VALID); desc->completed_blocks = i; if (((hw_desc->len * (i + 1)) % desc->period_len) == 0) @@ -1162,7 +1165,7 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) axi_chan_irq_clear(chan, status); dev_vdbg(chip->dev, "%s %u IRQ status: 0x%08x\n", - axi_chan_name(chan), i, status); + axi_chan_name(chan), i, status); if (status & DWAXIDMAC_IRQ_ALL_ERR) axi_chan_handle_err(chan, status); @@ -1358,16 +1361,27 @@ static int __maybe_unused axi_dma_runtime_resume(struct device *dev) static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { + unsigned int handshake = dma_spec->args[0]; struct dw_axi_dma *dw = ofdma->of_dma_data; - struct axi_dma_chan *chan; + struct axi_dma_chan *chan = NULL; struct dma_chan *dchan; - dchan = dma_get_any_slave_channel(&dw->dma); + if (dw->hdata->use_handshake_as_channel_number) { + if (handshake >= dw->hdata->nr_channels) + return NULL; + + chan = &dw->chan[handshake]; + dchan = dma_get_slave_channel(&chan->vc.chan); + } else { + dchan = dma_get_any_slave_channel(&dw->dma); + } + if (!dchan) return NULL; - chan = dchan_to_axi_dma_chan(dchan); - chan->hw_handshake_num = dma_spec->args[0]; + if (!chan) + chan = dchan_to_axi_dma_chan(dchan); + chan->hw_handshake_num = handshake; return dchan; } @@ -1451,7 +1465,7 @@ static int axi_req_irqs(struct platform_device *pdev, struct axi_dma_chip *chip) if (chip->irq[i] < 0) return chip->irq[i]; ret = devm_request_irq(chip->dev, chip->irq[i], dw_axi_dma_interrupt, - IRQF_SHARED, KBUILD_MODNAME, chip); + IRQF_SHARED, KBUILD_MODNAME, chip); if (ret < 0) return ret; } @@ -1506,6 +1520,8 @@ static int dw_probe(struct platform_device *pdev) return ret; } + chip->dw->hdata->use_handshake_as_channel_number = !!(flags & AXI_DMA_FLAG_ARG0_AS_CHAN); + chip->dw->hdata->use_cfg2 = !!(flags & AXI_DMA_FLAG_USE_CFG2); chip->core_clk = devm_clk_get(chip->dev, "core-clk"); @@ -1645,13 +1661,15 @@ static void dw_remove(struct platform_device *pdev) of_dma_controller_free(chip->dev->of_node); list_for_each_entry_safe(chan, _chan, &dw->dma.channels, - vc.chan.device_node) { + vc.chan.device_node) { list_del(&chan->vc.chan.device_node); tasklet_kill(&chan->vc.task); } } static const struct dev_pm_ops dw_axi_dma_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) SET_RUNTIME_PM_OPS(axi_dma_runtime_suspend, axi_dma_runtime_resume, NULL) }; @@ -1662,6 +1680,9 @@ static const struct of_device_id dw_dma_of_id_table[] = { .compatible = "intel,kmb-axi-dma", .data = (void *)AXI_DMA_FLAG_HAS_APB_REGS, }, { + .compatible = "sophgo,cv1800b-axi-dma", + .data = (void *)AXI_DMA_FLAG_ARG0_AS_CHAN, + }, { .compatible = "starfive,jh7110-axi-dma", .data = (void *)(AXI_DMA_FLAG_HAS_RESETS | AXI_DMA_FLAG_USE_CFG2), }, { diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index b842e6a8d90d..67cc199e24d1 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -34,6 +34,7 @@ struct dw_axi_dma_hcfg { bool reg_map_8_channels; bool restrict_axi_burst_len; bool use_cfg2; + bool use_handshake_as_channel_number; }; struct axi_dma_chan { diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index e7d698b352d3..03b2c2188351 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -7,6 +7,7 @@ */ #include <linux/module.h> +#include <linux/delay.h> #include <linux/device.h> #include <linux/kernel.h> #include <linux/dmaengine.h> @@ -29,6 +30,11 @@ struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) return container_of(vd, struct dw_edma_desc, vd); } +enum dw_edma_irq_event { + DW_EDMA_IRQ_DONE = BIT(0), + DW_EDMA_IRQ_ABORT = BIT(1), +}; + static inline u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) { @@ -40,141 +46,63 @@ u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) return cpu_addr; } -static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) -{ - struct dw_edma_burst *burst; - - burst = kzalloc_obj(*burst, GFP_NOWAIT); - if (unlikely(!burst)) - return NULL; - - INIT_LIST_HEAD(&burst->list); - if (chunk->burst) { - /* Create and add new element into the linked list */ - chunk->bursts_alloc++; - list_add_tail(&burst->list, &chunk->burst->list); - } else { - /* List head */ - chunk->bursts_alloc = 0; - chunk->burst = burst; - } - - return burst; -} - -static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) -{ - struct dw_edma_chip *chip = desc->chan->dw->chip; - struct dw_edma_chan *chan = desc->chan; - struct dw_edma_chunk *chunk; - - chunk = kzalloc_obj(*chunk, GFP_NOWAIT); - if (unlikely(!chunk)) - return NULL; - - INIT_LIST_HEAD(&chunk->list); - chunk->chan = chan; - /* Toggling change bit (CB) in each chunk, this is a mechanism to - * inform the eDMA HW block that this is a new linked list ready - * to be consumed. - * - Odd chunks originate CB equal to 0 - * - Even chunks originate CB equal to 1 - */ - chunk->cb = !(desc->chunks_alloc % 2); - if (chan->dir == EDMA_DIR_WRITE) { - chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr; - chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr; - } else { - chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr; - chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr; - } - - if (desc->chunk) { - /* Create and add new element into the linked list */ - if (!dw_edma_alloc_burst(chunk)) { - kfree(chunk); - return NULL; - } - desc->chunks_alloc++; - list_add_tail(&chunk->list, &desc->chunk->list); - } else { - /* List head */ - chunk->burst = NULL; - desc->chunks_alloc = 0; - desc->chunk = chunk; - } - - return chunk; -} - -static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) +static struct dw_edma_desc * +dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst) { struct dw_edma_desc *desc; - desc = kzalloc_obj(*desc, GFP_NOWAIT); + desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT); if (unlikely(!desc)) return NULL; desc->chan = chan; - if (!dw_edma_alloc_chunk(desc)) { - kfree(desc); - return NULL; - } + desc->nburst = nburst; + desc->cb = true; return desc; } -static void dw_edma_free_burst(struct dw_edma_chunk *chunk) +static void vchan_free_desc(struct virt_dma_desc *vdesc) { - struct dw_edma_burst *child, *_next; - - /* Remove all the list elements */ - list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { - list_del(&child->list); - kfree(child); - chunk->bursts_alloc--; - } - - /* Remove the list head */ - kfree(child); - chunk->burst = NULL; + kfree(vd2dw_edma_desc(vdesc)); } -static void dw_edma_free_chunk(struct dw_edma_desc *desc) +static void dw_edma_core_start(struct dw_edma_desc *desc, bool first) { - struct dw_edma_chunk *child, *_next; + struct dw_edma_chan *chan = desc->chan; + size_t i = 0; - if (!desc->chunk) + if (chan->non_ll) { + chan->dw->core->non_ll_start(chan, &desc->burst[desc->start_burst]); + desc->done_burst = desc->start_burst; + desc->start_burst += 1; return; + } + + for (i = 0; i + desc->start_burst < desc->nburst; i++) { + u32 idx = i + desc->start_burst; + + if (i == chan->ll_max) + break; - /* Remove all the list elements */ - list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { - dw_edma_free_burst(child); - list_del(&child->list); - kfree(child); - desc->chunks_alloc--; + dw_edma_core_ll_data(chan, &desc->burst[idx], + i, desc->cb, + idx == desc->nburst - 1 || i == chan->ll_max - 1); } - /* Remove the list head */ - kfree(child); - desc->chunk = NULL; -} + desc->done_burst = desc->start_burst; + desc->start_burst += i; -static void dw_edma_free_desc(struct dw_edma_desc *desc) -{ - dw_edma_free_chunk(desc); - kfree(desc); -} + dw_edma_core_ll_link(chan, i, desc->cb, chan->ll_region.paddr); -static void vchan_free_desc(struct virt_dma_desc *vdesc) -{ - dw_edma_free_desc(vd2dw_edma_desc(vdesc)); + if (first) + dw_edma_core_ch_enable(chan); + + dw_edma_core_ch_doorbell(chan); } static int dw_edma_start_transfer(struct dw_edma_chan *chan) { - struct dw_edma *dw = chan->dw; - struct dw_edma_chunk *child; struct dw_edma_desc *desc; struct virt_dma_desc *vd; @@ -186,21 +114,42 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan) if (!desc) return 0; - child = list_first_entry_or_null(&desc->chunk->list, - struct dw_edma_chunk, list); - if (!child) - return 0; + dw_edma_core_start(desc, !desc->start_burst); - dw_edma_core_start(dw, child, !desc->xfer_sz); - desc->xfer_sz += child->ll_region.sz; - dw_edma_free_burst(child); - list_del(&child->list); - kfree(child); - desc->chunks_alloc--; + desc->cb = !desc->cb; return 1; } +static void dw_edma_terminate_vdesc(struct virt_dma_desc *vd) +{ + list_del(&vd->node); + dma_cookie_complete(&vd->tx); + vchan_terminate_vdesc(vd); +} + +static void dw_edma_terminate_vdesc_list(struct list_head *head) +{ + struct virt_dma_desc *vd, *_vd; + + list_for_each_entry_safe(vd, _vd, head, node) + dw_edma_terminate_vdesc(vd); +} + +/* Must be called with vc.lock held. */ +static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan) +{ + /* + * This order must not be reversed. Cookies are assigned when + * descriptors are submitted, so desc_issued contains older cookies + * than desc_submitted. Completing desc_submitted first could move + * chan->vc.chan.completed_cookie backwards when desc_issued is + * terminated afterwards. + */ + dw_edma_terminate_vdesc_list(&chan->vc.desc_issued); + dw_edma_terminate_vdesc_list(&chan->vc.desc_submitted); +} + static void dw_edma_device_caps(struct dma_chan *dchan, struct dma_slave_caps *caps) { @@ -219,10 +168,56 @@ static void dw_edma_device_caps(struct dma_chan *dchan, } } +static enum dw_edma_ch_irq_mode +dw_edma_get_default_irq_mode(struct dw_edma_chan *chan) +{ + struct dw_edma_chip *chip = chan->dw->chip; + + return chip->flags & DW_EDMA_CHIP_LOCAL ? DW_EDMA_CH_IRQ_LOCAL : + DW_EDMA_CH_IRQ_REMOTE; +} + static int dw_edma_device_config(struct dma_chan *dchan, struct dma_slave_config *config) { struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + bool cfg_non_ll; + int non_ll = 0; + + chan->non_ll = false; + if (chan->dw->chip->mf == EDMA_MF_HDMA_NATIVE) { + if (config->peripheral_config && + config->peripheral_size != sizeof(int)) { + dev_err(dchan->device->dev, + "config param peripheral size mismatch\n"); + return -EINVAL; + } + + /* + * When there is no valid LLP base address available then the + * default DMA ops will use the non-LL mode. + * + * Cases where LL mode is enabled and client wants to use the + * non-LL mode then also client can do so via providing the + * peripheral_config param. + */ + cfg_non_ll = chan->dw->chip->cfg_non_ll; + if (config->peripheral_config) { + non_ll = *(int *)config->peripheral_config; + + if (cfg_non_ll && !non_ll) { + dev_err(dchan->device->dev, "invalid configuration\n"); + return -EINVAL; + } + } + + if (cfg_non_ll || non_ll) + chan->non_ll = true; + } else if (config->peripheral_config) { + dev_err(dchan->device->dev, + "peripheral config param applicable only for HDMA\n"); + return -EINVAL; + } memcpy(&chan->config, config, sizeof(*config)); chan->configured = true; @@ -230,11 +225,27 @@ static int dw_edma_device_config(struct dma_chan *dchan, return 0; } +static struct dma_slave_config * +dw_edma_device_get_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + struct dw_edma_chan *chan; + + if (config) + return config; + + chan = dchan2dw_edma_chan(dchan); + + return &chan->config; +} + static int dw_edma_device_pause(struct dma_chan *dchan) { struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) err = -EPERM; else if (chan->status != EDMA_ST_BUSY) @@ -252,6 +263,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) { err = -EPERM; } else if (chan->status != EDMA_ST_PAUSE) { @@ -260,7 +273,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan) err = -EPERM; } else { chan->status = EDMA_ST_BUSY; - dw_edma_start_transfer(chan); + if (!dw_edma_start_transfer(chan)) + chan->status = EDMA_ST_IDLE; } return err; @@ -271,25 +285,29 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) { - /* Do nothing */ + dw_edma_terminate_all_descs(chan); } else if (chan->status == EDMA_ST_PAUSE) { + dw_edma_terminate_all_descs(chan); chan->status = EDMA_ST_IDLE; - chan->configured = false; } else if (chan->status == EDMA_ST_IDLE) { - chan->configured = false; + dw_edma_terminate_all_descs(chan); } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { /* * The channel is in a false BUSY state, probably didn't * receive or lost an interrupt */ + dw_edma_terminate_all_descs(chan); chan->status = EDMA_ST_IDLE; - chan->configured = false; } else if (chan->request > EDMA_REQ_PAUSE) { err = -EPERM; } else { chan->request = EDMA_REQ_STOP; } + if (chan->status == EDMA_ST_IDLE) + chan->request = EDMA_REQ_NONE; return err; } @@ -299,11 +317,9 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long flags; - if (!chan->configured) - return; - spin_lock_irqsave(&chan->vc.lock, flags); - if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && + if (chan->configured && vchan_issue_pending(&chan->vc) && + chan->request == EDMA_REQ_NONE && chan->status == EDMA_ST_IDLE) { chan->status = EDMA_ST_BUSY; dw_edma_start_transfer(chan); @@ -336,8 +352,10 @@ dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, vd = vchan_find_desc(&chan->vc, cookie); if (vd) { desc = vd2dw_edma_desc(vd); - if (desc) - residue = desc->alloc_sz - desc->xfer_sz; + + residue = desc->alloc_sz; + if (desc && desc->done_burst) + residue -= desc->burst[desc->done_burst - 1].xfer_sz; } spin_unlock_irqrestore(&chan->vc.lock, flags); @@ -348,18 +366,18 @@ ret_residue: } static struct dma_async_tx_descriptor * -dw_edma_device_transfer(struct dw_edma_transfer *xfer) +dw_edma_device_transfer(struct dw_edma_transfer *xfer, + struct dma_slave_config *config) { struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); enum dma_transfer_direction dir = xfer->direction; struct scatterlist *sg = NULL; - struct dw_edma_chunk *chunk; struct dw_edma_burst *burst; struct dw_edma_desc *desc; u64 src_addr, dst_addr; size_t fsz = 0; - u32 cnt = 0; - int i; + size_t cnt = 0; + u32 i; if (!chan->configured) return NULL; @@ -415,20 +433,12 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) return NULL; } - desc = dw_edma_alloc_desc(chan); - if (unlikely(!desc)) - goto err_alloc; - - chunk = dw_edma_alloc_chunk(desc); - if (unlikely(!chunk)) - goto err_alloc; - if (xfer->type == EDMA_XFER_INTERLEAVED) { src_addr = xfer->xfer.il->src_start; dst_addr = xfer->xfer.il->dst_start; } else { - src_addr = chan->config.src_addr; - dst_addr = chan->config.dst_addr; + src_addr = config->src_addr; + dst_addr = config->dst_addr; } if (dir == DMA_DEV_TO_MEM) @@ -446,19 +456,15 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) fsz = xfer->xfer.il->frame_size; } + desc = dw_edma_alloc_desc(chan, cnt); + if (unlikely(!desc)) + return NULL; + for (i = 0; i < cnt; i++) { if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) break; - if (chunk->bursts_alloc == chan->ll_max) { - chunk = dw_edma_alloc_chunk(desc); - if (unlikely(!chunk)) - goto err_alloc; - } - - burst = dw_edma_alloc_burst(chunk); - if (unlikely(!burst)) - goto err_alloc; + burst = desc->burst + i; if (xfer->type == EDMA_XFER_CYCLIC) burst->sz = xfer->xfer.cyclic.len; @@ -467,8 +473,8 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) else if (xfer->type == EDMA_XFER_INTERLEAVED) burst->sz = xfer->xfer.il->sgl[i % fsz].size; - chunk->ll_region.sz += burst->sz; desc->alloc_sz += burst->sz; + burst->xfer_sz = desc->alloc_sz; if (dir == DMA_DEV_TO_MEM) { burst->sar = src_addr; @@ -523,19 +529,14 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) } return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); - -err_alloc: - if (desc) - dw_edma_free_desc(desc); - - return NULL; } static struct dma_async_tx_descriptor * -dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, - unsigned int len, - enum dma_transfer_direction direction, - unsigned long flags, void *context) +dw_edma_device_prep_config_sg(struct dma_chan *dchan, struct scatterlist *sgl, + unsigned int len, + enum dma_transfer_direction direction, + unsigned long flags, + struct dma_slave_config *config) { struct dw_edma_transfer xfer; @@ -546,7 +547,10 @@ dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, xfer.flags = flags; xfer.type = EDMA_XFER_SCATTER_GATHER; - return dw_edma_device_transfer(&xfer); + if (config && dw_edma_device_config(dchan, config)) + return NULL; + + return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, config)); } static struct dma_async_tx_descriptor * @@ -565,7 +569,7 @@ dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, xfer.flags = flags; xfer.type = EDMA_XFER_CYCLIC; - return dw_edma_device_transfer(&xfer); + return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL)); } static struct dma_async_tx_descriptor * @@ -581,7 +585,7 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, xfer.flags = flags; xfer.type = EDMA_XFER_INTERLEAVED; - return dw_edma_device_transfer(&xfer); + return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL)); } static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, @@ -595,8 +599,14 @@ static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, return; desc = vd2dw_edma_desc(vd); - if (desc) - residue = desc->alloc_sz - desc->xfer_sz; + if (desc) { + residue = desc->alloc_sz; + + if (result == DMA_TRANS_NOERROR) + residue -= desc->burst[desc->start_burst - 1].xfer_sz; + else if (desc->done_burst) + residue -= desc->burst[desc->done_burst - 1].xfer_sz; + } res = &vd->tx_result; res->result = result; @@ -610,35 +620,41 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) unsigned long flags; spin_lock_irqsave(&chan->vc.lock, flags); + if (chan->status == EDMA_ST_PAUSE) { + spin_unlock_irqrestore(&chan->vc.lock, flags); + return; + } + vd = vchan_next_desc(&chan->vc); if (vd) { switch (chan->request) { case EDMA_REQ_NONE: + case EDMA_REQ_PAUSE: desc = vd2dw_edma_desc(vd); - if (!desc->chunks_alloc) { + if (desc->start_burst >= desc->nburst) { dw_hdma_set_callback_result(vd, DMA_TRANS_NOERROR); list_del(&vd->node); vchan_cookie_complete(vd); } + if (chan->request == EDMA_REQ_PAUSE) { + chan->request = EDMA_REQ_NONE; + chan->status = EDMA_ST_PAUSE; + break; + } + /* Continue transferring if there are remaining chunks or issued requests. */ chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; break; case EDMA_REQ_STOP: - list_del(&vd->node); - vchan_cookie_complete(vd); + dw_edma_terminate_all_descs(chan); chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; break; - case EDMA_REQ_PAUSE: - chan->request = EDMA_REQ_NONE; - chan->status = EDMA_ST_PAUSE; - break; - default: break; } @@ -653,40 +669,189 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) spin_lock_irqsave(&chan->vc.lock, flags); vd = vchan_next_desc(&chan->vc); - if (vd) { + if (vd && chan->request == EDMA_REQ_STOP) { + dw_edma_terminate_all_descs(chan); + } else if (vd) { dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED); list_del(&vd->node); vchan_cookie_complete(vd); } - spin_unlock_irqrestore(&chan->vc.lock, flags); chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; + spin_unlock_irqrestore(&chan->vc.lock, flags); } -static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) +static void dw_edma_irq_work(struct work_struct *work) +{ + struct dw_edma_chan *chan = container_of(work, struct dw_edma_chan, + irq_work); + unsigned int events; + + do { + events = atomic_xchg(&chan->irq_pending, 0); + + if (events & DW_EDMA_IRQ_DONE) + dw_edma_done_interrupt(chan); + if (events & DW_EDMA_IRQ_ABORT) + dw_edma_abort_interrupt(chan); + } while (atomic_read(&chan->irq_pending)); +} + +static void dw_edma_queue_irq_work(struct dw_edma_chan *chan, + enum dw_edma_irq_event event) +{ + atomic_or(event, &chan->irq_pending); + queue_work(chan->dw->wq, &chan->irq_work); +} + +static void dw_edma_done_interrupt_deferred(struct dw_edma_chan *chan) +{ + dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_DONE); +} + +static void dw_edma_abort_interrupt_deferred(struct dw_edma_chan *chan) +{ + dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_ABORT); +} + +static void dw_edma_emul_irq_ack(struct irq_data *d) +{ + struct dw_edma *dw = irq_data_get_irq_chip_data(d); + + dw_edma_core_ack_emulated_irq(dw); +} + +/* + * irq_chip implementation for interrupt-emulation doorbells. + * + * The emulated source has no mask/unmask mechanism. With handle_level_irq(), + * the flow is therefore: + * 1) .irq_ack() deasserts the source + * 2) registered handlers (if any) are dispatched + * Since deassertion is already done in .irq_ack(), handlers do not need to take + * care of it, hence IRQCHIP_ONESHOT_SAFE. + */ +static struct irq_chip dw_edma_emul_irqchip = { + .name = "dw-edma-emul", + .irq_ack = dw_edma_emul_irq_ack, + .flags = IRQCHIP_ONESHOT_SAFE | IRQCHIP_SKIP_SET_WAKE, +}; + +static int dw_edma_emul_irq_alloc(struct dw_edma *dw) +{ + struct dw_edma_chip *chip = dw->chip; + int virq; + + chip->db_irq = 0; + chip->db_offset = ~0; + + if (chip->flags & DW_EDMA_CHIP_PARTIAL) + return 0; + + /* + * Only meaningful when the core provides the deassert sequence + * for interrupt emulation. + */ + if (!dw->core->ack_emulated_irq) + return 0; + + /* + * Allocate a single, requestable Linux virtual IRQ number. + * Use >= 1 so that 0 can remain a "not available" sentinel. + */ + virq = irq_alloc_desc(NUMA_NO_NODE); + if (virq < 0) + return virq; + + irq_set_chip_and_handler(virq, &dw_edma_emul_irqchip, handle_level_irq); + irq_set_status_flags(virq, IRQ_LEVEL); + irq_set_chip_data(virq, dw); + irq_set_noprobe(virq); + + chip->db_irq = virq; + chip->db_offset = dw_edma_core_db_offset(dw); + + return 0; +} + +static void dw_edma_emul_irq_free(struct dw_edma *dw) +{ + struct dw_edma_chip *chip = dw->chip; + + if (!chip) + return; + if (chip->db_irq <= 0) + return; + + irq_free_descs(chip->db_irq, 1); + chip->db_irq = 0; + chip->db_offset = ~0; +} + +static inline irqreturn_t dw_edma_interrupt_emulated(void *data) +{ + struct dw_edma_irq *dw_irq = data; + struct dw_edma *dw = dw_irq->dw; + int db_irq = dw->chip->db_irq; + + if (db_irq > 0) { + /* + * Interrupt emulation may assert the IRQ line without updating the + * normal DONE/ABORT status bits. With a shared IRQ handler we + * cannot reliably detect such events by status registers alone, so + * always perform the core-specific deassert sequence. + */ + generic_handle_irq(db_irq); + return IRQ_HANDLED; + } + return IRQ_NONE; +} + +static inline irqreturn_t dw_edma_interrupt_write_inner(int irq, void *data) { struct dw_edma_irq *dw_irq = data; return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE, - dw_edma_done_interrupt, - dw_edma_abort_interrupt); + dw_edma_done_interrupt_deferred, + dw_edma_abort_interrupt_deferred); } -static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) +static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data) { struct dw_edma_irq *dw_irq = data; return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ, - dw_edma_done_interrupt, - dw_edma_abort_interrupt); + dw_edma_done_interrupt_deferred, + dw_edma_abort_interrupt_deferred); } -static irqreturn_t dw_edma_interrupt_common(int irq, void *data) +static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) { irqreturn_t ret = IRQ_NONE; - ret |= dw_edma_interrupt_write(irq, data); - ret |= dw_edma_interrupt_read(irq, data); + ret |= dw_edma_interrupt_write_inner(irq, data); + ret |= dw_edma_interrupt_emulated(data); + + return ret; +} + +static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) +{ + irqreturn_t ret = IRQ_NONE; + + ret |= dw_edma_interrupt_read_inner(irq, data); + ret |= dw_edma_interrupt_emulated(data); + + return ret; +} + +static inline irqreturn_t dw_edma_interrupt_common(int irq, void *data) +{ + irqreturn_t ret = IRQ_NONE; + + ret |= dw_edma_interrupt_write_inner(irq, data); + ret |= dw_edma_interrupt_read_inner(irq, data); + ret |= dw_edma_interrupt_emulated(data); return ret; } @@ -701,21 +866,51 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) return 0; } -static void dw_edma_free_chan_resources(struct dma_chan *dchan) +static void dw_edma_wait_termination(struct dma_chan *dchan) { + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long timeout = jiffies + msecs_to_jiffies(5000); - int ret; + bool stopping; + /* + * A STOP may be deferred to a later interrupt while the channel is still + * running. Wait until that handler completes the termination. + */ while (time_before(jiffies, timeout)) { - ret = dw_edma_device_terminate_all(dchan); - if (!ret) - break; + scoped_guard(spinlock_irqsave, &chan->vc.lock) + stopping = chan->request == EDMA_REQ_STOP; - if (time_after_eq(jiffies, timeout)) + if (!stopping) return; - cpu_relax(); + fsleep(1000); } + + dev_warn(chan->dw->chip->dev, + "timeout waiting for channel termination\n"); +} + +static void dw_edma_device_synchronize(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + dw_edma_wait_termination(dchan); + cancel_work_sync(&chan->irq_work); + atomic_set(&chan->irq_pending, 0); + vchan_synchronize(&chan->vc); +} + +static void dw_edma_free_chan_resources(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + dw_edma_device_terminate_all(dchan); + dw_edma_device_synchronize(dchan); + + scoped_guard(spinlock_irqsave, &chan->vc.lock) + chan->configured = false; + + vchan_free_chan_resources(&chan->vc); } static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) @@ -737,6 +932,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) chan = &dw->chan[i]; chan->dw = dw; + chan->func_no = chip->func_no; if (i < dw->wr_ch_cnt) { chan->id = i; @@ -749,12 +945,16 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) chan->configured = false; chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; + chan->irq_mode = dw_edma_get_default_irq_mode(chan); + INIT_WORK(&chan->irq_work, dw_edma_irq_work); + atomic_set(&chan->irq_pending, 0); if (chan->dir == EDMA_DIR_WRITE) - chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ); + chan->ll_region = chip->ll_region_wr[chan->id]; else - chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ); - chan->ll_max -= 1; + chan->ll_region = chip->ll_region_rd[chan->id]; + + chan->ll_max = chan->ll_region.sz / EDMA_LL_SZ - 1; dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", str_write_read(chan->dir == EDMA_DIR_WRITE), @@ -770,11 +970,10 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) irq = &dw->irq[pos]; if (chan->dir == EDMA_DIR_WRITE) - irq->wr_mask |= BIT(chan->id); + bitmap_set(irq->wr_mask, chan->id, 1); else - irq->rd_mask |= BIT(chan->id); + bitmap_set(irq->rd_mask, chan->id, 1); - irq->dw = dw; memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", @@ -813,9 +1012,10 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) dma->device_pause = dw_edma_device_pause; dma->device_resume = dw_edma_device_resume; dma->device_terminate_all = dw_edma_device_terminate_all; + dma->device_synchronize = dw_edma_device_synchronize; dma->device_issue_pending = dw_edma_device_issue_pending; dma->device_tx_status = dw_edma_device_tx_status; - dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; + dma->device_prep_config_sg = dw_edma_device_prep_config_sg; dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma; @@ -833,19 +1033,12 @@ static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) } } -static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) -{ - while (*mask * alloc < cnt) - (*mask)++; -} - static int dw_edma_irq_request(struct dw_edma *dw, u32 *wr_alloc, u32 *rd_alloc) { struct dw_edma_chip *chip = dw->chip; struct device *dev = dw->chip->dev; - u32 wr_mask = 1; - u32 rd_mask = 1; + struct msi_desc *msi_desc; int i, err = 0; u32 ch_cnt; int irq; @@ -862,6 +1055,7 @@ static int dw_edma_irq_request(struct dw_edma *dw, if (chip->nr_irqs == 1) { /* Common IRQ shared among all channels */ irq = chip->ops->irq_vector(dev, 0); + dw->irq[0].dw = dw; err = request_irq(irq, dw_edma_interrupt_common, IRQF_SHARED, dw->name, &dw->irq[0]); if (err) { @@ -882,11 +1076,9 @@ static int dw_edma_irq_request(struct dw_edma *dw, dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt); } - dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt); - dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); - for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { irq = chip->ops->irq_vector(dev, i); + dw->irq[i].dw = dw; err = request_irq(irq, i < *wr_alloc ? dw_edma_interrupt_write : @@ -895,9 +1087,12 @@ static int dw_edma_irq_request(struct dw_edma *dw, &dw->irq[i]); if (err) goto err_irq_free; - - if (irq_get_msi_desc(irq)) + msi_desc = irq_get_msi_desc(irq); + if (msi_desc) { get_cached_msi_msg(irq, &dw->irq[i].msi); + if (!msi_desc->pci.msi_attrib.is_msix) + dw->irq[i].msi.data = dw->irq[0].msi.data + i; + } } dw->nr_irqs = i; @@ -914,12 +1109,37 @@ err_irq_free: return err; } +static int dw_edma_check_partial(struct dw_edma_chip *chip, + u16 hw_wr_ch_cnt, u16 hw_rd_ch_cnt) +{ + if (!(chip->flags & DW_EDMA_CHIP_PARTIAL)) + return 0; + + if (chip->mf != EDMA_MF_EDMA_UNROLL && + chip->mf != EDMA_MF_HDMA_COMPAT) + return 0; + + /* + * Direction-wide registers are shared by all channels in that + * direction, so a direction must have a single owner. + */ + if ((chip->ll_wr_cnt && chip->ll_wr_cnt != hw_wr_ch_cnt) || + (chip->ll_rd_cnt && chip->ll_rd_cnt != hw_rd_ch_cnt)) + return -EOPNOTSUPP; + + return 0; +} + int dw_edma_probe(struct dw_edma_chip *chip) { struct device *dev; struct dw_edma *dw; + u16 hw_wr_ch_cnt; + u16 hw_rd_ch_cnt; u32 wr_alloc = 0; u32 rd_alloc = 0; + u16 max_wr_cnt; + u16 max_rd_cnt; int i, err; if (!chip) @@ -929,26 +1149,50 @@ int dw_edma_probe(struct dw_edma_chip *chip) if (!dev || !chip->ops) return -EINVAL; + if (chip->flags & DW_EDMA_CHIP_PARTIAL) { + switch (chip->mf) { + case EDMA_MF_EDMA_UNROLL: + case EDMA_MF_HDMA_COMPAT: + case EDMA_MF_HDMA_NATIVE: + break; + default: + return -EOPNOTSUPP; + } + } + dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); if (!dw) return -ENOMEM; dw->chip = chip; - if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) + if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) { dw_hdma_v0_core_register(dw); - else + max_wr_cnt = HDMA_MAX_WR_CH; + max_rd_cnt = HDMA_MAX_RD_CH; + } else { dw_edma_v0_core_register(dw); + max_wr_cnt = EDMA_MAX_WR_CH; + max_rd_cnt = EDMA_MAX_RD_CH; + } raw_spin_lock_init(&dw->lock); - dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, - dw_edma_core_ch_count(dw, EDMA_DIR_WRITE)); - dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); + /* + * chip->ll_*_cnt describes the channels exposed by this instance. Keep + * the usable hardware counts separate for partial ownership checks. + */ + hw_wr_ch_cnt = min(dw_edma_core_ch_count(dw, EDMA_DIR_WRITE), + max_wr_cnt); + hw_rd_ch_cnt = min(dw_edma_core_ch_count(dw, EDMA_DIR_READ), + max_rd_cnt); + + err = dw_edma_check_partial(chip, hw_wr_ch_cnt, hw_rd_ch_cnt); + if (err) + return err; - dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, - dw_edma_core_ch_count(dw, EDMA_DIR_READ)); - dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); + dw->wr_ch_cnt = min(chip->ll_wr_cnt, hw_wr_ch_cnt); + dw->rd_ch_cnt = min(chip->ll_rd_cnt, hw_rd_ch_cnt); if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) return -EINVAL; @@ -965,13 +1209,39 @@ int dw_edma_probe(struct dw_edma_chip *chip) snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s", dev_name(chip->dev)); - /* Disable eDMA, only to establish the ideal initial conditions */ - dw_edma_core_off(dw); + if (chip->flags & DW_EDMA_CHIP_PARTIAL) { + /* + * Do not reset the shared controller, but drain stale state + * from resources represented by this instance. + */ + err = dw_edma_core_quiesce(dw); + if (err) + return err; + } else { + /* Disable eDMA only when this instance owns the controller. */ + dw_edma_core_off(dw); + } + + /* + * Deferred IRQ works are queued from the hard IRQ handlers, so the + * workqueue must exist before any IRQ is requested. + */ + dw->wq = alloc_workqueue("dw-edma:%s", WQ_UNBOUND | WQ_HIGHPRI, 0, + dev_name(chip->dev)); + if (!dw->wq) + return -ENOMEM; /* Request IRQs */ err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc); - if (err) + if (err) { + destroy_workqueue(dw->wq); return err; + } + + /* Allocate a dedicated virtual IRQ for interrupt-emulation doorbells */ + err = dw_edma_emul_irq_alloc(dw); + if (err) + dev_warn(dev, "Failed to allocate emulation IRQ: %d\n", err); /* Setup write/read channels */ err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc); @@ -988,6 +1258,8 @@ int dw_edma_probe(struct dw_edma_chip *chip) err_irq_free: for (i = (dw->nr_irqs - 1); i >= 0; i--) free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); + dw_edma_emul_irq_free(dw); + destroy_workqueue(dw->wq); return err; } @@ -998,18 +1270,26 @@ int dw_edma_remove(struct dw_edma_chip *chip) struct dw_edma_chan *chan, *_chan; struct device *dev = chip->dev; struct dw_edma *dw = chip->dw; - int i; + int i, err = 0; /* Skip removal if no private data found */ if (!dw) return -ENODEV; - /* Disable eDMA */ - dw_edma_core_off(dw); + if (chip->flags & DW_EDMA_CHIP_PARTIAL) + err = dw_edma_core_quiesce(dw); + else + dw_edma_core_off(dw); /* Free irqs */ for (i = (dw->nr_irqs - 1); i >= 0; i--) free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); + dw_edma_emul_irq_free(dw); + + for (i = 0; i < dw->wr_ch_cnt + dw->rd_ch_cnt; i++) + cancel_work_sync(&dw->chan[i].irq_work); + + destroy_workqueue(dw->wq); /* Deregister eDMA device */ dma_async_device_unregister(&dw->dma); @@ -1019,7 +1299,7 @@ int dw_edma_remove(struct dw_edma_chip *chip) list_del(&chan->vc.chan.device_node); } - return 0; + return err; } EXPORT_SYMBOL_GPL(dw_edma_remove); diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index 71894b9e0b15..f6a5ad317567 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -9,8 +9,10 @@ #ifndef _DW_EDMA_CORE_H #define _DW_EDMA_CORE_H +#include <linux/atomic.h> #include <linux/msi.h> #include <linux/dma/edma.h> +#include <linux/workqueue.h> #include "../virt-dma.h" @@ -43,32 +45,24 @@ struct dw_edma_chan; struct dw_edma_chunk; struct dw_edma_burst { - struct list_head list; u64 sar; u64 dar; u32 sz; -}; - -struct dw_edma_chunk { - struct list_head list; - struct dw_edma_chan *chan; - struct dw_edma_burst *burst; - - u32 bursts_alloc; - - u8 cb; - struct dw_edma_region ll_region; /* Linked list */ + /* precalulate summary of previous burst total size */ + u32 xfer_sz; }; struct dw_edma_desc { struct virt_dma_desc vd; struct dw_edma_chan *chan; - struct dw_edma_chunk *chunk; - - u32 chunks_alloc; u32 alloc_sz; - u32 xfer_sz; + + size_t done_burst; + size_t start_burst; + u8 cb; + size_t nburst; + struct dw_edma_burst burst[] __counted_by(nburst); }; struct dw_edma_chan { @@ -76,23 +70,32 @@ struct dw_edma_chan { struct dw_edma *dw; int id; enum dw_edma_dir dir; + u8 func_no; u32 ll_max; + struct dw_edma_region ll_region; /* Linked list */ struct msi_msg msi; + enum dw_edma_ch_irq_mode irq_mode; + enum dw_edma_request request; enum dw_edma_status status; u8 configured; struct dma_slave_config config; + bool non_ll; + + struct work_struct irq_work; + atomic_t irq_pending; }; struct dw_edma_irq { struct msi_msg msi; - u32 wr_mask; - u32 rd_mask; struct dw_edma *dw; + + DECLARE_BITMAP(wr_mask, HDMA_MAX_WR_CH); + DECLARE_BITMAP(rd_mask, HDMA_MAX_RD_CH); }; struct dw_edma { @@ -108,7 +111,13 @@ struct dw_edma { struct dw_edma_chan *chan; - raw_spinlock_t lock; /* Only for legacy */ + /* + * WQ_HIGHPRI keeps completion processing responsive under heavy load; + * WQ_UNBOUND lets different channels run on different CPUs. + */ + struct workqueue_struct *wq; + + raw_spinlock_t lock; /* Protect v0 shared registers */ struct dw_edma_chip *chip; @@ -119,13 +128,22 @@ typedef void (*dw_edma_handler_t)(struct dw_edma_chan *); struct dw_edma_core_ops { void (*off)(struct dw_edma *dw); + int (*quiesce)(struct dw_edma *dw); + int (*ch_quiesce)(struct dw_edma_chan *chan); u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir); enum dma_status (*ch_status)(struct dw_edma_chan *chan); irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, dw_edma_handler_t done, dw_edma_handler_t abort); - void (*start)(struct dw_edma_chunk *chunk, bool first); + void (*non_ll_start)(struct dw_edma_chan *chan, struct dw_edma_burst *child); + void (*ll_data)(struct dw_edma_chan *chan, struct dw_edma_burst *burst, + u32 idx, bool cb, bool irq); + void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr); + void (*ch_doorbell)(struct dw_edma_chan *chan); + void (*ch_enable)(struct dw_edma_chan *chan); void (*ch_config)(struct dw_edma_chan *chan); void (*debugfs_on)(struct dw_edma *dw); + void (*ack_emulated_irq)(struct dw_edma *dw); + resource_size_t (*db_offset)(struct dw_edma *dw); }; struct dw_edma_sg { @@ -163,6 +181,14 @@ struct dw_edma_chan *dchan2dw_edma_chan(struct dma_chan *dchan) return vc2dw_edma_chan(to_virt_chan(dchan)); } +static inline u64 dw_edma_core_get_ll_paddr(struct dw_edma_chan *chan) +{ + if (chan->dir == EDMA_DIR_WRITE) + return chan->dw->chip->ll_region_wr[chan->id].paddr; + + return chan->dw->chip->ll_region_rd[chan->id].paddr; +} + static inline void dw_edma_core_off(struct dw_edma *dw) { @@ -170,6 +196,18 @@ void dw_edma_core_off(struct dw_edma *dw) } static inline +int dw_edma_core_quiesce(struct dw_edma *dw) +{ + return dw->core->quiesce(dw); +} + +static inline +int dw_edma_core_ch_quiesce(struct dw_edma_chan *chan) +{ + return chan->dw->core->ch_quiesce(chan); +} + +static inline u16 dw_edma_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { return dw->core->ch_count(dw, dir); @@ -189,15 +227,32 @@ dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, } static inline -void dw_edma_core_start(struct dw_edma *dw, struct dw_edma_chunk *chunk, bool first) +void dw_edma_core_ch_config(struct dw_edma_chan *chan) { - dw->core->start(chunk, first); + chan->dw->core->ch_config(chan); } -static inline -void dw_edma_core_ch_config(struct dw_edma_chan *chan) +static inline void +dw_edma_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst, + u32 idx, bool cb, bool irq) { - chan->dw->core->ch_config(chan); + chan->dw->core->ll_data(chan, burst, idx, cb, irq); +} + +static inline void +dw_edma_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr) +{ + chan->dw->core->ll_link(chan, idx, cb, addr); +} + +static inline void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan) +{ + chan->dw->core->ch_doorbell(chan); +} + +static inline void dw_edma_core_ch_enable(struct dw_edma_chan *chan) +{ + chan->dw->core->ch_enable(chan); } static inline @@ -206,4 +261,30 @@ void dw_edma_core_debugfs_on(struct dw_edma *dw) dw->core->debugfs_on(dw); } +static inline int dw_edma_core_ack_emulated_irq(struct dw_edma *dw) +{ + if (!dw->core->ack_emulated_irq) + return -EOPNOTSUPP; + + dw->core->ack_emulated_irq(dw); + return 0; +} + +static inline resource_size_t +dw_edma_core_db_offset(struct dw_edma *dw) +{ + return dw->core->db_offset(dw); +} + +static inline bool +dw_edma_core_ch_ignore_irq(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + + if (dw->chip->flags & DW_EDMA_CHIP_LOCAL) + return chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE; + else + return chan->irq_mode == DW_EDMA_CH_IRQ_LOCAL; +} + #endif /* _DW_EDMA_CORE_H */ diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 83230acaa597..9f237ba916de 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -14,14 +14,36 @@ #include <linux/pci-epf.h> #include <linux/msi.h> #include <linux/bitfield.h> +#include <linux/sizes.h> #include "dw-edma-core.h" -#define DW_PCIE_VSEC_DMA_ID 0x6 -#define DW_PCIE_VSEC_DMA_BAR GENMASK(10, 8) -#define DW_PCIE_VSEC_DMA_MAP GENMASK(2, 0) -#define DW_PCIE_VSEC_DMA_WR_CH GENMASK(9, 0) -#define DW_PCIE_VSEC_DMA_RD_CH GENMASK(25, 16) +/* Synopsys */ +#define DW_PCIE_SYNOPSYS_VSEC_DMA_ID 0x6 +#define DW_PCIE_SYNOPSYS_VSEC_DMA_BAR GENMASK(10, 8) +#define DW_PCIE_SYNOPSYS_VSEC_DMA_MAP GENMASK(2, 0) +#define DW_PCIE_SYNOPSYS_VSEC_DMA_WR_CH GENMASK(9, 0) +#define DW_PCIE_SYNOPSYS_VSEC_DMA_RD_CH GENMASK(25, 16) + +/* AMD MDB (Xilinx) specific defines */ +#define PCI_DEVICE_ID_XILINX_B054 0xb054 +#define PCI_DEVICE_ID_XILINX_B00F 0xb00f + +#define DW_PCIE_XILINX_MDB_VSEC_DMA_ID 0x6 +#define DW_PCIE_XILINX_MDB_VSEC_ID 0x20 +#define DW_PCIE_XILINX_MDB_VSEC_DMA_BAR GENMASK(10, 8) +#define DW_PCIE_XILINX_MDB_VSEC_DMA_MAP GENMASK(2, 0) +#define DW_PCIE_XILINX_MDB_VSEC_DMA_WR_CH GENMASK(9, 0) +#define DW_PCIE_XILINX_MDB_VSEC_DMA_RD_CH GENMASK(25, 16) + +#define DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_HIGH 0xc +#define DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_LOW 0x8 +#define DW_PCIE_XILINX_MDB_INVALID_ADDR (~0ULL) + +#define DW_PCIE_XILINX_MDB_LL_OFF_GAP 0x200000 +#define DW_PCIE_XILINX_MDB_LL_SIZE 0x800 +#define DW_PCIE_XILINX_MDB_DT_OFF_GAP 0x100000 +#define DW_PCIE_XILINX_MDB_DT_SIZE 0x800 #define DW_BLOCK(a, b, c) \ { \ @@ -33,6 +55,8 @@ struct dw_edma_block { enum pci_barno bar; off_t off; + u64 paddr; + bool paddr_valid; size_t sz; }; @@ -40,18 +64,36 @@ struct dw_edma_pcie_data { /* eDMA registers location */ struct dw_edma_block rg; /* eDMA memory linked list location */ - struct dw_edma_block ll_wr[EDMA_MAX_WR_CH]; - struct dw_edma_block ll_rd[EDMA_MAX_RD_CH]; + struct dw_edma_block ll_wr[HDMA_MAX_WR_CH]; + struct dw_edma_block ll_rd[HDMA_MAX_RD_CH]; /* eDMA memory data location */ - struct dw_edma_block dt_wr[EDMA_MAX_WR_CH]; - struct dw_edma_block dt_rd[EDMA_MAX_RD_CH]; + struct dw_edma_block dt_wr[HDMA_MAX_WR_CH]; + struct dw_edma_block dt_rd[HDMA_MAX_RD_CH]; /* Other */ enum dw_edma_map_format mf; u8 irqs; u16 wr_ch_cnt; u16 rd_ch_cnt; + u64 devmem_phys_off; + bool cfg_non_ll; }; +struct dw_edma_pcie_match_data { + const struct dw_edma_pcie_data *data; + const struct dw_edma_plat_ops *plat_ops; + /* + * Mandatory callback. It may leave @pdata unchanged when the static + * template already describes the device. + */ + int (*parse_caps)(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata); + unsigned long flags; + u32 chip_flags; +}; + +#define DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF BIT(0) +#define DW_EDMA_PCIE_F_REG_OFFSET BIT(1) + static const struct dw_edma_pcie_data snps_edda_data = { /* eDMA registers location */ .rg.bar = BAR_0, @@ -90,6 +132,77 @@ static const struct dw_edma_pcie_data snps_edda_data = { .rd_ch_cnt = 2, }; +static const struct dw_edma_pcie_data xilinx_mdb_data = { + /* MDB registers location */ + .rg.bar = BAR_0, + .rg.off = SZ_4K, /* 4 Kbytes */ + .rg.sz = SZ_8K, /* 8 Kbytes */ + + /* Other */ + .mf = EDMA_MF_HDMA_NATIVE, + .irqs = 1, + .wr_ch_cnt = 8, + .rd_ch_cnt = 8, +}; + +static const struct dw_edma_pcie_data xilinx_cpm6_dma_data = { + /* MDB registers location */ + .rg.bar = BAR_0, + .rg.off = SZ_4K, /* 4 Kbytes */ + .rg.sz = SZ_8K, /* 8 Kbytes */ + + /* Other */ + .mf = EDMA_MF_HDMA_NATIVE, + .irqs = 1, + .wr_ch_cnt = 8, + .rd_ch_cnt = 8, +}; + +static void dw_edma_set_chan_region_offset(struct dw_edma_pcie_data *pdata, + enum pci_barno bar, off_t start_off, + off_t ll_off_gap, size_t ll_size, + off_t dt_off_gap, size_t dt_size) +{ + u16 wr_ch = pdata->wr_ch_cnt; + u16 rd_ch = pdata->rd_ch_cnt; + off_t off; + u16 i; + + off = start_off; + + /* Write channel LL region */ + for (i = 0; i < wr_ch; i++) { + pdata->ll_wr[i].bar = bar; + pdata->ll_wr[i].off = off; + pdata->ll_wr[i].sz = ll_size; + off += ll_off_gap; + } + + /* Read channel LL region */ + for (i = 0; i < rd_ch; i++) { + pdata->ll_rd[i].bar = bar; + pdata->ll_rd[i].off = off; + pdata->ll_rd[i].sz = ll_size; + off += ll_off_gap; + } + + /* Write channel data region */ + for (i = 0; i < wr_ch; i++) { + pdata->dt_wr[i].bar = bar; + pdata->dt_wr[i].off = off; + pdata->dt_wr[i].sz = dt_size; + off += dt_off_gap; + } + + /* Read channel data region */ + for (i = 0; i < rd_ch; i++) { + pdata->dt_rd[i].bar = bar; + pdata->dt_rd[i].off = off; + pdata->dt_rd[i].sz = dt_size; + off += dt_off_gap; + } +} + static int dw_edma_pcie_irq_vector(struct device *dev, unsigned int nr) { return pci_irq_vector(to_pci_dev(dev), nr); @@ -114,15 +227,15 @@ static const struct dw_edma_plat_ops dw_edma_pcie_plat_ops = { .pci_address = dw_edma_pcie_address, }; -static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, - struct dw_edma_pcie_data *pdata) +static void dw_edma_pcie_get_synopsys_dma_data(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata) { u32 val, map; u16 vsec; u64 off; vsec = pci_find_vsec_capability(pdev, PCI_VENDOR_ID_SYNOPSYS, - DW_PCIE_VSEC_DMA_ID); + DW_PCIE_SYNOPSYS_VSEC_DMA_ID); if (!vsec) return; @@ -131,9 +244,9 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, PCI_VNDR_HEADER_LEN(val) != 0x18) return; - pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability DMA\n"); + pci_dbg(pdev, "Detected Synopsys PCIe Vendor-Specific Extended Capability DMA\n"); pci_read_config_dword(pdev, vsec + 0x8, &val); - map = FIELD_GET(DW_PCIE_VSEC_DMA_MAP, val); + map = FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_MAP, val); if (map != EDMA_MF_EDMA_LEGACY && map != EDMA_MF_EDMA_UNROLL && map != EDMA_MF_HDMA_COMPAT && @@ -141,13 +254,13 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, return; pdata->mf = map; - pdata->rg.bar = FIELD_GET(DW_PCIE_VSEC_DMA_BAR, val); + pdata->rg.bar = FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_BAR, val); pci_read_config_dword(pdev, vsec + 0xc, &val); pdata->wr_ch_cnt = min_t(u16, pdata->wr_ch_cnt, - FIELD_GET(DW_PCIE_VSEC_DMA_WR_CH, val)); + FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_WR_CH, val)); pdata->rd_ch_cnt = min_t(u16, pdata->rd_ch_cnt, - FIELD_GET(DW_PCIE_VSEC_DMA_RD_CH, val)); + FIELD_GET(DW_PCIE_SYNOPSYS_VSEC_DMA_RD_CH, val)); pci_read_config_dword(pdev, vsec + 0x14, &val); off = val; @@ -157,18 +270,146 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, pdata->rg.off = off; } +static void dw_edma_pcie_get_xilinx_dma_data(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata) +{ + u32 val, map; + u16 vsec; + u64 off; + + pdata->devmem_phys_off = DW_PCIE_XILINX_MDB_INVALID_ADDR; + + vsec = pci_find_vsec_capability(pdev, PCI_VENDOR_ID_XILINX, + DW_PCIE_XILINX_MDB_VSEC_DMA_ID); + if (!vsec) + return; + + pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER, &val); + if (PCI_VNDR_HEADER_REV(val) != 0x00 || + PCI_VNDR_HEADER_LEN(val) != 0x18) + return; + + pci_dbg(pdev, "Detected Xilinx PCIe Vendor-Specific Extended Capability DMA\n"); + pci_read_config_dword(pdev, vsec + 0x8, &val); + map = FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_MAP, val); + if (map != EDMA_MF_HDMA_NATIVE) + return; + + pdata->mf = map; + pdata->rg.bar = FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_BAR, val); + + pci_read_config_dword(pdev, vsec + 0xc, &val); + pdata->wr_ch_cnt = min(pdata->wr_ch_cnt, + FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_WR_CH, val)); + pdata->rd_ch_cnt = min(pdata->rd_ch_cnt, + FIELD_GET(DW_PCIE_XILINX_MDB_VSEC_DMA_RD_CH, val)); + + pci_read_config_dword(pdev, vsec + 0x14, &val); + off = val; + pci_read_config_dword(pdev, vsec + 0x10, &val); + off <<= 32; + off |= val; + pdata->rg.off = off; + + vsec = pci_find_vsec_capability(pdev, PCI_VENDOR_ID_XILINX, + DW_PCIE_XILINX_MDB_VSEC_ID); + if (!vsec) + return; + + pci_read_config_dword(pdev, + vsec + DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_HIGH, + &val); + off = val; + pci_read_config_dword(pdev, + vsec + DW_PCIE_XILINX_MDB_DEVMEM_OFF_REG_LOW, + &val); + off <<= 32; + off |= val; + pdata->devmem_phys_off = off; +} + +static int +dw_edma_pcie_parse_synopsys_caps(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata) +{ + dw_edma_pcie_get_synopsys_dma_data(pdev, pdata); + + return 0; +} + +static int +dw_edma_pcie_parse_xilinx_caps(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata) +{ + dw_edma_pcie_get_xilinx_dma_data(pdev, pdata); + + /* + * There is no valid address found for the LL memory space on the + * device side. In the absence of LL base address use the non-LL mode or + * simple mode supported by the HDMA IP. + */ + if (pdata->devmem_phys_off == DW_PCIE_XILINX_MDB_INVALID_ADDR) { + pdata->cfg_non_ll = true; + return 0; + } + + /* + * Configure the channel LL and data blocks if number of channels + * enabled in VSEC capability are more than the channels configured in + * xilinx_mdb_data. + */ + dw_edma_set_chan_region_offset(pdata, BAR_2, 0, + DW_PCIE_XILINX_MDB_LL_OFF_GAP, + DW_PCIE_XILINX_MDB_LL_SIZE, + DW_PCIE_XILINX_MDB_DT_OFF_GAP, + DW_PCIE_XILINX_MDB_DT_SIZE); + + return 0; +} + +static u64 dw_edma_get_phys_addr(struct pci_dev *pdev, + const struct dw_edma_pcie_match_data *match, + struct dw_edma_pcie_data *pdata, + enum pci_barno bar) +{ + if (match->flags & DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF) + return pdata->devmem_phys_off; + + return pci_bus_address(pdev, bar); +} + +static u64 dw_edma_get_block_addr(struct pci_dev *pdev, + const struct dw_edma_pcie_match_data *match, + struct dw_edma_pcie_data *pdata, + const struct dw_edma_block *block) +{ + if (block->paddr_valid) + return block->paddr; + + return dw_edma_get_phys_addr(pdev, match, pdata, block->bar) + + block->off; +} + static int dw_edma_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *pid) { - struct dw_edma_pcie_data *pdata = (void *)pid->driver_data; + const struct dw_edma_pcie_match_data *match = (void *)pid->driver_data; + const struct dw_edma_pcie_data *pdata; struct device *dev = &pdev->dev; struct dw_edma_chip *chip; int err, nr_irqs; int i, mask; - struct dw_edma_pcie_data *vsec_data __free(kfree) = - kmalloc_obj(*vsec_data); - if (!vsec_data) + if (!match) + return -ENODEV; + pdata = match->data; + + if (!pdata) + return -ENODEV; + + struct dw_edma_pcie_data *dma_data __free(kfree) = + kmemdup(pdata, sizeof(*dma_data), GFP_KERNEL); + if (!dma_data) return -ENOMEM; /* Enable PCI device */ @@ -178,23 +419,25 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return err; } - memcpy(vsec_data, pdata, sizeof(struct dw_edma_pcie_data)); + /* Let device-specific discovery override the static template data. */ + if (!match->parse_caps || !match->plat_ops) + return -EINVAL; - /* - * Tries to find if exists a PCIe Vendor-Specific Extended Capability - * for the DMA, if one exists, then reconfigures it. - */ - dw_edma_pcie_get_vsec_dma_data(pdev, vsec_data); + err = match->parse_caps(pdev, dma_data); + if (err) + return err; /* Mapping PCI BAR regions */ - mask = BIT(vsec_data->rg.bar); - for (i = 0; i < vsec_data->wr_ch_cnt; i++) { - mask |= BIT(vsec_data->ll_wr[i].bar); - mask |= BIT(vsec_data->dt_wr[i].bar); + mask = BIT(dma_data->rg.bar); + for (i = 0; i < dma_data->wr_ch_cnt; i++) { + mask |= BIT(dma_data->ll_wr[i].bar); + if (dma_data->dt_wr[i].sz) + mask |= BIT(dma_data->dt_wr[i].bar); } - for (i = 0; i < vsec_data->rd_ch_cnt; i++) { - mask |= BIT(vsec_data->ll_rd[i].bar); - mask |= BIT(vsec_data->dt_rd[i].bar); + for (i = 0; i < dma_data->rd_ch_cnt; i++) { + mask |= BIT(dma_data->ll_rd[i].bar); + if (dma_data->dt_rd[i].sz) + mask |= BIT(dma_data->dt_rd[i].bar); } err = pcim_iomap_regions(pdev, mask, pci_name(pdev)); if (err) { @@ -217,7 +460,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; /* IRQs allocation */ - nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data->irqs, + nr_irqs = pci_alloc_irq_vectors(pdev, 1, dma_data->irqs, PCI_IRQ_MSI | PCI_IRQ_MSIX); if (nr_irqs < 1) { pci_err(pdev, "fail to alloc IRQ vector (number of IRQs=%u)\n", @@ -228,64 +471,75 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, /* Data structure initialization */ chip->dev = dev; - chip->mf = vsec_data->mf; + chip->mf = dma_data->mf; + chip->flags = match->chip_flags; + chip->func_no = PCI_FUNC(pdev->devfn); chip->nr_irqs = nr_irqs; - chip->ops = &dw_edma_pcie_plat_ops; + chip->ops = match->plat_ops; + chip->cfg_non_ll = dma_data->cfg_non_ll; - chip->ll_wr_cnt = vsec_data->wr_ch_cnt; - chip->ll_rd_cnt = vsec_data->rd_ch_cnt; + chip->ll_wr_cnt = dma_data->wr_ch_cnt; + chip->ll_rd_cnt = dma_data->rd_ch_cnt; - chip->reg_base = pcim_iomap_table(pdev)[vsec_data->rg.bar]; + chip->reg_base = pcim_iomap_table(pdev)[dma_data->rg.bar]; if (!chip->reg_base) return -ENOMEM; + if (match->flags & DW_EDMA_PCIE_F_REG_OFFSET) + chip->reg_base += dma_data->rg.off; - for (i = 0; i < chip->ll_wr_cnt; i++) { + for (i = 0; i < chip->ll_wr_cnt && !dma_data->cfg_non_ll; i++) { struct dw_edma_region *ll_region = &chip->ll_region_wr[i]; struct dw_edma_region *dt_region = &chip->dt_region_wr[i]; - struct dw_edma_block *ll_block = &vsec_data->ll_wr[i]; - struct dw_edma_block *dt_block = &vsec_data->dt_wr[i]; + struct dw_edma_block *ll_block = &dma_data->ll_wr[i]; + struct dw_edma_block *dt_block = &dma_data->dt_wr[i]; ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar]; if (!ll_region->vaddr.io) return -ENOMEM; ll_region->vaddr.io += ll_block->off; - ll_region->paddr = pci_bus_address(pdev, ll_block->bar); - ll_region->paddr += ll_block->off; + ll_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data, + ll_block); ll_region->sz = ll_block->sz; + if (!dt_block->sz) + continue; + dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar]; if (!dt_region->vaddr.io) return -ENOMEM; dt_region->vaddr.io += dt_block->off; - dt_region->paddr = pci_bus_address(pdev, dt_block->bar); - dt_region->paddr += dt_block->off; + dt_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data, + dt_block); dt_region->sz = dt_block->sz; } - for (i = 0; i < chip->ll_rd_cnt; i++) { + for (i = 0; i < chip->ll_rd_cnt && !dma_data->cfg_non_ll; i++) { struct dw_edma_region *ll_region = &chip->ll_region_rd[i]; struct dw_edma_region *dt_region = &chip->dt_region_rd[i]; - struct dw_edma_block *ll_block = &vsec_data->ll_rd[i]; - struct dw_edma_block *dt_block = &vsec_data->dt_rd[i]; + struct dw_edma_block *ll_block = &dma_data->ll_rd[i]; + struct dw_edma_block *dt_block = &dma_data->dt_rd[i]; ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar]; if (!ll_region->vaddr.io) return -ENOMEM; ll_region->vaddr.io += ll_block->off; - ll_region->paddr = pci_bus_address(pdev, ll_block->bar); - ll_region->paddr += ll_block->off; + ll_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data, + ll_block); ll_region->sz = ll_block->sz; + if (!dt_block->sz) + continue; + dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar]; if (!dt_region->vaddr.io) return -ENOMEM; dt_region->vaddr.io += dt_block->off; - dt_region->paddr = pci_bus_address(pdev, dt_block->bar); - dt_region->paddr += dt_block->off; + dt_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data, + dt_block); dt_region->sz = dt_block->sz; } @@ -302,32 +556,40 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, pci_dbg(pdev, "Version:\tUnknown (0x%x)\n", chip->mf); pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p)\n", - vsec_data->rg.bar, vsec_data->rg.off, vsec_data->rg.sz, + dma_data->rg.bar, dma_data->rg.off, dma_data->rg.sz, chip->reg_base); for (i = 0; i < chip->ll_wr_cnt; i++) { pci_dbg(pdev, "L. List:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data->ll_wr[i].bar, - vsec_data->ll_wr[i].off, chip->ll_region_wr[i].sz, + i, dma_data->ll_wr[i].bar, + dma_data->ll_wr[i].off, chip->ll_region_wr[i].sz, chip->ll_region_wr[i].vaddr.io, &chip->ll_region_wr[i].paddr); + if (!dma_data->dt_wr[i].sz) + continue; + pci_dbg(pdev, "Data:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data->dt_wr[i].bar, - vsec_data->dt_wr[i].off, chip->dt_region_wr[i].sz, - chip->dt_region_wr[i].vaddr.io, &chip->dt_region_wr[i].paddr); + i, dma_data->dt_wr[i].bar, + dma_data->dt_wr[i].off, chip->dt_region_wr[i].sz, + chip->dt_region_wr[i].vaddr.io, + &chip->dt_region_wr[i].paddr); } for (i = 0; i < chip->ll_rd_cnt; i++) { pci_dbg(pdev, "L. List:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data->ll_rd[i].bar, - vsec_data->ll_rd[i].off, chip->ll_region_rd[i].sz, + i, dma_data->ll_rd[i].bar, + dma_data->ll_rd[i].off, chip->ll_region_rd[i].sz, chip->ll_region_rd[i].vaddr.io, &chip->ll_region_rd[i].paddr); + if (!dma_data->dt_rd[i].sz) + continue; + pci_dbg(pdev, "Data:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data->dt_rd[i].bar, - vsec_data->dt_rd[i].off, chip->dt_region_rd[i].sz, - chip->dt_region_rd[i].vaddr.io, &chip->dt_region_rd[i].paddr); + i, dma_data->dt_rd[i].bar, + dma_data->dt_rd[i].off, chip->dt_region_rd[i].sz, + chip->dt_region_rd[i].vaddr.io, + &chip->dt_region_rd[i].paddr); } pci_dbg(pdev, "Nr. IRQs:\t%u\n", chip->nr_irqs); @@ -360,13 +622,34 @@ static void dw_edma_pcie_remove(struct pci_dev *pdev) err = dw_edma_remove(chip); if (err) pci_warn(pdev, "can't remove device properly: %d\n", err); - - /* Freeing IRQs */ - pci_free_irq_vectors(pdev); } +static const struct dw_edma_pcie_match_data snps_edda_match_data = { + .data = &snps_edda_data, + .plat_ops = &dw_edma_pcie_plat_ops, + .parse_caps = dw_edma_pcie_parse_synopsys_caps, +}; + +static const struct dw_edma_pcie_match_data xilinx_mdb_match_data = { + .data = &xilinx_mdb_data, + .plat_ops = &dw_edma_pcie_plat_ops, + .parse_caps = dw_edma_pcie_parse_xilinx_caps, + .flags = DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF, +}; + +static const struct dw_edma_pcie_match_data xilinx_cpm6_dma_match_data = { + .data = &xilinx_cpm6_dma_data, + .plat_ops = &dw_edma_pcie_plat_ops, + .parse_caps = dw_edma_pcie_parse_xilinx_caps, + .flags = DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF, +}; + static const struct pci_device_id dw_edma_pcie_id_table[] = { - { PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_data) }, + { PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_match_data) }, + { PCI_VDEVICE(XILINX, PCI_DEVICE_ID_XILINX_B054), + .driver_data = (kernel_ulong_t)&xilinx_mdb_match_data }, + { PCI_VDEVICE(XILINX, PCI_DEVICE_ID_XILINX_B00F), + .driver_data = (kernel_ulong_t)&xilinx_cpm6_dma_match_data }, { } }; MODULE_DEVICE_TABLE(pci, dw_edma_pcie_id_table); diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index b75fdaffad9a..abc1bd4c0be2 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -7,6 +7,7 @@ */ #include <linux/bitfield.h> +#include <linux/iopoll.h> #include <linux/irqreturn.h> #include <linux/io-64-nonatomic-lo-hi.h> @@ -25,6 +26,8 @@ enum dw_edma_control { DW_EDMA_V0_LLE = BIT(9), }; +#define EDMA_V0_FUNC_NUM_MASK GENMASK(16, 12) + static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) { return dw->chip->reg_base; @@ -159,7 +162,93 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, #define GET_CH_32(dw, dir, ch, name) \ readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name)) +static u32 dw_edma_v0_func_num(struct dw_edma_chan *chan) +{ + return FIELD_PREP(EDMA_V0_FUNC_NUM_MASK, chan->func_no); +} + /* eDMA management callbacks */ +static void dw_edma_v0_core_ch_power(struct dw_edma *dw, + enum dw_edma_dir dir, u16 id, bool enable) +{ + u32 value = enable ? BIT(0) : 0; + + if (WARN_ON_ONCE(id >= EDMA_V0_MAX_NR_CH)) + return; + + switch (id) { + case 0: + SET_RW_COMPAT(dw, dir, ch0_pwr_en, value); + break; + case 1: + SET_RW_COMPAT(dw, dir, ch1_pwr_en, value); + break; + case 2: + SET_RW_COMPAT(dw, dir, ch2_pwr_en, value); + break; + case 3: + SET_RW_COMPAT(dw, dir, ch3_pwr_en, value); + break; + case 4: + SET_RW_COMPAT(dw, dir, ch4_pwr_en, value); + break; + case 5: + SET_RW_COMPAT(dw, dir, ch5_pwr_en, value); + break; + case 6: + SET_RW_COMPAT(dw, dir, ch6_pwr_en, value); + break; + case 7: + SET_RW_COMPAT(dw, dir, ch7_pwr_en, value); + break; + } +} + +static int dw_edma_v0_core_engine_disable(struct dw_edma *dw, + enum dw_edma_dir dir) +{ + u32 value; + int ret; + + SET_RW_32(dw, dir, engine_en, 0); + ret = read_poll_timeout(GET_RW_32, value, !(value & BIT(0)), 100, + 200000, false, dw, dir, engine_en); + if (ret) + dev_warn(dw->chip->dev, "%s engine did not stop within 200ms\n", + dir == EDMA_DIR_WRITE ? "write" : "read"); + + return ret; +} + +static int dw_edma_v0_core_dir_off(struct dw_edma *dw, enum dw_edma_dir dir) +{ + u16 count, id; + int ret = 0; + + scoped_guard(raw_spinlock_irqsave, &dw->lock) + SET_RW_32(dw, dir, int_mask, + EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); + + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { + /* + * DWC PCIe Controller Databook 6.10a-lca06, "Legacy DMA + * and HDMA Software Compatibility": HDMA compatibility mode + * does not implement ENGINE_EN, but retains CHi_PWR_EN for + * per-channel enable and disable. + */ + count = dir == EDMA_DIR_WRITE ? dw->wr_ch_cnt : dw->rd_ch_cnt; + for (id = 0; id < count; id++) + dw_edma_v0_core_ch_power(dw, dir, id, false); + } else { + ret = dw_edma_v0_core_engine_disable(dw, dir); + } + + SET_RW_32(dw, dir, int_clear, + EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); + + return ret; +} + static void dw_edma_v0_core_off(struct dw_edma *dw) { SET_BOTH_32(dw, int_mask, @@ -169,6 +258,33 @@ static void dw_edma_v0_core_off(struct dw_edma *dw) SET_BOTH_32(dw, engine_en, 0); } +static int dw_edma_v0_core_quiesce(struct dw_edma *dw) +{ + int ret = 0; + int err; + + if (dw->wr_ch_cnt) + ret = dw_edma_v0_core_dir_off(dw, EDMA_DIR_WRITE); + if (dw->rd_ch_cnt) { + err = dw_edma_v0_core_dir_off(dw, EDMA_DIR_READ); + if (!ret) + ret = err; + } + + return ret; +} + +/* + * The unrolled eDMA and HDMA compatibility register maps share interrupt + * control per direction, so the whole direction is quiesced. Callers must + * own the direction entirely and prevent the peer from programming it after + * this point. Partial ownership mode validates direction granularity. + */ +static int dw_edma_v0_core_ch_quiesce(struct dw_edma_chan *chan) +{ + return dw_edma_v0_core_dir_off(chan->dw, chan->dir); +} + static u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { u32 num_ch; @@ -218,18 +334,6 @@ static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan) FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id))); } -static u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir) -{ - return FIELD_GET(EDMA_V0_DONE_INT_MASK, - GET_RW_32(dw, dir, int_status)); -} - -static u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir) -{ - return FIELD_GET(EDMA_V0_ABORT_INT_MASK, - GET_RW_32(dw, dir, int_status)); -} - static irqreturn_t dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, dw_edma_handler_t done, dw_edma_handler_t abort) @@ -239,7 +343,8 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, irqreturn_t ret = IRQ_NONE; struct dw_edma_chan *chan; unsigned long off; - u32 mask; + unsigned long *mask; + u32 sts; if (dir == EDMA_DIR_WRITE) { total = dw->wr_ch_cnt; @@ -251,22 +356,38 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, mask = dw_irq->rd_mask; } - val = dw_edma_v0_core_status_done_int(dw, dir); - val &= mask; + /* + * DONE and ABORT status share one register, and on remote setups + * every read is a non-posted round trip across the PCIe link. Take + * one snapshot and derive both views from it. An abort raised + * after the snapshot is deferred, not lost: only bits observed in + * the snapshot are ever cleared below, so its status remains set and + * triggers another handler pass. + */ + sts = GET_RW_32(dw, dir, int_status); + + val = FIELD_GET(EDMA_V0_DONE_INT_MASK, sts); + val &= *mask; for_each_set_bit(pos, &val, total) { chan = &dw->chan[pos + off]; + if (unlikely(dw_edma_core_ch_ignore_irq(chan))) + continue; + dw_edma_v0_core_clear_done_int(chan); done(chan); ret = IRQ_HANDLED; } - val = dw_edma_v0_core_status_abort_int(dw, dir); - val &= mask; + val = FIELD_GET(EDMA_V0_ABORT_INT_MASK, sts); + val &= *mask; for_each_set_bit(pos, &val, total) { chan = &dw->chan[pos + off]; + if (unlikely(dw_edma_core_ch_ignore_irq(chan))) + continue; + dw_edma_v0_core_clear_abort_int(chan); abort(chan); @@ -276,77 +397,90 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, return ret; } -static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, +static void dw_edma_v0_write_ll_data(struct dw_edma_chan *chan, int i, u32 control, u32 size, u64 sar, u64 dar) { ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); - if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { - struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + struct dw_edma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs; - lli->control = control; lli->transfer_size = size; lli->sar.reg = sar; lli->dar.reg = dar; + dma_wmb(); + lli->control = control; } else { - struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; + struct dw_edma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs; - writel(control, &lli->control); writel(size, &lli->transfer_size); writeq(sar, &lli->sar.reg); writeq(dar, &lli->dar.reg); + writel(control, &lli->control); } } -static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk, +static void dw_edma_v0_write_ll_link(struct dw_edma_chan *chan, int i, u32 control, u64 pointer) { ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); - if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { - struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + struct dw_edma_v0_llp *llp = chan->ll_region.vaddr.mem + ofs; - llp->control = control; llp->llp.reg = pointer; + dma_wmb(); + llp->control = control; } else { - struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; + struct dw_edma_v0_llp __iomem *llp = chan->ll_region.vaddr.io + ofs; - writel(control, &llp->control); writeq(pointer, &llp->llp.reg); + writel(control, &llp->control); } } -static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) +static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan) { - struct dw_edma_burst *child; - struct dw_edma_chan *chan = chunk->chan; - u32 control = 0, i = 0; - int j; - - if (chunk->cb) - control = DW_EDMA_V0_CB; - - j = chunk->bursts_alloc; - list_for_each_entry(child, &chunk->burst->list, list) { - j--; - if (!j) { - control |= DW_EDMA_V0_LIE; - if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - control |= DW_EDMA_V0_RIE; - } - - dw_edma_v0_write_ll_data(chunk, i++, control, child->sz, - child->sar, child->dar); - } - - control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB; - if (!chunk->cb) - control |= DW_EDMA_V0_CB; + struct dw_edma *dw = chan->dw; + unsigned long flags; + u32 tmp; - dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); + /* Enable engine */ + SET_RW_32(dw, chan->dir, engine_en, BIT(0)); + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) + dw_edma_v0_core_ch_power(dw, chan->dir, chan->id, true); + /* Interrupt mask/unmask - done, abort */ + raw_spin_lock_irqsave(&dw->lock, flags); + + tmp = GET_RW_32(dw, chan->dir, int_mask); + if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE) { + tmp |= FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); + tmp |= FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); + } else { + tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); + tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); + } + SET_RW_32(dw, chan->dir, int_mask, tmp); + /* Linked list error */ + tmp = GET_RW_32(dw, chan->dir, linked_list_err_en); + tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id)); + SET_RW_32(dw, chan->dir, linked_list_err_en, tmp); + + raw_spin_unlock_irqrestore(&dw->lock, flags); + + /* Channel control */ + SET_CH_32(dw, chan->dir, chan->id, ch_control1, + DW_EDMA_V0_CCS | DW_EDMA_V0_LLE | + dw_edma_v0_func_num(chan)); + /* Linked list */ + /* llp is not aligned on 64bit -> keep 32bit accesses */ + SET_CH_32(dw, chan->dir, chan->id, llp.lsb, + lower_32_bits(chan->ll_region.paddr)); + SET_CH_32(dw, chan->dir, chan->id, llp.msb, + upper_32_bits(chan->ll_region.paddr)); } -static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk) +static void dw_edma_v0_sync_ll_data(struct dw_edma_chan *chan) { /* * In case of remote eDMA engine setup, the DW PCIe RP/EP internal @@ -356,82 +490,8 @@ static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk) * LL memory in a hope that the MRd TLP will return only after the * last MWr TLP is completed */ - if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - readl(chunk->ll_region.vaddr.io); -} - -static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) -{ - struct dw_edma_chan *chan = chunk->chan; - struct dw_edma *dw = chan->dw; - u32 tmp; - - dw_edma_v0_core_write_chunk(chunk); - - if (first) { - /* Enable engine */ - SET_RW_32(dw, chan->dir, engine_en, BIT(0)); - if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { - switch (chan->id) { - case 0: - SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en, - BIT(0)); - break; - case 1: - SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en, - BIT(0)); - break; - case 2: - SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en, - BIT(0)); - break; - case 3: - SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en, - BIT(0)); - break; - case 4: - SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en, - BIT(0)); - break; - case 5: - SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en, - BIT(0)); - break; - case 6: - SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en, - BIT(0)); - break; - case 7: - SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en, - BIT(0)); - break; - } - } - /* Interrupt unmask - done, abort */ - tmp = GET_RW_32(dw, chan->dir, int_mask); - tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); - tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); - SET_RW_32(dw, chan->dir, int_mask, tmp); - /* Linked list error */ - tmp = GET_RW_32(dw, chan->dir, linked_list_err_en); - tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id)); - SET_RW_32(dw, chan->dir, linked_list_err_en, tmp); - /* Channel control */ - SET_CH_32(dw, chan->dir, chan->id, ch_control1, - (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE)); - /* Linked list */ - /* llp is not aligned on 64bit -> keep 32bit accesses */ - SET_CH_32(dw, chan->dir, chan->id, llp.lsb, - lower_32_bits(chunk->ll_region.paddr)); - SET_CH_32(dw, chan->dir, chan->id, llp.msb, - upper_32_bits(chunk->ll_region.paddr)); - } - - dw_edma_v0_sync_ll_data(chunk); - - /* Doorbell */ - SET_RW_32(dw, chan->dir, doorbell, - FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id)); + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + readl(chan->ll_region.vaddr.io); } static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan) @@ -503,20 +563,99 @@ static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan) } } +static void +dw_edma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst, + u32 idx, bool cb, bool irq) +{ + u32 control = 0; + + if (cb) + control |= DW_EDMA_V0_CB; + + if (irq) { + control |= DW_EDMA_V0_LIE; + + /* + * A local instance never issues transfers on a remote-routed + * channel: on CHIP_LOCAL instances, REMOTE routing denotes a + * channel handed over to the remote side, which programs the + * linked list through its own instance. The remote-only + * recipe (LIE|RIE with the local interrupt masked) is thus + * applied by the instance that owns the transfer, and the + * LIE-only write below never executes for a remote-routed + * channel. + */ + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) && + chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE) + control |= DW_EDMA_V0_RIE; + } + + dw_edma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar, + burst->dar); +} + +static void +dw_edma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr) +{ + u32 control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB; + + if (!cb) + control |= DW_EDMA_V0_CB; + + dw_edma_v0_write_ll_link(chan, idx, control, addr); +} + +static void dw_edma_v0_core_ch_doorbell(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + + dw_edma_v0_sync_ll_data(chan); + + /* Doorbell */ + SET_RW_32(dw, chan->dir, doorbell, + FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id)); +} + /* eDMA debugfs callbacks */ static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw) { dw_edma_v0_debugfs_on(dw); } +static void dw_edma_v0_core_ack_emulated_irq(struct dw_edma *dw) +{ + /* + * Interrupt emulation may assert the IRQ without setting + * DONE/ABORT status bits. A zero write to INT_CLEAR deasserts the + * emulated IRQ, while being a no-op for real interrupts. + */ + SET_BOTH_32(dw, int_clear, 0); +} + +static resource_size_t dw_edma_v0_core_db_offset(struct dw_edma *dw) +{ + /* + * rd_int_status is chosen arbitrarily, but wr_int_status would be + * equally suitable. + */ + return offsetof(struct dw_edma_v0_regs, rd_int_status); +} + static const struct dw_edma_core_ops dw_edma_v0_core = { .off = dw_edma_v0_core_off, + .quiesce = dw_edma_v0_core_quiesce, + .ch_quiesce = dw_edma_v0_core_ch_quiesce, .ch_count = dw_edma_v0_core_ch_count, .ch_status = dw_edma_v0_core_ch_status, .handle_int = dw_edma_v0_core_handle_int, - .start = dw_edma_v0_core_start, + .ll_data = dw_edma_v0_core_ll_data, + .ll_link = dw_edma_v0_core_ll_link, + .ch_doorbell = dw_edma_v0_core_ch_doorbell, + .ch_enable = dw_edma_v0_core_ch_enable, .ch_config = dw_edma_v0_core_ch_config, .debugfs_on = dw_edma_v0_core_debugfs_on, + .ack_emulated_irq = dw_edma_v0_core_ack_emulated_irq, + .db_offset = dw_edma_v0_core_db_offset, }; void dw_edma_v0_core_register(struct dw_edma *dw) diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index e3f8db4fe909..36ee72efcd31 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -49,20 +49,82 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) writel(value, &(__dw_ch_regs(dw, EDMA_DIR_READ, ch)->name)); \ } while (0) +static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val) +{ + val &= ~(HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN | + HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_REMOTE_STOP_INT_EN | + HDMA_V0_ABORT_INT_MASK | HDMA_V0_STOP_INT_MASK); + + /* + * HDMA_INT_STATUS.STOP and .ABORT are latched only when LSIE and + * LAIE are enabled. A remote handler needs those status bits to + * identify the source of the IMWr, so keep local generation enabled + * and mask the local interrupt pins instead. + */ + val |= HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_LOCAL_STOP_INT_EN; + + if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE) + val |= HDMA_V0_REMOTE_ABORT_INT_EN | + HDMA_V0_REMOTE_STOP_INT_EN | + HDMA_V0_ABORT_INT_MASK | HDMA_V0_STOP_INT_MASK; + + return val; +} + /* HDMA management callbacks */ +static void dw_hdma_v0_core_ch_off(struct dw_edma *dw, enum dw_edma_dir dir, + u16 id) +{ + SET_CH_32(dw, dir, id, int_setup, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, ch_en, 0); + SET_CH_32(dw, dir, id, int_clear, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); +} + static void dw_hdma_v0_core_off(struct dw_edma *dw) { int id; + enum dw_edma_dir dir; + + dir = EDMA_DIR_WRITE; + for (id = 0; id < dw->wr_ch_cnt; id++) { + SET_CH_32(dw, dir, id, int_setup, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, int_clear, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, ch_en, 0); + } - for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) { - SET_BOTH_CH_32(dw, id, int_setup, - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); - SET_BOTH_CH_32(dw, id, int_clear, - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); - SET_BOTH_CH_32(dw, id, ch_en, 0); + dir = EDMA_DIR_READ; + for (id = 0; id < dw->rd_ch_cnt; id++) { + SET_CH_32(dw, dir, id, int_setup, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, int_clear, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, ch_en, 0); } } +static int dw_hdma_v0_core_quiesce(struct dw_edma *dw) +{ + int id; + + for (id = 0; id < dw->wr_ch_cnt; id++) + dw_hdma_v0_core_ch_off(dw, EDMA_DIR_WRITE, id); + for (id = 0; id < dw->rd_ch_cnt; id++) + dw_hdma_v0_core_ch_off(dw, EDMA_DIR_READ, id); + + return 0; +} + +static int dw_hdma_v0_core_ch_quiesce(struct dw_edma_chan *chan) +{ + dw_hdma_v0_core_ch_off(chan->dw, chan->dir, chan->id); + + return 0; +} + static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { /* @@ -79,7 +141,7 @@ static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan) u32 tmp; tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK, - GET_CH_32(dw, chan->id, chan->dir, ch_stat)); + GET_CH_32(dw, chan->dir, chan->id, ch_stat)); if (tmp == 1) return DMA_IN_PROGRESS; @@ -118,7 +180,7 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, unsigned long total, pos, val; irqreturn_t ret = IRQ_NONE; struct dw_edma_chan *chan; - unsigned long off, mask; + unsigned long off, *mask; if (dir == EDMA_DIR_WRITE) { total = dw->wr_ch_cnt; @@ -130,8 +192,10 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, mask = dw_irq->rd_mask; } - for_each_set_bit(pos, &mask, total) { + for_each_set_bit(pos, mask, total) { chan = &dw->chan[pos + off]; + if (unlikely(dw_edma_core_ch_ignore_irq(chan))) + continue; val = dw_hdma_v0_core_status_int(chan); if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) { @@ -152,66 +216,73 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, return ret; } -static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, +static void dw_hdma_v0_write_ll_data(struct dw_edma_chan *chan, int i, u32 control, u32 size, u64 sar, u64 dar) { ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli); - if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { - struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + struct dw_hdma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs; - lli->control = control; lli->transfer_size = size; lli->sar.reg = sar; lli->dar.reg = dar; + dma_wmb(); + lli->control = control; } else { - struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; + struct dw_hdma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs; - writel(control, &lli->control); writel(size, &lli->transfer_size); writeq(sar, &lli->sar.reg); writeq(dar, &lli->dar.reg); + writel(control, &lli->control); } } -static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk, +static void dw_hdma_v0_write_ll_link(struct dw_edma_chan *chan, int i, u32 control, u64 pointer) { ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli); - if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { - struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + struct dw_hdma_v0_llp *llp = chan->ll_region.vaddr.mem + ofs; - llp->control = control; llp->llp.reg = pointer; + dma_wmb(); + llp->control = control; } else { - struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; + struct dw_hdma_v0_llp __iomem *llp = chan->ll_region.vaddr.io + ofs; - writel(control, &llp->control); writeq(pointer, &llp->llp.reg); + writel(control, &llp->control); } } -static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) +static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan) { - struct dw_edma_burst *child; - u32 control = 0, i = 0; - - if (chunk->cb) - control = DW_HDMA_V0_CB; - - list_for_each_entry(child, &chunk->burst->list, list) - dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz, - child->sar, child->dar); - - control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB; - if (!chunk->cb) - control |= DW_HDMA_V0_CB; + struct dw_edma *dw = chan->dw; + u32 tmp; - dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); + /* Enable engine */ + SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0)); + /* Interrupt unmask - stop, abort */ + tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup); + tmp = dw_hdma_v0_core_int_setup(chan, tmp); + SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp); + /* Channel control */ + SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN); + /* Linked list */ + /* llp is not aligned on 64bit -> keep 32bit accesses */ + SET_CH_32(dw, chan->dir, chan->id, llp.lsb, + lower_32_bits(chan->ll_region.paddr)); + SET_CH_32(dw, chan->dir, chan->id, llp.msb, + upper_32_bits(chan->ll_region.paddr)); + /* Set consumer cycle */ + SET_CH_32(dw, chan->dir, chan->id, cycle_sync, + HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); } -static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk) +static void dw_hdma_v0_sync_ll_data(struct dw_edma_chan *chan) { /* * In case of remote HDMA engine setup, the DW PCIe RP/EP internal @@ -221,46 +292,45 @@ static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk) * LL memory in a hope that the MRd TLP will return only after the * last MWr TLP is completed */ - if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - readl(chunk->ll_region.vaddr.io); + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + readl(chan->ll_region.vaddr.io); } -static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) +static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chan *chan, + struct dw_edma_burst *child) { - struct dw_edma_chan *chan = chunk->chan; struct dw_edma *dw = chan->dw; - u32 tmp; + u32 val; - dw_hdma_v0_core_write_chunk(chunk); - - if (first) { - /* Enable engine */ - SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0)); - /* Interrupt unmask - stop, abort */ - tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup); - tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); - /* Interrupt enable - stop, abort */ - tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN; - if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN; - SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp); - /* Channel control */ - SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN); - /* Linked list */ - /* llp is not aligned on 64bit -> keep 32bit accesses */ - SET_CH_32(dw, chan->dir, chan->id, llp.lsb, - lower_32_bits(chunk->ll_region.paddr)); - SET_CH_32(dw, chan->dir, chan->id, llp.msb, - upper_32_bits(chunk->ll_region.paddr)); - } - /* Set consumer cycle */ - SET_CH_32(dw, chan->dir, chan->id, cycle_sync, - HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); + SET_CH_32(dw, chan->dir, chan->id, ch_en, HDMA_V0_CH_EN); - dw_hdma_v0_sync_ll_data(chunk); + /* Source address */ + SET_CH_32(dw, chan->dir, chan->id, sar.lsb, + lower_32_bits(child->sar)); + SET_CH_32(dw, chan->dir, chan->id, sar.msb, + upper_32_bits(child->sar)); - /* Doorbell */ - SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); + /* Destination address */ + SET_CH_32(dw, chan->dir, chan->id, dar.lsb, + lower_32_bits(child->dar)); + SET_CH_32(dw, chan->dir, chan->id, dar.msb, + upper_32_bits(child->dar)); + + /* Transfer size */ + SET_CH_32(dw, chan->dir, chan->id, transfer_size, child->sz); + + /* Interrupt setup */ + val = GET_CH_32(dw, chan->dir, chan->id, int_setup); + val = dw_hdma_v0_core_int_setup(chan, val); + SET_CH_32(dw, chan->dir, chan->id, int_setup, val); + + /* Channel control setup */ + val = GET_CH_32(dw, chan->dir, chan->id, control1); + val &= ~HDMA_V0_LINKLIST_EN; + SET_CH_32(dw, chan->dir, chan->id, control1, val); + + SET_CH_32(dw, chan->dir, chan->id, doorbell, + HDMA_V0_DOORBELL_START); } static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan) @@ -275,6 +345,43 @@ static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan) SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi); /* config MSI data */ SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data); + /* Configure the requester function number used by outbound TLPs. */ + SET_CH_32(dw, chan->dir, chan->id, func_num, + FIELD_PREP(HDMA_V0_FUNC_NUM_PF_MASK, chan->func_no)); +} + +static void +dw_hdma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst, + u32 idx, bool cb, bool irq) +{ + u32 control = 0; + + if (cb) + control |= DW_HDMA_V0_CB; + + dw_hdma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar, + burst->dar); +} + +static void +dw_hdma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr) +{ + u32 control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB; + + if (!cb) + control |= DW_HDMA_V0_CB; + + dw_hdma_v0_write_ll_link(chan, idx, control, addr); +} + +static void dw_hdma_v0_core_ch_doorbell(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + + dw_hdma_v0_sync_ll_data(chan); + + /* Doorbell */ + SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); } /* HDMA debugfs callbacks */ @@ -283,14 +390,27 @@ static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw) dw_hdma_v0_debugfs_on(dw); } +static resource_size_t dw_hdma_v0_core_db_offset(struct dw_edma *dw) +{ + /* Implement once the correct offset is known. */ + return ~0; +} + static const struct dw_edma_core_ops dw_hdma_v0_core = { .off = dw_hdma_v0_core_off, + .quiesce = dw_hdma_v0_core_quiesce, + .ch_quiesce = dw_hdma_v0_core_ch_quiesce, .ch_count = dw_hdma_v0_core_ch_count, .ch_status = dw_hdma_v0_core_ch_status, .handle_int = dw_hdma_v0_core_handle_int, - .start = dw_hdma_v0_core_start, + .non_ll_start = dw_hdma_v0_core_non_ll_start, + .ll_data = dw_hdma_v0_core_ll_data, + .ll_link = dw_hdma_v0_core_ll_link, + .ch_doorbell = dw_hdma_v0_core_ch_doorbell, + .ch_enable = dw_hdma_v0_core_ch_enable, .ch_config = dw_hdma_v0_core_ch_config, .debugfs_on = dw_hdma_v0_core_debugfs_on, + .db_offset = dw_hdma_v0_core_db_offset, }; void dw_hdma_v0_core_register(struct dw_edma *dw) diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h index eab5fd7177e5..2124c162a62f 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h +++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h @@ -11,7 +11,8 @@ #include <linux/dmaengine.h> -#define HDMA_V0_MAX_NR_CH 8 +#define HDMA_V0_MAX_NR_CH 64 +#define HDMA_V0_CH_EN BIT(0) #define HDMA_V0_LOCAL_ABORT_INT_EN BIT(6) #define HDMA_V0_REMOTE_ABORT_INT_EN BIT(5) #define HDMA_V0_LOCAL_STOP_INT_EN BIT(4) @@ -23,6 +24,7 @@ #define HDMA_V0_CONSUMER_CYCLE_BIT BIT(0) #define HDMA_V0_DOORBELL_START BIT(0) #define HDMA_V0_CH_STATUS_MASK GENMASK(1, 0) +#define HDMA_V0_FUNC_NUM_PF_MASK GENMASK(7, 0) struct dw_hdma_v0_ch_regs { u32 ch_en; /* 0x0000 */ diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c index 8eceb96d058c..311e55a97ba9 100644 --- a/drivers/dma/ep93xx_dma.c +++ b/drivers/dma/ep93xx_dma.c @@ -20,7 +20,6 @@ #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/module.h> -#include <linux/mod_devicetable.h> #include <linux/of_dma.h> #include <linux/overflow.h> #include <linux/platform_device.h> @@ -1587,18 +1586,11 @@ static const struct of_device_id ep93xx_dma_of_ids[] = { }; MODULE_DEVICE_TABLE(of, ep93xx_dma_of_ids); -static const struct platform_device_id ep93xx_dma_driver_ids[] = { - { "ep93xx-dma-m2p", 0 }, - { "ep93xx-dma-m2m", 1 }, - { }, -}; - static struct platform_driver ep93xx_dma_driver = { .driver = { .name = "ep93xx-dma", .of_match_table = ep93xx_dma_of_ids, }, - .id_table = ep93xx_dma_driver_ids, .probe = ep93xx_dma_probe, }; diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index 6a38738e56e2..bb7531c456df 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -905,8 +905,7 @@ void fsl_edma_free_chan_resources(struct dma_chan *chan) fsl_chan->is_sw = false; fsl_chan->srcid = 0; fsl_chan->is_remote = false; - if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_CHCLK) - clk_disable_unprepare(fsl_chan->clk); + clk_disable_unprepare(fsl_chan->clk); } void fsl_edma_cleanup_vchan(struct dma_device *dmadev) diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index dbcdd1e68319..d9fb717b5b53 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -317,10 +317,8 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, return NULL; i = fsl_chan - fsl_edma->chans; - fsl_chan->priority = dma_spec->args[1]; - fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; - fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; - fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; + if (!b_chmux && i != dma_spec->args[0]) + continue; if ((dma_spec->args[2] & FSL_EDMA_EVEN_CH) && (i & 0x1)) continue; @@ -328,17 +326,15 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1)) continue; - if (!b_chmux && i == dma_spec->args[0]) { - chan = dma_get_slave_channel(chan); - chan->device->privatecnt++; - return chan; - } else if (b_chmux && !fsl_chan->srcid) { - /* if controller support channel mux, choose a free channel */ - chan = dma_get_slave_channel(chan); - chan->device->privatecnt++; - fsl_chan->srcid = dma_spec->args[0]; - return chan; - } + fsl_chan->srcid = dma_spec->args[0]; + fsl_chan->priority = dma_spec->args[1]; + fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; + fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; + fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; + + chan = dma_get_slave_channel(chan); + chan->device->privatecnt++; + return chan; } return NULL; } @@ -418,6 +414,8 @@ static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engi errirq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s-err", dev_name(&pdev->dev)); + if (!errirq_name) + return -ENOMEM; ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, fsl_edma3_err_handler_shared, 0, errirq_name, fsl_edma); @@ -709,16 +707,14 @@ static int fsl_edma_probe(struct platform_device *pdev) int ret, i; drvdata = device_get_match_data(&pdev->dev); - if (!drvdata) { - dev_err(&pdev->dev, "unable to find driver data\n"); - return -EINVAL; - } + if (!drvdata) + return dev_err_probe(&pdev->dev, -EINVAL, + "unable to find driver data\n"); ret = of_property_read_u32(np, "dma-channels", &chans); - if (ret) { - dev_err(&pdev->dev, "Can't get dma-channels.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't get dma-channels.\n"); fsl_edma = devm_kzalloc(&pdev->dev, struct_size(fsl_edma, chans, chans), GFP_KERNEL); @@ -742,10 +738,10 @@ static int fsl_edma_probe(struct platform_device *pdev) if (drvdata->flags & FSL_EDMA_DRV_HAS_DMACLK) { fsl_edma->dmaclk = devm_clk_get_enabled(&pdev->dev, "dma"); - if (IS_ERR(fsl_edma->dmaclk)) { - dev_err(&pdev->dev, "Missing DMA block clock.\n"); - return PTR_ERR(fsl_edma->dmaclk); - } + if (IS_ERR(fsl_edma->dmaclk)) + return dev_err_probe(&pdev->dev, + PTR_ERR(fsl_edma->dmaclk), + "Missing DMA block clock.\n"); } ret = of_property_read_variable_u32_array(np, "dma-channel-mask", chan_mask, 1, 2); @@ -769,11 +765,10 @@ static int fsl_edma_probe(struct platform_device *pdev) sprintf(clkname, "dmamux%d", i); fsl_edma->muxclk[i] = devm_clk_get_enabled(&pdev->dev, clkname); - if (IS_ERR(fsl_edma->muxclk[i])) { - dev_err(&pdev->dev, "Missing DMAMUX block clock.\n"); - /* on error: disable all previously enabled clks */ - return PTR_ERR(fsl_edma->muxclk[i]); - } + if (IS_ERR(fsl_edma->muxclk[i])) + return dev_err_probe(&pdev->dev, + PTR_ERR(fsl_edma->muxclk[i]), + "Missing DMAMUX block clock.\n"); } fsl_edma->big_endian = of_property_read_bool(np, "big-endian"); @@ -882,22 +877,17 @@ static int fsl_edma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, fsl_edma); - ret = dma_async_device_register(&fsl_edma->dma_dev); - if (ret) { - dev_err(&pdev->dev, - "Can't register Freescale eDMA engine. (%d)\n", ret); - return ret; - } + ret = dmaenginem_async_device_register(&fsl_edma->dma_dev); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register Freescale eDMA engine.\n"); - ret = of_dma_controller_register(np, + ret = devm_of_dma_controller_register(&pdev->dev, np, drvdata->dmamuxs ? fsl_edma_xlate : fsl_edma3_xlate, fsl_edma); - if (ret) { - dev_err(&pdev->dev, - "Can't register Freescale eDMA of_dma. (%d)\n", ret); - dma_async_device_unregister(&fsl_edma->dma_dev); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register Freescale eDMA of_dma.\n"); /* enable round robin arbitration */ if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) @@ -908,12 +898,9 @@ static int fsl_edma_probe(struct platform_device *pdev) static void fsl_edma_remove(struct platform_device *pdev) { - struct device_node *np = pdev->dev.of_node; struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev); fsl_edma_irq_exit(pdev, fsl_edma); - of_dma_controller_free(np); - dma_async_device_unregister(&fsl_edma->dma_dev); fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); } diff --git a/drivers/dma/fsl-edma-trace.h b/drivers/dma/fsl-edma-trace.h index d3541301a247..45d964a3726d 100644 --- a/drivers/dma/fsl-edma-trace.h +++ b/drivers/dma/fsl-edma-trace.h @@ -19,14 +19,16 @@ DECLARE_EVENT_CLASS(edma_log_io, __field(struct fsl_edma_engine *, edma) __field(void __iomem *, addr) __field(u32, value) + __field(void __iomem *, membase) ), TP_fast_assign( __entry->edma = edma; __entry->addr = addr; __entry->value = value; + __entry->membase = edma->membase; ), TP_printk("offset %08x: value %08x", - (u32)(__entry->addr - __entry->edma->membase), __entry->value) + (u32)(__entry->addr - __entry->membase), __entry->value) ); DEFINE_EVENT(edma_log_io, edma_readl, diff --git a/drivers/dma/fsl-qdma.c b/drivers/dma/fsl-qdma.c index 0bbff9df362f..df843fad0ece 100644 --- a/drivers/dma/fsl-qdma.c +++ b/drivers/dma/fsl-qdma.c @@ -1127,22 +1127,19 @@ static int fsl_qdma_probe(struct platform_device *pdev) struct device_node *np = pdev->dev.of_node; ret = of_property_read_u32(np, "dma-channels", &chans); - if (ret) { - dev_err(&pdev->dev, "Can't get dma-channels.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't get dma-channels.\n"); ret = of_property_read_u32(np, "block-offset", &blk_off); - if (ret) { - dev_err(&pdev->dev, "Can't get block-offset.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't get block-offset.\n"); ret = of_property_read_u32(np, "block-number", &blk_num); - if (ret) { - dev_err(&pdev->dev, "Can't get block-number.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't get block-number.\n"); blk_num = min_t(int, blk_num, num_online_cpus()); @@ -1167,10 +1164,8 @@ static int fsl_qdma_probe(struct platform_device *pdev) return -ENOMEM; ret = of_property_read_u32(np, "fsl,dma-queues", &queues); - if (ret) { - dev_err(&pdev->dev, "Can't get queues.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "Can't get queues.\n"); fsl_qdma->desc_allocated = 0; fsl_qdma->n_chans = chans; @@ -1231,28 +1226,24 @@ static int fsl_qdma_probe(struct platform_device *pdev) fsl_qdma->dma_dev.device_terminate_all = fsl_qdma_terminate_all; ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(40)); - if (ret) { - dev_err(&pdev->dev, "dma_set_mask failure.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "dma_set_mask failure.\n"); platform_set_drvdata(pdev, fsl_qdma); ret = fsl_qdma_reg_init(fsl_qdma); - if (ret) { - dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't Initialize the qDMA engine.\n"); ret = fsl_qdma_irq_init(pdev, fsl_qdma); if (ret) return ret; ret = dma_async_device_register(&fsl_qdma->dma_dev); - if (ret) { - dev_err(&pdev->dev, "Can't register NXP Layerscape qDMA engine.\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register NXP Layerscape qDMA engine.\n"); return 0; } diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index 99945845d8b5..2d86f61105e5 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -242,9 +242,9 @@ static void fill_cfd_frame(struct fsl_re_cmpnd_frame *cf, u8 index, u32 efrl = length & FSL_RE_CF_LENGTH_MASK; efrl |= final << FSL_RE_CF_FINAL_SHIFT; - cf[index].efrl32 = efrl; - cf[index].addr_high = upper_32_bits(addr); - cf[index].addr_low = lower_32_bits(addr); + cf[index].efrl32 = cpu_to_be32(efrl); + cf[index].addr_high = cpu_to_be32(upper_32_bits(addr)); + cf[index].addr_low = cpu_to_be32(lower_32_bits(addr)); } static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan, @@ -256,9 +256,10 @@ static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan, dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan); INIT_LIST_HEAD(&desc->node); - desc->hwdesc.fmt32 = FSL_RE_FRAME_FORMAT << FSL_RE_HWDESC_FMT_SHIFT; - desc->hwdesc.lbea32 = upper_32_bits(paddr); - desc->hwdesc.addr_low = lower_32_bits(paddr); + desc->hwdesc.fmt32 = cpu_to_be32(FSL_RE_FRAME_FORMAT << + FSL_RE_HWDESC_FMT_SHIFT); + desc->hwdesc.lbea32 = cpu_to_be32(upper_32_bits(paddr)); + desc->hwdesc.addr_low = cpu_to_be32(lower_32_bits(paddr)); desc->cf_addr = cf; desc->cf_paddr = paddr; @@ -374,11 +375,11 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq( for (i = 2, j = 0; j < save_src_cnt; i++, j++) fill_cfd_frame(cf, i, len, src[j], 0); + /* Fill the last frame and mark it final */ if (cont_q) - fill_cfd_frame(cf, i++, len, dest, 0); - - /* Setting the final bit in the last source buffer frame in CFD */ - cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT; + fill_cfd_frame(cf, i, len, dest, 1); + else + fill_cfd_frame(cf, i - 1, len, src[j - 1], 1); return &desc->async_tx; } @@ -504,16 +505,16 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq( p[save_src_cnt + 2] = 1; fill_cfd_frame(cf, i++, len, dest[0], 0); fill_cfd_frame(cf, i++, len, dest[1], 0); - fill_cfd_frame(cf, i++, len, dest[1], 0); + fill_cfd_frame(cf, i++, len, dest[1], 1); } else { dev_err(re_chan->dev, "PQ tx continuation error!\n"); return NULL; } + } else { + /* Mark the last source buffer frame final */ + fill_cfd_frame(cf, i - 1, len, src[j - 1], 1); } - /* Setting the final bit in the last source buffer frame in CFD */ - cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT; - return &desc->async_tx; } @@ -656,8 +657,7 @@ static int fsl_re_chan_probe(struct platform_device *ofdev, goto err_free; } - chan->jrregs = (struct fsl_re_chan_cfg *)((u8 *)re_priv->re_regs + - off + ptr); + chan->jrregs = re_priv->base + off + ptr; /* read irq property from dts */ chan->irq = irq_of_parse_and_map(np, 0); @@ -745,38 +745,36 @@ err_free: /* Probe function for RAID Engine */ static int fsl_re_probe(struct platform_device *ofdev) { + struct fsl_re_ctrl __iomem *re_regs; struct fsl_re_drv_private *re_priv; struct device_node *child; u32 off; u8 ridx = 0; struct dma_device *dma_dev; - struct resource *res; int rc; struct device *dev = &ofdev->dev; + /* IOMAP the entire RAID Engine region */ + re_regs = devm_platform_ioremap_resource(ofdev, 0); + if (IS_ERR(re_regs)) + return PTR_ERR(re_regs); + re_priv = devm_kzalloc(dev, sizeof(*re_priv), GFP_KERNEL); if (!re_priv) return -ENOMEM; - res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - - /* IOMAP the entire RAID Engine region */ - re_priv->re_regs = devm_ioremap(dev, res->start, resource_size(res)); - if (!re_priv->re_regs) - return -EBUSY; + re_priv->base = re_regs; /* Program the RE mode */ - out_be32(&re_priv->re_regs->global_config, FSL_RE_NON_DPAA_MODE); + out_be32(&re_regs->global_config, FSL_RE_NON_DPAA_MODE); /* Program Galois Field polynomial */ - out_be32(&re_priv->re_regs->galois_field_config, FSL_RE_GFM_POLY); + out_be32(&re_regs->galois_field_config, FSL_RE_GFM_POLY); dev_info(dev, "version %x, mode %x, gfp %x\n", - in_be32(&re_priv->re_regs->re_version_id), - in_be32(&re_priv->re_regs->global_config), - in_be32(&re_priv->re_regs->galois_field_config)); + in_be32(&re_regs->re_version_id), + in_be32(&re_regs->global_config), + in_be32(&re_regs->galois_field_config)); dma_dev = &re_priv->dma_dev; dma_dev->dev = dev; diff --git a/drivers/dma/fsl_raid.h b/drivers/dma/fsl_raid.h index 69d743c04973..adbfede330a7 100644 --- a/drivers/dma/fsl_raid.h +++ b/drivers/dma/fsl_raid.h @@ -256,7 +256,7 @@ struct fsl_re_hw_desc { struct fsl_re_drv_private { u8 total_chans; struct dma_device dma_dev; - struct fsl_re_ctrl *re_regs; + void __iomem *base; struct fsl_re_chan *re_jrs[FSL_RE_MAX_CHANS]; struct dma_pool *cf_desc_pool; struct dma_pool *hw_desc_pool; @@ -273,7 +273,7 @@ struct fsl_re_chan { struct device *dev; struct fsl_re_drv_private *re_dev; struct dma_chan chan; - struct fsl_re_chan_cfg *jrregs; + struct fsl_re_chan_cfg __iomem *jrregs; int irq; struct tasklet_struct irqtask; u32 alloc_count; diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c index 32a0e95c6a20..c751a2e49e6d 100644 --- a/drivers/dma/hisi_dma.c +++ b/drivers/dma/hisi_dma.c @@ -983,7 +983,7 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num), GFP_KERNEL); if (!hdma_dev) - return -EINVAL; + return -ENOMEM; hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2]; hdma_dev->pdev = pdev; @@ -1037,6 +1037,7 @@ static const struct pci_device_id hisi_dma_pci_tbl[] = { { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) }, { 0, } }; +MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl); static struct pci_driver hisi_dma_pci_driver = { .name = "hisi_dma", @@ -1050,4 +1051,3 @@ MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>"); MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver"); MODULE_LICENSE("GPL v2"); -MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl); diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index c37d233535f9..82b07cf942ef 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -158,11 +158,7 @@ static const struct device_type idxd_cdev_file_type = { static void idxd_cdev_dev_release(struct device *dev) { struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); - struct idxd_cdev_context *cdev_ctx; - struct idxd_wq *wq = idxd_cdev->wq; - cdev_ctx = &ictx[wq->idxd->data->type]; - ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); kfree(idxd_cdev); } @@ -292,6 +288,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) fdev->parent = cdev_dev(idxd_cdev); fdev->bus = &dsa_bus_type; fdev->type = &idxd_cdev_file_type; + idxd_wq_get(wq); rc = dev_set_name(fdev, "file%d", ctx->id); if (rc < 0) { @@ -305,13 +302,14 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) goto failed_dev_add; } - idxd_wq_get(wq); mutex_unlock(&wq->wq_lock); return 0; failed_dev_add: failed_dev_name: + mutex_unlock(&wq->wq_lock); put_device(fdev); + return rc; failed_ida: failed_set_pasid: if (device_user_pasid_enabled(idxd)) @@ -582,11 +580,15 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) void idxd_wq_del_cdev(struct idxd_wq *wq) { + struct idxd_cdev_context *cdev_ctx; struct idxd_cdev *idxd_cdev; idxd_cdev = wq->idxd_cdev; wq->idxd_cdev = NULL; cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); + + cdev_ctx = &ictx[wq->idxd->data->type]; + ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); put_device(cdev_dev(idxd_cdev)); } diff --git a/drivers/dma/idxd/defaults.c b/drivers/dma/idxd/defaults.c index 2bbbcd02a0da..26ebfa2ca144 100644 --- a/drivers/dma/idxd/defaults.c +++ b/drivers/dma/idxd/defaults.c @@ -8,6 +8,7 @@ int idxd_load_iaa_device_defaults(struct idxd_device *idxd) struct idxd_engine *engine; struct idxd_group *group; struct idxd_wq *wq; + int i; if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) return 0; @@ -41,11 +42,12 @@ int idxd_load_iaa_device_defaults(struct idxd_device *idxd) /* set driver_name to "crypto" */ strscpy_pad(wq->driver_name, "crypto"); - engine = idxd->engines[0]; - - /* set engine group to 0 */ - engine->group = idxd->groups[0]; - engine->group->num_engines++; + /* assign all engines to group 0 */ + for (i = 0; i < idxd->max_engines; i++) { + engine = idxd->engines[i]; + engine->group = group; + group->num_engines++; + } return 0; } diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index c26128529ff4..131138483b87 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -175,6 +175,7 @@ void idxd_wq_free_resources(struct idxd_wq *wq) free_descs(wq); dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); sbitmap_queue_free(&wq->sbq); + wq->type = IDXD_WQT_NONE; } EXPORT_SYMBOL_NS_GPL(idxd_wq_free_resources, "IDXD"); @@ -382,7 +383,6 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq) lockdep_assert_held(&wq->wq_lock); wq->state = IDXD_WQ_DISABLED; memset(wq->wqcfg, 0, idxd->wqcfg_size); - wq->type = IDXD_WQT_NONE; wq->threshold = 0; wq->priority = 0; wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; @@ -831,8 +831,7 @@ static void idxd_device_evl_free(struct idxd_device *idxd) struct device *dev = &idxd->pdev->dev; struct idxd_evl *evl = idxd->evl; - gencfg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); - if (!gencfg.evl_en) + if (!evl) return; mutex_lock(&evl->lock); @@ -1125,7 +1124,11 @@ int idxd_device_config(struct idxd_device *idxd) { int rc; - lockdep_assert_held(&idxd->dev_lock); + guard(spinlock)(&idxd->dev_lock); + + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return 0; + rc = idxd_wqs_setup(idxd); if (rc < 0) return rc; @@ -1332,6 +1335,11 @@ void idxd_wq_free_irq(struct idxd_wq *wq) free_irq(ie->vector, ie); idxd_flush_pending_descs(ie); + + /* The interrupt might have been already released by FLR */ + if (ie->int_handle == INVALID_INT_HANDLE) + return; + if (idxd->request_int_handles) idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX); idxd_device_clear_perm_entry(idxd, ie); @@ -1340,6 +1348,23 @@ void idxd_wq_free_irq(struct idxd_wq *wq) ie->pasid = IOMMU_PASID_INVALID; } +void idxd_wq_flush_descs(struct idxd_wq *wq) +{ + struct idxd_irq_entry *ie = &wq->ie; + struct idxd_device *idxd = wq->idxd; + + guard(mutex)(&wq->wq_lock); + + if (wq->state != IDXD_WQ_ENABLED || wq->type != IDXD_WQT_KERNEL) + return; + + idxd_flush_pending_descs(ie); + if (idxd->request_int_handles) + idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX); + idxd_device_clear_perm_entry(idxd, ie); + ie->int_handle = INVALID_INT_HANDLE; +} + int idxd_wq_request_irq(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; @@ -1454,11 +1479,7 @@ int idxd_drv_enable_wq(struct idxd_wq *wq) } } - rc = 0; - spin_lock(&idxd->dev_lock); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - rc = idxd_device_config(idxd); - spin_unlock(&idxd->dev_lock); + rc = idxd_device_config(idxd); if (rc < 0) { dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc); goto err; @@ -1533,7 +1554,6 @@ void idxd_drv_disable_wq(struct idxd_wq *wq) idxd_wq_reset(wq); idxd_wq_free_resources(wq); percpu_ref_exit(&wq->wq_active); - wq->type = IDXD_WQT_NONE; wq->client_count = 0; } EXPORT_SYMBOL_NS_GPL(idxd_drv_disable_wq, "IDXD"); @@ -1554,10 +1574,7 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev) } /* Device configuration */ - spin_lock(&idxd->dev_lock); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - rc = idxd_device_config(idxd); - spin_unlock(&idxd->dev_lock); + rc = idxd_device_config(idxd); if (rc < 0) return -ENXIO; diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index dbecd699237e..9937b671f637 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -194,6 +194,22 @@ static void idxd_dma_release(struct dma_device *device) kfree(idxd_dma); } +static int idxd_dma_terminate_all(struct dma_chan *c) +{ + struct idxd_wq *wq = to_idxd_wq(c); + + idxd_wq_flush_descs(wq); + + return 0; +} + +static void idxd_dma_synchronize(struct dma_chan *c) +{ + struct idxd_wq *wq = to_idxd_wq(c); + + idxd_wq_drain(wq); +} + int idxd_register_dma_device(struct idxd_device *idxd) { struct idxd_dma_dev *idxd_dma; @@ -224,6 +240,8 @@ int idxd_register_dma_device(struct idxd_device *idxd) dma->device_issue_pending = idxd_dma_issue_pending; dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources; dma->device_free_chan_resources = idxd_dma_free_chan_resources; + dma->device_terminate_all = idxd_dma_terminate_all; + dma->device_synchronize = idxd_dma_synchronize; rc = dma_async_device_register(dma); if (rc < 0) { diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index ea8c4daed38d..ce78b9a7c641 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -803,6 +803,7 @@ void idxd_wq_quiesce(struct idxd_wq *wq); int idxd_wq_init_percpu_ref(struct idxd_wq *wq); void idxd_wq_free_irq(struct idxd_wq *wq); int idxd_wq_request_irq(struct idxd_wq *wq); +void idxd_wq_flush_descs(struct idxd_wq *wq); /* submission */ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index fb80803d5b57..4b827a329756 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -159,18 +159,12 @@ static void idxd_cleanup_interrupts(struct idxd_device *idxd) static void idxd_clean_wqs(struct idxd_device *idxd) { - struct idxd_wq *wq; struct device *conf_dev; int i; for (i = 0; i < idxd->max_wqs; i++) { - wq = idxd->wqs[i]; - if (idxd->hw.wq_cap.op_config) - bitmap_free(wq->opcap_bmap); - kfree(wq->wqcfg); - conf_dev = wq_confdev(wq); + conf_dev = wq_confdev(idxd->wqs[i]); put_device(conf_dev); - kfree(wq); } bitmap_free(idxd->wq_enable_map); kfree(idxd->wqs); @@ -212,7 +206,6 @@ static int idxd_setup_wqs(struct idxd_device *idxd) rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); if (rc < 0) { put_device(conf_dev); - kfree(wq); goto err_unwind; } @@ -227,7 +220,6 @@ static int idxd_setup_wqs(struct idxd_device *idxd) wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); if (!wq->wqcfg) { put_device(conf_dev); - kfree(wq); rc = -ENOMEM; goto err_unwind; } @@ -235,9 +227,7 @@ static int idxd_setup_wqs(struct idxd_device *idxd) if (idxd->hw.wq_cap.op_config) { wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); if (!wq->opcap_bmap) { - kfree(wq->wqcfg); put_device(conf_dev); - kfree(wq); rc = -ENOMEM; goto err_unwind; } @@ -252,13 +242,8 @@ static int idxd_setup_wqs(struct idxd_device *idxd) err_unwind: while (--i >= 0) { - wq = idxd->wqs[i]; - if (idxd->hw.wq_cap.op_config) - bitmap_free(wq->opcap_bmap); - kfree(wq->wqcfg); - conf_dev = wq_confdev(wq); + conf_dev = wq_confdev(idxd->wqs[i]); put_device(conf_dev); - kfree(wq); } bitmap_free(idxd->wq_enable_map); @@ -270,15 +255,12 @@ err_free_wqs: static void idxd_clean_engines(struct idxd_device *idxd) { - struct idxd_engine *engine; struct device *conf_dev; int i; for (i = 0; i < idxd->max_engines; i++) { - engine = idxd->engines[i]; - conf_dev = engine_confdev(engine); + conf_dev = engine_confdev(idxd->engines[i]); put_device(conf_dev); - kfree(engine); } kfree(idxd->engines); } @@ -313,7 +295,6 @@ static int idxd_setup_engines(struct idxd_device *idxd) rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); if (rc < 0) { put_device(conf_dev); - kfree(engine); goto err; } @@ -324,10 +305,8 @@ static int idxd_setup_engines(struct idxd_device *idxd) err: while (--i >= 0) { - engine = idxd->engines[i]; - conf_dev = engine_confdev(engine); + conf_dev = engine_confdev(idxd->engines[i]); put_device(conf_dev); - kfree(engine); } kfree(idxd->engines); @@ -336,13 +315,10 @@ static int idxd_setup_engines(struct idxd_device *idxd) static void idxd_clean_groups(struct idxd_device *idxd) { - struct idxd_group *group; int i; for (i = 0; i < idxd->max_groups; i++) { - group = idxd->groups[i]; - put_device(group_confdev(group)); - kfree(group); + put_device(group_confdev(idxd->groups[i])); } kfree(idxd->groups); } @@ -377,7 +353,6 @@ static int idxd_setup_groups(struct idxd_device *idxd) rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); if (rc < 0) { put_device(conf_dev); - kfree(group); goto err; } @@ -402,7 +377,6 @@ static int idxd_setup_groups(struct idxd_device *idxd) while (--i >= 0) { group = idxd->groups[i]; put_device(group_confdev(group)); - kfree(group); } kfree(idxd->groups); @@ -973,7 +947,8 @@ static void idxd_device_config_restore(struct idxd_device *idxd, idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit; - idxd->evl->size = saved_evl->size; + if (idxd->evl) + idxd->evl->size = saved_evl->size; for (i = 0; i < idxd->max_groups; i++) { struct idxd_group *saved_group, *group; @@ -1104,12 +1079,10 @@ static void idxd_reset_done(struct pci_dev *pdev) idxd_device_config_restore(idxd, idxd->idxd_saved); /* Re-configure IDXD device if allowed. */ - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { - rc = idxd_device_config(idxd); - if (rc < 0) { - dev_err(dev, "HALT: %s config fails\n", idxd_name); - goto out; - } + rc = idxd_device_config(idxd); + if (rc < 0) { + dev_err(dev, "HALT: %s config fails\n", idxd_name); + goto out; } /* Bind IDXD device to driver. */ @@ -1147,6 +1120,7 @@ static void idxd_reset_done(struct pci_dev *pdev) } out: kfree(idxd->idxd_saved); + idxd->idxd_saved = NULL; } static const struct pci_error_handlers idxd_error_handler = { diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 7782f8c51c32..6a25e1fd0e62 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -397,6 +397,17 @@ static void idxd_device_flr(struct work_struct *work) dev_err(&idxd->pdev->dev, "FLR failed\n"); } +static void idxd_wqs_flush_descs(struct idxd_device *idxd) +{ + int i; + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *wq = idxd->wqs[i]; + + idxd_wq_flush_descs(wq); + } +} + static irqreturn_t idxd_halt(struct idxd_device *idxd) { union gensts_reg gensts; @@ -415,6 +426,11 @@ static irqreturn_t idxd_halt(struct idxd_device *idxd) } else if (gensts.reset_type == IDXD_DEVICE_RESET_FLR) { idxd->state = IDXD_DEV_HALTED; idxd_mask_error_interrupts(idxd); + /* Flush all pending descriptors, and disable + * interrupts, they will be re-enabled when FLR + * concludes. + */ + idxd_wqs_flush_descs(idxd); dev_dbg(&idxd->pdev->dev, "idxd halted, doing FLR. After FLR, configs are restored\n"); INIT_WORK(&idxd->work, idxd_device_flr); diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index f95411363ea9..1dce26d4da83 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -10,9 +10,6 @@ #endif /* PCI Config */ -#define PCI_DEVICE_ID_INTEL_DSA_GNRD 0x11fb -#define PCI_DEVICE_ID_INTEL_DSA_DMR 0x1212 -#define PCI_DEVICE_ID_INTEL_IAA_DMR 0x1216 #define PCI_DEVICE_ID_INTEL_IAA_PTL 0xb02d #define PCI_DEVICE_ID_INTEL_IAA_WCL 0xfd2d diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 6db1c5fcedc5..03217041b8b3 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -138,7 +138,7 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, */ list_for_each_entry_safe(d, t, &flist, list) { list_del_init(&d->list); - idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true, + idxd_dma_complete_txd(d, IDXD_COMPLETE_ABORT, true, NULL, NULL); } } diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index cc2c83d7f710..6d251095c350 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1836,6 +1836,7 @@ static void idxd_conf_device_release(struct device *dev) { struct idxd_device *idxd = confdev_to_idxd(dev); + destroy_workqueue(idxd->wq); kfree(idxd->groups); bitmap_free(idxd->wq_enable_map); kfree(idxd->wqs); diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 4c8196d78001..36368835a845 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -2265,34 +2265,24 @@ static int sdma_probe(struct platform_device *pdev) if (IS_ERR(sdma->regs)) return PTR_ERR(sdma->regs); - sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); + sdma->clk_ipg = devm_clk_get_prepared(&pdev->dev, "ipg"); if (IS_ERR(sdma->clk_ipg)) return PTR_ERR(sdma->clk_ipg); - sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb"); + sdma->clk_ahb = devm_clk_get_prepared(&pdev->dev, "ahb"); if (IS_ERR(sdma->clk_ahb)) return PTR_ERR(sdma->clk_ahb); - ret = clk_prepare(sdma->clk_ipg); - if (ret) - return ret; - - ret = clk_prepare(sdma->clk_ahb); - if (ret) - goto err_clk; - ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, dev_name(&pdev->dev), sdma); if (ret) - goto err_irq; + return ret; sdma->irq = irq; sdma->script_addrs = kzalloc_obj(*sdma->script_addrs); - if (!sdma->script_addrs) { - ret = -ENOMEM; - goto err_irq; - } + if (!sdma->script_addrs) + return -ENOMEM; /* initially no scripts available */ saddr_arr = (s32 *)sdma->script_addrs; @@ -2333,11 +2323,11 @@ static int sdma_probe(struct platform_device *pdev) ret = sdma_init(sdma); if (ret) - goto err_init; + return ret; ret = sdma_event_remap(sdma); if (ret) - goto err_init; + return ret; if (sdma->drvdata->script_addrs) sdma_add_scripts(sdma, sdma->drvdata->script_addrs); @@ -2363,20 +2353,20 @@ static int sdma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, sdma); - ret = dma_async_device_register(&sdma->dma_device); - if (ret) { - dev_err(&pdev->dev, "unable to register\n"); - goto err_init; - } + ret = dmaenginem_async_device_register(&sdma->dma_device); + if (ret) + return dev_err_probe(&pdev->dev, ret, "unable to register\n"); if (np) { - ret = of_dma_controller_register(np, sdma_xlate, sdma); - if (ret) { - dev_err(&pdev->dev, "failed to register controller\n"); - goto err_register; - } + ret = devm_of_dma_controller_register(&pdev->dev, np, + sdma_xlate, sdma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to register controller\n"); + + struct device_node *sdma_parent_np __free(device_node) = of_get_parent(np); - spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus"); + spba_bus = of_get_compatible_child(sdma_parent_np, "fsl,spba-bus"); ret = of_address_to_resource(spba_bus, 0, &spba_res); if (!ret) { sdma->spba_start_addr = spba_res.start; @@ -2401,16 +2391,6 @@ static int sdma_probe(struct platform_device *pdev) } return 0; - -err_register: - dma_async_device_unregister(&sdma->dma_device); -err_init: - kfree(sdma->script_addrs); -err_irq: - clk_unprepare(sdma->clk_ahb); -err_clk: - clk_unprepare(sdma->clk_ipg); - return ret; } static void sdma_remove(struct platform_device *pdev) @@ -2419,10 +2399,6 @@ static void sdma_remove(struct platform_device *pdev) int i; devm_free_irq(&pdev->dev, sdma->irq, sdma); - dma_async_device_unregister(&sdma->dma_device); - kfree(sdma->script_addrs); - clk_unprepare(sdma->clk_ahb); - clk_unprepare(sdma->clk_ipg); /* Kill the tasklet */ for (i = 0; i < MAX_DMA_CHANNELS; i++) { struct sdma_channel *sdmac = &sdma->channel[i]; diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 12a4a4860a74..e8a880f338c6 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -140,12 +140,6 @@ struct ioatdma_chan { int prev_intr_coalesce; }; -struct ioat_sysfs_entry { - struct attribute attr; - ssize_t (*show)(struct dma_chan *, char *); - ssize_t (*store)(struct dma_chan *, const char *, size_t); -}; - /** * struct ioat_sed_ent - wrapper around super extended hardware descriptor * @hw: hardware SED @@ -195,11 +189,8 @@ struct ioat_ring_ent { struct ioat_sed_ent *sed; }; -extern const struct sysfs_ops ioat_sysfs_ops; -extern struct ioat_sysfs_entry ioat_version_attr; -extern struct ioat_sysfs_entry ioat_cap_attr; extern int ioat_pending_level; -extern struct kobj_type ioat_ktype; +extern const struct kobj_type ioat_ktype; extern struct kmem_cache *ioat_cache; extern struct kmem_cache *ioat_sed_cache; @@ -402,7 +393,7 @@ void ioat_issue_pending(struct dma_chan *chan); /* IOAT Init functions */ bool is_bwd_ioat(struct pci_dev *pdev); struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); -void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type); +void ioat_kobject_add(struct ioatdma_device *ioat_dma, const struct kobj_type *type); void ioat_kobject_del(struct ioatdma_device *ioat_dma); int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma); void ioat_stop(struct ioatdma_chan *ioat_chan); diff --git a/drivers/dma/ioat/sysfs.c b/drivers/dma/ioat/sysfs.c index 168adf28c5b1..e796ddb5383f 100644 --- a/drivers/dma/ioat/sysfs.c +++ b/drivers/dma/ioat/sysfs.c @@ -14,6 +14,12 @@ #include "../dmaengine.h" +struct ioat_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct dma_chan *, char *); + ssize_t (*store)(struct dma_chan *, const char *, size_t); +}; + static ssize_t cap_show(struct dma_chan *c, char *page) { struct dma_device *dma = c->device; @@ -26,7 +32,7 @@ static ssize_t cap_show(struct dma_chan *c, char *page) dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : ""); } -struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap); +static const struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap); static ssize_t version_show(struct dma_chan *c, char *page) { @@ -36,15 +42,15 @@ static ssize_t version_show(struct dma_chan *c, char *page) return sprintf(page, "%d.%d\n", ioat_dma->version >> 4, ioat_dma->version & 0xf); } -struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version); +static const struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version); static ssize_t ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page) { - struct ioat_sysfs_entry *entry; + const struct ioat_sysfs_entry *entry; struct ioatdma_chan *ioat_chan; - entry = container_of(attr, struct ioat_sysfs_entry, attr); + entry = container_of_const(attr, struct ioat_sysfs_entry, attr); ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); if (!entry->show) @@ -56,10 +62,10 @@ static ssize_t ioat_attr_store(struct kobject *kobj, struct attribute *attr, const char *page, size_t count) { - struct ioat_sysfs_entry *entry; + const struct ioat_sysfs_entry *entry; struct ioatdma_chan *ioat_chan; - entry = container_of(attr, struct ioat_sysfs_entry, attr); + entry = container_of_const(attr, struct ioat_sysfs_entry, attr); ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); if (!entry->store) @@ -67,12 +73,12 @@ const char *page, size_t count) return entry->store(&ioat_chan->dma_chan, page, count); } -const struct sysfs_ops ioat_sysfs_ops = { +static const struct sysfs_ops ioat_sysfs_ops = { .show = ioat_attr_show, .store = ioat_attr_store, }; -void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type) +void ioat_kobject_add(struct ioatdma_device *ioat_dma, const struct kobj_type *type) { struct dma_device *dma = &ioat_dma->dma_dev; struct dma_chan *c; @@ -114,7 +120,7 @@ static ssize_t ring_size_show(struct dma_chan *c, char *page) return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1); } -static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); +static const struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); static ssize_t ring_active_show(struct dma_chan *c, char *page) { @@ -123,7 +129,7 @@ static ssize_t ring_active_show(struct dma_chan *c, char *page) /* ...taken outside the lock, no need to be precise */ return sprintf(page, "%d\n", ioat_ring_active(ioat_chan)); } -static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); +static const struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); static ssize_t intr_coalesce_show(struct dma_chan *c, char *page) { @@ -148,9 +154,9 @@ size_t count) return count; } -static struct ioat_sysfs_entry intr_coalesce_attr = __ATTR_RW(intr_coalesce); +static const struct ioat_sysfs_entry intr_coalesce_attr = __ATTR_RW(intr_coalesce); -static struct attribute *ioat_attrs[] = { +static const struct attribute *const ioat_attrs[] = { &ring_size_attr.attr, &ring_active_attr.attr, &ioat_cap_attr.attr, @@ -160,7 +166,7 @@ static struct attribute *ioat_attrs[] = { }; ATTRIBUTE_GROUPS(ioat); -struct kobj_type ioat_ktype = { +const struct kobj_type ioat_ktype = { .sysfs_ops = &ioat_sysfs_ops, .default_groups = ioat_groups, }; diff --git a/drivers/dma/loongson/Kconfig b/drivers/dma/loongson/Kconfig new file mode 100644 index 000000000000..c4e62dce5d4f --- /dev/null +++ b/drivers/dma/loongson/Kconfig @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Loongson DMA controllers drivers +# +if MACH_LOONGSON32 || MACH_LOONGSON64 || COMPILE_TEST + +config LOONGSON1_APB_DMA + tristate "Loongson1 APB DMA support" + depends on MACH_LOONGSON32 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + This selects support for the APB DMA controller in Loongson1 SoCs, + which is required by Loongson1 NAND and audio support. + +config LOONGSON2_APB_DMA + tristate "Loongson2 APB DMA support" + depends on MACH_LOONGSON64 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the Loongson2 APB DMA controller driver. The + DMA controller is having single DMA channel which can be + configured for different peripherals like audio, nand, sdio + etc which is in APB bus. + + This DMA controller transfers data from memory to peripheral fifo. + It does not support memory to memory data transfer. + +config LOONGSON2_APB_CMC_DMA + tristate "Loongson2 Chain Multi-Channel DMA support" + depends on MACH_LOONGSON64 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the Loongson Chain Multi-Channel DMA controller driver. + It is discovered on the Loongson-2K chip (Loongson-2K0300/Loongson-2K3000), + which has 4/8 channels internally, enabling bidirectional data transfer + between devices and memory. + +endif diff --git a/drivers/dma/loongson/Makefile b/drivers/dma/loongson/Makefile new file mode 100644 index 000000000000..48c19781e729 --- /dev/null +++ b/drivers/dma/loongson/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_LOONGSON1_APB_DMA) += loongson1-apb-dma.o +obj-$(CONFIG_LOONGSON2_APB_DMA) += loongson2-apb-dma.o +obj-$(CONFIG_LOONGSON2_APB_CMC_DMA) += loongson2-apb-cmc-dma.o diff --git a/drivers/dma/loongson1-apb-dma.c b/drivers/dma/loongson/loongson1-apb-dma.c index 2e347aba9af8..89786cbd20ab 100644 --- a/drivers/dma/loongson1-apb-dma.c +++ b/drivers/dma/loongson/loongson1-apb-dma.c @@ -16,8 +16,8 @@ #include <linux/platform_device.h> #include <linux/slab.h> -#include "dmaengine.h" -#include "virt-dma.h" +#include "../dmaengine.h" +#include "../virt-dma.h" /* Loongson-1 DMA Control Register */ #define LS1X_DMA_CTRL 0x0 diff --git a/drivers/dma/loongson/loongson2-apb-cmc-dma.c b/drivers/dma/loongson/loongson2-apb-cmc-dma.c new file mode 100644 index 000000000000..1c9a542edc85 --- /dev/null +++ b/drivers/dma/loongson/loongson2-apb-cmc-dma.c @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Loongson-2 Chain Multi-Channel DMA Controller driver + * + * Copyright (C) 2024-2026 Loongson Technology Corporation Limited + */ + +#include <linux/acpi.h> +#include <linux/acpi_dma.h> +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "../dmaengine.h" +#include "../virt-dma.h" + +#define LOONGSON2_CMCDMA_ISR 0x0 /* DMA Interrupt Status Register */ +#define LOONGSON2_CMCDMA_IFCR 0x4 /* DMA Interrupt Flag Clear Register */ +#define LOONGSON2_CMCDMA_CCR 0x8 /* DMA Channel Configuration Register */ +#define LOONGSON2_CMCDMA_CNDTR 0xc /* DMA Channel Transmit Count Register */ +#define LOONGSON2_CMCDMA_CPAR 0x10 /* DMA Channel Peripheral Address Register */ +#define LOONGSON2_CMCDMA_CMAR 0x14 /* DMA Channel Memory Address Register */ + +/* Bitfields of DMA interrupt status register */ +#define LOONGSON2_CMCDMA_TCI BIT(1) /* Transfer Complete Interrupt */ +#define LOONGSON2_CMCDMA_HTI BIT(2) /* Half Transfer Interrupt */ +#define LOONGSON2_CMCDMA_TEI BIT(3) /* Transfer Error Interrupt */ + +#define LOONGSON2_CMCDMA_MASKI \ + (LOONGSON2_CMCDMA_TCI | LOONGSON2_CMCDMA_HTI | LOONGSON2_CMCDMA_TEI) + +/* Bitfields of DMA channel x Configuration Register */ +#define LOONGSON2_CMCDMA_CCR_EN BIT(0) /* Stream Enable */ +#define LOONGSON2_CMCDMA_CCR_TCIE BIT(1) /* Transfer Complete Interrupt Enable */ +#define LOONGSON2_CMCDMA_CCR_HTIE BIT(2) /* Half Transfer Complete Interrupt Enable */ +#define LOONGSON2_CMCDMA_CCR_TEIE BIT(3) /* Transfer Error Interrupt Enable */ +#define LOONGSON2_CMCDMA_CCR_DIR BIT(4) /* Data Transfer Direction */ +#define LOONGSON2_CMCDMA_CCR_CIRC BIT(5) /* Circular mode */ +#define LOONGSON2_CMCDMA_CCR_PINC BIT(6) /* Peripheral increment mode */ +#define LOONGSON2_CMCDMA_CCR_MINC BIT(7) /* Memory increment mode */ +#define LOONGSON2_CMCDMA_CCR_PSIZE_MASK GENMASK(9, 8) +#define LOONGSON2_CMCDMA_CCR_MSIZE_MASK GENMASK(11, 10) +#define LOONGSON2_CMCDMA_CCR_PL_MASK GENMASK(13, 12) +#define LOONGSON2_CMCDMA_CCR_M2M BIT(14) + +#define LOONGSON2_CMCDMA_CCR_CFG_MASK \ + (LOONGSON2_CMCDMA_CCR_PINC | LOONGSON2_CMCDMA_CCR_MINC | LOONGSON2_CMCDMA_CCR_PL_MASK) + +#define LOONGSON2_CMCDMA_CCR_IRQ_MASK \ + (LOONGSON2_CMCDMA_CCR_TCIE | LOONGSON2_CMCDMA_CCR_HTIE | LOONGSON2_CMCDMA_CCR_TEIE) + +#define LOONGSON2_CMCDMA_STREAM_MASK \ + (LOONGSON2_CMCDMA_CCR_CFG_MASK | LOONGSON2_CMCDMA_CCR_IRQ_MASK) + +#define LOONGSON2_CMCDMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) + +#define LOONSON2_CMCDMA_MAX_DATA_ITEMS SZ_64K + +struct loongson2_cmc_dma_chan_reg { + u32 ccr; + u32 cndtr; + u32 cpar; + u32 cmar; +}; + +struct loongson2_cmc_dma_sg_req { + u32 len; + struct loongson2_cmc_dma_chan_reg chan_reg; +}; + +struct loongson2_cmc_dma_desc { + struct virt_dma_desc vdesc; + bool cyclic; + u32 num_sgs; + struct loongson2_cmc_dma_sg_req sg_req[] __counted_by(num_sgs); +}; + +struct loongson2_cmc_dma_chan { + struct virt_dma_chan vchan; + struct dma_slave_config dma_sconfig; + struct loongson2_cmc_dma_desc *desc; + u32 id; + u32 irq; + u32 next_sg; + struct loongson2_cmc_dma_chan_reg chan_reg; +}; + +struct loongson2_cmc_dma_dev { + struct dma_device ddev; + struct clk *dma_clk; + void __iomem *base; + u32 nr_channels; + u32 chan_reg_offset; + struct loongson2_cmc_dma_chan chan[] __counted_by(nr_channels); +}; + +struct loongson2_cmc_dma_config { + u32 max_channels; + u32 chan_reg_offset; +}; + +static const struct loongson2_cmc_dma_config ls2k0300_cmc_dma_config = { + .max_channels = 8, + .chan_reg_offset = 0x14, +}; + +static const struct loongson2_cmc_dma_config ls2k3000_cmc_dma_config = { + .max_channels = 4, + .chan_reg_offset = 0x18, +}; + +static struct loongson2_cmc_dma_dev *lmdma_get_dev(struct loongson2_cmc_dma_chan *lchan) +{ + return container_of(lchan->vchan.chan.device, struct loongson2_cmc_dma_dev, ddev); +} + +static struct loongson2_cmc_dma_chan *to_lmdma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct loongson2_cmc_dma_chan, vchan.chan); +} + +static struct loongson2_cmc_dma_desc *to_lmdma_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct loongson2_cmc_dma_desc, vdesc); +} + +static struct device *chan2dev(struct loongson2_cmc_dma_chan *lchan) +{ + return &lchan->vchan.chan.dev->device; +} + +static u32 loongson2_cmc_dma_read(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id) +{ + return readl(lddev->base + (reg + lddev->chan_reg_offset * id)); +} + +static void loongson2_cmc_dma_write(struct loongson2_cmc_dma_dev *lddev, u32 reg, u32 id, u32 val) +{ + writel(val, lddev->base + (reg + lddev->chan_reg_offset * id)); +} + +static int loongson2_cmc_dma_get_width(enum dma_slave_buswidth width) +{ + switch (width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + case DMA_SLAVE_BUSWIDTH_2_BYTES: + case DMA_SLAVE_BUSWIDTH_4_BYTES: + return ffs(width) - 1; + default: + return -EINVAL; + } +} + +static int loongson2_cmc_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *config) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + + memcpy(&lchan->dma_sconfig, config, sizeof(*config)); + + return 0; +} + +static void loongson2_cmc_dma_irq_clear(struct loongson2_cmc_dma_chan *lchan, u32 flags) +{ + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + u32 ifcr; + + ifcr = flags << (4 * lchan->id); + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_IFCR, 0, ifcr); +} + +static void loongson2_cmc_dma_stop(struct loongson2_cmc_dma_chan *lchan) +{ + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + u32 ccr; + + ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id); + ccr &= ~(LOONGSON2_CMCDMA_CCR_IRQ_MASK | LOONGSON2_CMCDMA_CCR_EN); + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, ccr); + + loongson2_cmc_dma_irq_clear(lchan, LOONGSON2_CMCDMA_MASKI); +} + +static int loongson2_cmc_dma_terminate_all(struct dma_chan *chan) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + + LIST_HEAD(head); + + scoped_guard(spinlock_irqsave, &lchan->vchan.lock) { + if (lchan->desc) { + vchan_terminate_vdesc(&lchan->desc->vdesc); + loongson2_cmc_dma_stop(lchan); + lchan->desc = NULL; + } + vchan_get_all_descriptors(&lchan->vchan, &head); + } + + vchan_dma_desc_free_list(&lchan->vchan, &head); + + return 0; +} + +static void loongson2_cmc_dma_synchronize(struct dma_chan *chan) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + + vchan_synchronize(&lchan->vchan); +} + +static void loongson2_cmc_dma_start_transfer(struct loongson2_cmc_dma_chan *lchan) +{ + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + struct loongson2_cmc_dma_sg_req *sg_req; + struct loongson2_cmc_dma_chan_reg *reg; + struct virt_dma_desc *vdesc; + + loongson2_cmc_dma_stop(lchan); + + if (!lchan->desc) { + vdesc = vchan_next_desc(&lchan->vchan); + if (!vdesc) + return; + + list_del(&vdesc->node); + lchan->desc = to_lmdma_desc(vdesc); + lchan->next_sg = 0; + } + + if (lchan->next_sg == lchan->desc->num_sgs) + lchan->next_sg = 0; + + sg_req = &lchan->desc->sg_req[lchan->next_sg]; + reg = &sg_req->chan_reg; + + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, reg->ccr); + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CNDTR, lchan->id, reg->cndtr); + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CPAR, lchan->id, reg->cpar); + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CMAR, lchan->id, reg->cmar); + + lchan->next_sg++; + + /* Start DMA */ + reg->ccr |= LOONGSON2_CMCDMA_CCR_EN; + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, lchan->id, reg->ccr); +} + +static void loongson2_cmc_dma_configure_next_sg(struct loongson2_cmc_dma_chan *lchan) +{ + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + struct loongson2_cmc_dma_sg_req *sg_req; + u32 ccr, id = lchan->id; + + if (lchan->next_sg == lchan->desc->num_sgs) + lchan->next_sg = 0; + + /* Stop to update mem addr */ + ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, id); + ccr &= ~LOONGSON2_CMCDMA_CCR_EN; + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, id, ccr); + + sg_req = &lchan->desc->sg_req[lchan->next_sg]; + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CMAR, id, sg_req->chan_reg.cmar); + + /* Start transition */ + ccr |= LOONGSON2_CMCDMA_CCR_EN; + loongson2_cmc_dma_write(lddev, LOONGSON2_CMCDMA_CCR, id, ccr); +} + +static void loongson2_cmc_dma_handle_chan_done(struct loongson2_cmc_dma_chan *lchan) +{ + if (!lchan->desc) + return; + + if (lchan->desc->cyclic) { + vchan_cyclic_callback(&lchan->desc->vdesc); + /* LOONGSON2_CMCDMA_CCR_CIRC mode don't need update register */ + if (lchan->desc->num_sgs == 1) + return; + loongson2_cmc_dma_configure_next_sg(lchan); + lchan->next_sg++; + } else { + if (lchan->next_sg == lchan->desc->num_sgs) { + vchan_cookie_complete(&lchan->desc->vdesc); + lchan->desc = NULL; + } + loongson2_cmc_dma_start_transfer(lchan); + } +} + +static irqreturn_t loongson2_cmc_dma_chan_irq(int irq, void *devid) +{ + struct loongson2_cmc_dma_chan *lchan = devid; + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + struct device *dev = chan2dev(lchan); + u32 ists, status, ccr; + + scoped_guard(spinlock, &lchan->vchan.lock) { + ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id); + ists = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_ISR, 0); + status = (ists >> (4 * lchan->id)) & LOONGSON2_CMCDMA_MASKI; + + loongson2_cmc_dma_irq_clear(lchan, status); + + if (status & LOONGSON2_CMCDMA_TCI) { + loongson2_cmc_dma_handle_chan_done(lchan); + status &= ~LOONGSON2_CMCDMA_TCI; + } + + if (status & LOONGSON2_CMCDMA_HTI) + status &= ~LOONGSON2_CMCDMA_HTI; + + if (status & LOONGSON2_CMCDMA_TEI) { + dev_err(dev, "DMA Transform Error.\n"); + if (!(ccr & LOONGSON2_CMCDMA_CCR_EN)) + dev_err(dev, "Channel disabled by HW.\n"); + } + } + + return IRQ_HANDLED; +} + +static void loongson2_cmc_dma_issue_pending(struct dma_chan *chan) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + + guard(spinlock_irqsave)(&lchan->vchan.lock); + + if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) { + dev_dbg(chan2dev(lchan), "vchan %pK: issued\n", &lchan->vchan); + loongson2_cmc_dma_start_transfer(lchan); + } +} + +static int loongson2_cmc_dma_set_xfer_param(struct loongson2_cmc_dma_chan *lchan, + enum dma_transfer_direction direction, + enum dma_slave_buswidth *buswidth, u32 buf_len) +{ + struct dma_slave_config sconfig = lchan->dma_sconfig; + struct device *dev = chan2dev(lchan); + int dev_width; + u32 ccr; + + switch (direction) { + case DMA_MEM_TO_DEV: + dev_width = loongson2_cmc_dma_get_width(sconfig.dst_addr_width); + if (dev_width < 0) { + dev_err(dev, "DMA_MEM_TO_DEV bus width not supported\n"); + return dev_width; + } + lchan->chan_reg.cpar = sconfig.dst_addr; + ccr = LOONGSON2_CMCDMA_CCR_DIR; + *buswidth = sconfig.dst_addr_width; + break; + case DMA_DEV_TO_MEM: + dev_width = loongson2_cmc_dma_get_width(sconfig.src_addr_width); + if (dev_width < 0) { + dev_err(dev, "DMA_DEV_TO_MEM bus width not supported\n"); + return dev_width; + } + lchan->chan_reg.cpar = sconfig.src_addr; + ccr = LOONGSON2_CMCDMA_CCR_MINC; + *buswidth = sconfig.src_addr_width; + break; + default: + return -EINVAL; + } + + ccr |= FIELD_PREP(LOONGSON2_CMCDMA_CCR_PSIZE_MASK, dev_width) | + FIELD_PREP(LOONGSON2_CMCDMA_CCR_MSIZE_MASK, dev_width); + + /* Set DMA control register */ + lchan->chan_reg.ccr &= ~(LOONGSON2_CMCDMA_CCR_PSIZE_MASK | LOONGSON2_CMCDMA_CCR_MSIZE_MASK); + lchan->chan_reg.ccr |= ccr; + + return 0; +} + +static struct dma_async_tx_descriptor * +loongson2_cmc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, u32 sg_len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + struct loongson2_cmc_dma_desc *desc; + enum dma_slave_buswidth buswidth; + struct scatterlist *sg; + u32 num_items, i; + int ret; + + desc = kzalloc_flex(*desc, sg_req, sg_len, GFP_NOWAIT); + if (!desc) + return ERR_PTR(-ENOMEM); + + for_each_sg(sgl, sg, sg_len, i) { + ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, sg_dma_len(sg)); + if (ret) + return ERR_PTR(ret); + + num_items = DIV_ROUND_UP(sg_dma_len(sg), buswidth); + if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) { + dev_err(chan2dev(lchan), "Number of items not supported\n"); + kfree(desc); + return ERR_PTR(-EINVAL); + } + + desc->sg_req[i].len = sg_dma_len(sg); + desc->sg_req[i].chan_reg.ccr = lchan->chan_reg.ccr; + desc->sg_req[i].chan_reg.cpar = lchan->chan_reg.cpar; + desc->sg_req[i].chan_reg.cmar = sg_dma_address(sg); + desc->sg_req[i].chan_reg.cndtr = num_items; + } + + desc->num_sgs = sg_len; + desc->cyclic = false; + + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); +} + +static struct dma_async_tx_descriptor * +loongson2_cmc_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + struct loongson2_cmc_dma_desc *desc; + enum dma_slave_buswidth buswidth; + u32 num_periods, num_items, i; + int ret; + + if (unlikely(buf_len % period_len)) + return ERR_PTR(-EINVAL); + + ret = loongson2_cmc_dma_set_xfer_param(lchan, direction, &buswidth, period_len); + if (ret) + return ERR_PTR(ret); + + num_items = DIV_ROUND_UP(period_len, buswidth); + if (num_items >= LOONSON2_CMCDMA_MAX_DATA_ITEMS) { + dev_err(chan2dev(lchan), "Number of items not supported\n"); + return ERR_PTR(-EINVAL); + } + + /* Enable Circular mode */ + if (buf_len == period_len) + lchan->chan_reg.ccr |= LOONGSON2_CMCDMA_CCR_CIRC; + + num_periods = DIV_ROUND_UP(buf_len, period_len); + desc = kzalloc_flex(*desc, sg_req, num_periods, GFP_NOWAIT); + if (!desc) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < num_periods; i++) { + desc->sg_req[i].len = period_len; + desc->sg_req[i].chan_reg.ccr = lchan->chan_reg.ccr; + desc->sg_req[i].chan_reg.cpar = lchan->chan_reg.cpar; + desc->sg_req[i].chan_reg.cmar = buf_addr; + desc->sg_req[i].chan_reg.cndtr = num_items; + buf_addr += period_len; + } + + desc->num_sgs = num_periods; + desc->cyclic = true; + + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); +} + +static size_t loongson2_cmc_dma_desc_residue(struct loongson2_cmc_dma_chan *lchan, + struct loongson2_cmc_dma_desc *desc, u32 next_sg) +{ + struct loongson2_cmc_dma_dev *lddev = lmdma_get_dev(lchan); + u32 residue, width, ndtr, ccr, i; + + ccr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CCR, lchan->id); + width = FIELD_GET(LOONGSON2_CMCDMA_CCR_PSIZE_MASK, ccr); + + ndtr = loongson2_cmc_dma_read(lddev, LOONGSON2_CMCDMA_CNDTR, lchan->id); + residue = ndtr << width; + + if (lchan->desc->cyclic && next_sg == 0) + return residue; + + for (i = next_sg; i < desc->num_sgs; i++) + residue += desc->sg_req[i].len; + + return residue; +} + +static enum dma_status loongson2_cmc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + struct virt_dma_desc *vdesc; + enum dma_status status; + + status = dma_cookie_status(chan, cookie, state); + if (status == DMA_COMPLETE || !state) + return status; + + scoped_guard(spinlock_irqsave, &lchan->vchan.lock) { + vdesc = vchan_find_desc(&lchan->vchan, cookie); + if (lchan->desc && cookie == lchan->desc->vdesc.tx.cookie) + state->residue = loongson2_cmc_dma_desc_residue(lchan, lchan->desc, + lchan->next_sg); + else if (vdesc) + state->residue = loongson2_cmc_dma_desc_residue(lchan, + to_lmdma_desc(vdesc), 0); + } + + return status; +} + +static void loongson2_cmc_dma_free_chan_resources(struct dma_chan *chan) +{ + vchan_free_chan_resources(to_virt_chan(chan)); +} + +static void loongson2_cmc_dma_desc_free(struct virt_dma_desc *vdesc) +{ + kfree(to_lmdma_desc(vdesc)); +} + +static bool loongson2_cmc_dma_acpi_filter(struct dma_chan *chan, void *param) +{ + struct loongson2_cmc_dma_chan *lchan = to_lmdma_chan(chan); + struct acpi_dma_spec *dma_spec = param; + + memset(&lchan->chan_reg, 0, sizeof(struct loongson2_cmc_dma_chan_reg)); + lchan->chan_reg.ccr = dma_spec->chan_id & LOONGSON2_CMCDMA_STREAM_MASK; + + return true; +} + +static int loongson2_cmc_dma_acpi_controller_register(struct loongson2_cmc_dma_dev *lddev) +{ + struct device *dev = lddev->ddev.dev; + struct acpi_dma_filter_info *info; + + if (!is_acpi_node(dev_fwnode(dev))) + return 0; + + info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + dma_cap_zero(info->dma_cap); + info->dma_cap = lddev->ddev.cap_mask; + info->filter_fn = loongson2_cmc_dma_acpi_filter; + + return devm_acpi_dma_controller_register(dev, acpi_dma_simple_xlate, info); +} + +static struct dma_chan *loongson2_cmc_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct loongson2_cmc_dma_dev *lddev = ofdma->of_dma_data; + struct device *dev = lddev->ddev.dev; + struct loongson2_cmc_dma_chan *lchan; + struct dma_chan *chan; + + if (dma_spec->args_count < 2) + return ERR_PTR(-EINVAL); + + if (dma_spec->args[0] >= lddev->nr_channels) { + dev_err(dev, "Invalid channel id.\n"); + return ERR_PTR(-EINVAL); + } + + lchan = &lddev->chan[dma_spec->args[0]]; + chan = dma_get_slave_channel(&lchan->vchan.chan); + if (!chan) { + dev_err(dev, "No more channels available.\n"); + return ERR_PTR(-EINVAL); + } + + memset(&lchan->chan_reg, 0, sizeof(struct loongson2_cmc_dma_chan_reg)); + lchan->chan_reg.ccr = dma_spec->args[1] & LOONGSON2_CMCDMA_STREAM_MASK; + + return chan; +} + +static int loongson2_cmc_dma_of_controller_register(struct loongson2_cmc_dma_dev *lddev) +{ + struct device *dev = lddev->ddev.dev; + + if (!is_of_node(dev_fwnode(dev))) + return 0; + + return of_dma_controller_register(dev->of_node, loongson2_cmc_dma_of_xlate, lddev); +} + +static int loongson2_cmc_dma_probe(struct platform_device *pdev) +{ + const struct loongson2_cmc_dma_config *config; + struct loongson2_cmc_dma_chan *lchan; + struct loongson2_cmc_dma_dev *lddev; + struct device *dev = &pdev->dev; + struct dma_device *ddev; + u32 nr_chans, i; + int ret; + + config = (const struct loongson2_cmc_dma_config *)device_get_match_data(dev); + if (!config) + return -EINVAL; + + ret = device_property_read_u32(dev, "dma-channels", &nr_chans); + if (ret || nr_chans > config->max_channels) { + dev_err(dev, "missing or invalid dma-channels property\n"); + nr_chans = config->max_channels; + } + + lddev = devm_kzalloc(dev, struct_size(lddev, chan, nr_chans), GFP_KERNEL); + if (!lddev) + return -ENOMEM; + + lddev->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(lddev->base)) + return PTR_ERR(lddev->base); + + platform_set_drvdata(pdev, lddev); + lddev->nr_channels = nr_chans; + lddev->chan_reg_offset = config->chan_reg_offset; + + lddev->dma_clk = devm_clk_get_optional_enabled(dev, NULL); + if (IS_ERR(lddev->dma_clk)) + return dev_err_probe(dev, PTR_ERR(lddev->dma_clk), "Failed to get dma clock\n"); + + ddev = &lddev->ddev; + ddev->dev = dev; + + dma_cap_zero(ddev->cap_mask); + dma_cap_set(DMA_SLAVE, ddev->cap_mask); + dma_cap_set(DMA_PRIVATE, ddev->cap_mask); + dma_cap_set(DMA_CYCLIC, ddev->cap_mask); + + ddev->device_free_chan_resources = loongson2_cmc_dma_free_chan_resources; + ddev->device_config = loongson2_cmc_dma_slave_config; + ddev->device_prep_slave_sg = loongson2_cmc_dma_prep_slave_sg; + ddev->device_prep_dma_cyclic = loongson2_cmc_dma_prep_dma_cyclic; + ddev->device_issue_pending = loongson2_cmc_dma_issue_pending; + ddev->device_synchronize = loongson2_cmc_dma_synchronize; + ddev->device_tx_status = loongson2_cmc_dma_tx_status; + ddev->device_terminate_all = loongson2_cmc_dma_terminate_all; + + ddev->max_sg_burst = LOONSON2_CMCDMA_MAX_DATA_ITEMS; + ddev->src_addr_widths = LOONGSON2_CMCDMA_BUSWIDTHS; + ddev->dst_addr_widths = LOONGSON2_CMCDMA_BUSWIDTHS; + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + INIT_LIST_HEAD(&ddev->channels); + + for (i = 0; i < nr_chans; i++) { + lchan = &lddev->chan[i]; + + lchan->id = i; + lchan->vchan.desc_free = loongson2_cmc_dma_desc_free; + vchan_init(&lchan->vchan, ddev); + } + + ret = dmaenginem_async_device_register(ddev); + if (ret) + return dev_err_probe(dev, ret, "Failed to register DMA engine device.\n"); + + for (i = 0; i < nr_chans; i++) { + lchan = &lddev->chan[i]; + + lchan->irq = platform_get_irq(pdev, i); + if (lchan->irq < 0) + return lchan->irq; + + ret = devm_request_irq(dev, lchan->irq, loongson2_cmc_dma_chan_irq, IRQF_SHARED, + dev_name(chan2dev(lchan)), lchan); + if (ret) + return ret; + } + + ret = loongson2_cmc_dma_acpi_controller_register(lddev); + if (ret) + return dev_err_probe(dev, ret, "Failed to register dma controller with ACPI.\n"); + + ret = loongson2_cmc_dma_of_controller_register(lddev); + if (ret) + return dev_err_probe(dev, ret, "Failed to register dma controller with FDT.\n"); + + dev_info(dev, "Loongson-2 Multi-Channel DMA Controller registered successfully.\n"); + + return 0; +} + +static void loongson2_cmc_dma_remove(struct platform_device *pdev) +{ + of_dma_controller_free(pdev->dev.of_node); +} + +static const struct of_device_id loongson2_cmc_dma_of_match[] = { + { .compatible = "loongson,ls2k0300-dma", .data = &ls2k0300_cmc_dma_config }, + { .compatible = "loongson,ls2k3000-dma", .data = &ls2k3000_cmc_dma_config }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, loongson2_cmc_dma_of_match); + +static const struct acpi_device_id loongson2_cmc_dma_acpi_match[] = { + { "LOON0014", .driver_data = (kernel_ulong_t)&ls2k3000_cmc_dma_config }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(acpi, loongson2_cmc_dma_acpi_match); + +static struct platform_driver loongson2_cmc_dma_driver = { + .driver = { + .name = "loongson2-apb-cmc-dma", + .of_match_table = loongson2_cmc_dma_of_match, + .acpi_match_table = loongson2_cmc_dma_acpi_match, + }, + .probe = loongson2_cmc_dma_probe, + .remove = loongson2_cmc_dma_remove, +}; +module_platform_driver(loongson2_cmc_dma_driver); + +MODULE_DESCRIPTION("Loongson-2 Chain Multi-Channel DMA Controller driver"); +MODULE_AUTHOR("Loongson Technology Corporation Limited"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/loongson2-apb-dma.c b/drivers/dma/loongson/loongson2-apb-dma.c index b981475e6779..aceb069e71fc 100644 --- a/drivers/dma/loongson2-apb-dma.c +++ b/drivers/dma/loongson/loongson2-apb-dma.c @@ -17,8 +17,8 @@ #include <linux/platform_device.h> #include <linux/slab.h> -#include "dmaengine.h" -#include "virt-dma.h" +#include "../dmaengine.h" +#include "../virt-dma.h" /* Global Configuration Register */ #define LDMA_ORDER_ERG 0x0 @@ -461,12 +461,11 @@ static int ls2x_dma_slave_config(struct dma_chan *chan, static void ls2x_dma_issue_pending(struct dma_chan *chan) { struct ls2x_dma_chan *lchan = to_ldma_chan(chan); - unsigned long flags; - spin_lock_irqsave(&lchan->vchan.lock, flags); + guard(spinlock_irqsave)(&lchan->vchan.lock); + if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) ls2x_dma_start_transfer(lchan); - spin_unlock_irqrestore(&lchan->vchan.lock, flags); } /* @@ -478,19 +477,18 @@ static void ls2x_dma_issue_pending(struct dma_chan *chan) static int ls2x_dma_terminate_all(struct dma_chan *chan) { struct ls2x_dma_chan *lchan = to_ldma_chan(chan); - unsigned long flags; LIST_HEAD(head); - spin_lock_irqsave(&lchan->vchan.lock, flags); - /* Setting stop cmd */ - ls2x_dma_write_cmd(lchan, LDMA_STOP); - if (lchan->desc) { - vchan_terminate_vdesc(&lchan->desc->vdesc); - lchan->desc = NULL; - } + scoped_guard(spinlock_irqsave, &lchan->vchan.lock) { + /* Setting stop cmd */ + ls2x_dma_write_cmd(lchan, LDMA_STOP); + if (lchan->desc) { + vchan_terminate_vdesc(&lchan->desc->vdesc); + lchan->desc = NULL; + } - vchan_get_all_descriptors(&lchan->vchan, &head); - spin_unlock_irqrestore(&lchan->vchan.lock, flags); + vchan_get_all_descriptors(&lchan->vchan, &head); + } vchan_dma_desc_free_list(&lchan->vchan, &head); return 0; @@ -511,14 +509,13 @@ static void ls2x_dma_synchronize(struct dma_chan *chan) static int ls2x_dma_pause(struct dma_chan *chan) { struct ls2x_dma_chan *lchan = to_ldma_chan(chan); - unsigned long flags; - spin_lock_irqsave(&lchan->vchan.lock, flags); + guard(spinlock_irqsave)(&lchan->vchan.lock); + if (lchan->desc && lchan->desc->status == DMA_IN_PROGRESS) { ls2x_dma_write_cmd(lchan, LDMA_STOP); lchan->desc->status = DMA_PAUSED; } - spin_unlock_irqrestore(&lchan->vchan.lock, flags); return 0; } @@ -526,14 +523,13 @@ static int ls2x_dma_pause(struct dma_chan *chan) static int ls2x_dma_resume(struct dma_chan *chan) { struct ls2x_dma_chan *lchan = to_ldma_chan(chan); - unsigned long flags; - spin_lock_irqsave(&lchan->vchan.lock, flags); + guard(spinlock_irqsave)(&lchan->vchan.lock); + if (lchan->desc && lchan->desc->status == DMA_PAUSED) { lchan->desc->status = DMA_IN_PROGRESS; ls2x_dma_write_cmd(lchan, LDMA_START); } - spin_unlock_irqrestore(&lchan->vchan.lock, flags); return 0; } @@ -550,22 +546,22 @@ static irqreturn_t ls2x_dma_isr(int irq, void *dev_id) struct ls2x_dma_chan *lchan = dev_id; struct ls2x_dma_desc *desc; - spin_lock(&lchan->vchan.lock); - desc = lchan->desc; - if (desc) { - if (desc->cyclic) { - vchan_cyclic_callback(&desc->vdesc); - } else { - desc->status = DMA_COMPLETE; - vchan_cookie_complete(&desc->vdesc); - ls2x_dma_start_transfer(lchan); + scoped_guard(spinlock, &lchan->vchan.lock) { + desc = lchan->desc; + if (desc) { + if (desc->cyclic) { + vchan_cyclic_callback(&desc->vdesc); + } else { + desc->status = DMA_COMPLETE; + vchan_cookie_complete(&desc->vdesc); + ls2x_dma_start_transfer(lchan); + } + + /* ls2x_dma_start_transfer() updates lchan->desc */ + if (!lchan->desc) + ls2x_dma_write_cmd(lchan, LDMA_STOP); } - - /* ls2x_dma_start_transfer() updates lchan->desc */ - if (!lchan->desc) - ls2x_dma_write_cmd(lchan, LDMA_STOP); } - spin_unlock(&lchan->vchan.lock); return IRQ_HANDLED; } @@ -616,17 +612,13 @@ static int ls2x_dma_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(priv->regs), "devm_platform_ioremap_resource failed.\n"); - priv->dma_clk = devm_clk_get(&pdev->dev, NULL); + priv->dma_clk = devm_clk_get_enabled(dev, NULL); if (IS_ERR(priv->dma_clk)) - return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "devm_clk_get failed.\n"); - - ret = clk_prepare_enable(priv->dma_clk); - if (ret) - return dev_err_probe(dev, ret, "clk_prepare_enable failed.\n"); + return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "Couldn't start the clock.\n"); ret = ls2x_dma_chan_init(pdev, priv); if (ret) - goto disable_clk; + return ret; ddev = &priv->ddev; ddev->dev = dev; @@ -650,25 +642,18 @@ static int ls2x_dma_probe(struct platform_device *pdev) ddev->dst_addr_widths = LDMA_SLAVE_BUSWIDTHS; ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); - ret = dma_async_device_register(&priv->ddev); + ret = dmaenginem_async_device_register(&priv->ddev); if (ret < 0) - goto disable_clk; + return dev_err_probe(dev, ret, "Failed to register DMA engine device.\n"); ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, priv); if (ret < 0) - goto unregister_dmac; + return dev_err_probe(dev, ret, "Failed to register dma controller.\n"); platform_set_drvdata(pdev, priv); dev_info(dev, "Loongson LS2X APB DMA driver registered successfully.\n"); return 0; - -unregister_dmac: - dma_async_device_unregister(&priv->ddev); -disable_clk: - clk_disable_unprepare(priv->dma_clk); - - return ret; } /* @@ -677,11 +662,7 @@ disable_clk: */ static void ls2x_dma_remove(struct platform_device *pdev) { - struct ls2x_dma_priv *priv = platform_get_drvdata(pdev); - of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&priv->ddev); - clk_disable_unprepare(priv->dma_clk); } static const struct of_device_id ls2x_dma_of_match_table[] = { diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index c269d84d7bd2..f74e9a328588 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -531,7 +531,7 @@ static int mtk_uart_apdma_probe(struct platform_device *pdev) for (i = 0; i < mtkd->dma_requests; i++) { c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL); if (!c) { - rc = -ENODEV; + rc = -ENOMEM; goto err_no_dma; } diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index d12e729ee12c..ed520737882b 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -51,7 +51,9 @@ #define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */ #define DCSR_EORINTR BIT(9) /* The end of Receive */ -#define DRCMR(n) ((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2)) +#define DRCMR_BASE 0x0100 +#define DRCMR_EXT_BASE_DEFAULT 0x1100 +#define DRCMR_REQ_LIMIT 64 #define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */ #define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */ @@ -154,6 +156,7 @@ struct mmp_pdma_phy { * @run_bits: Control bits in DCSR register for channel start/stop * @dma_width: DMA addressing width in bits (32 or 64). Determines the * DMA mask capability of the controller hardware. + * @drcmr_ext_base: Base DRCMR address for extended requests */ struct mmp_pdma_ops { /* Hardware Register Operations */ @@ -174,6 +177,7 @@ struct mmp_pdma_ops { /* Controller Configuration */ u32 run_bits; u32 dma_width; + u32 drcmr_ext_base; }; struct mmp_pdma_device { @@ -195,6 +199,13 @@ struct mmp_pdma_device { #define to_mmp_pdma_dev(dmadev) \ container_of(dmadev, struct mmp_pdma_device, device) +static u32 mmp_pdma_get_drcmr(struct mmp_pdma_device *pdev, u32 drcmr) +{ + if (drcmr < DRCMR_REQ_LIMIT) + return DRCMR_BASE + (drcmr << 2); + return pdev->ops->drcmr_ext_base + ((drcmr - DRCMR_REQ_LIMIT) << 2); +} + /* For 32-bit PDMA */ static void write_next_addr_32(struct mmp_pdma_phy *phy, dma_addr_t addr) { @@ -301,7 +312,7 @@ static void enable_chan(struct mmp_pdma_phy *phy) pdev = to_mmp_pdma_dev(phy->vchan->chan.device); - reg = DRCMR(phy->vchan->drcmr); + reg = mmp_pdma_get_drcmr(pdev, phy->vchan->drcmr); writel(DRCMR_MAPVLD | phy->idx, phy->base + reg); dalgn = readl(phy->base + DALGN); @@ -437,7 +448,7 @@ static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan) return; /* clear the channel mapping in DRCMR */ - reg = DRCMR(pchan->drcmr); + reg = mmp_pdma_get_drcmr(pdev, pchan->drcmr); writel(0, pchan->phy->base + reg); spin_lock_irqsave(&pdev->phy_lock, flags); @@ -701,7 +712,7 @@ mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, for_each_sg(sgl, sg, sg_len, i) { addr = sg_dma_address(sg); - avail = sg_dma_len(sgl); + avail = sg_dma_len(sg); do { len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES); @@ -1179,6 +1190,7 @@ static const struct mmp_pdma_ops marvell_pdma_v1_ops = { .get_desc_dst_addr = get_desc_dst_addr_32, .run_bits = (DCSR_RUN), .dma_width = 32, + .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT, }; static const struct mmp_pdma_ops spacemit_k1_pdma_ops = { @@ -1192,6 +1204,21 @@ static const struct mmp_pdma_ops spacemit_k1_pdma_ops = { .get_desc_dst_addr = get_desc_dst_addr_64, .run_bits = (DCSR_RUN | DCSR_LPAEEN), .dma_width = 64, + .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT, +}; + +static const struct mmp_pdma_ops spacemit_k3_pdma_ops = { + .write_next_addr = write_next_addr_64, + .read_src_addr = read_src_addr_64, + .read_dst_addr = read_dst_addr_64, + .set_desc_next_addr = set_desc_next_addr_64, + .set_desc_src_addr = set_desc_src_addr_64, + .set_desc_dst_addr = set_desc_dst_addr_64, + .get_desc_src_addr = get_desc_src_addr_64, + .get_desc_dst_addr = get_desc_dst_addr_64, + .run_bits = (DCSR_RUN | DCSR_LPAEEN | DCSR_EORIRQEN | DCSR_EORSTOPEN), + .dma_width = 64, + .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT, }; static const struct of_device_id mmp_pdma_dt_ids[] = { @@ -1202,6 +1229,9 @@ static const struct of_device_id mmp_pdma_dt_ids[] = { .compatible = "spacemit,k1-pdma", .data = &spacemit_k1_pdma_ops }, { + .compatible = "spacemit,k3-pdma", + .data = &spacemit_k3_pdma_ops + }, { /* sentinel */ } }; diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index cfb9962417ef..7acb3d29dad3 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -744,20 +744,19 @@ static int mxs_dma_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; const struct mxs_dma_type *dma_type; + struct device *dev = &pdev->dev; struct mxs_dma_engine *mxs_dma; int ret, i; - mxs_dma = devm_kzalloc(&pdev->dev, sizeof(*mxs_dma), GFP_KERNEL); + mxs_dma = devm_kzalloc(dev, sizeof(*mxs_dma), GFP_KERNEL); if (!mxs_dma) return -ENOMEM; ret = of_property_read_u32(np, "dma-channels", &mxs_dma->nr_channels); - if (ret) { - dev_err(&pdev->dev, "failed to read dma-channels\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "failed to read dma-channels\n"); - dma_type = (struct mxs_dma_type *)of_device_get_match_data(&pdev->dev); + dma_type = (struct mxs_dma_type *)of_device_get_match_data(dev); mxs_dma->type = dma_type->type; mxs_dma->dev_id = dma_type->id; @@ -765,7 +764,7 @@ static int mxs_dma_probe(struct platform_device *pdev) if (IS_ERR(mxs_dma->base)) return PTR_ERR(mxs_dma->base); - mxs_dma->clk = devm_clk_get(&pdev->dev, NULL); + mxs_dma->clk = devm_clk_get(dev, NULL); if (IS_ERR(mxs_dma->clk)) return PTR_ERR(mxs_dma->clk); @@ -795,10 +794,10 @@ static int mxs_dma_probe(struct platform_device *pdev) return ret; mxs_dma->pdev = pdev; - mxs_dma->dma_device.dev = &pdev->dev; + mxs_dma->dma_device.dev = dev; /* mxs_dma gets 65535 bytes maximum sg size */ - dma_set_max_seg_size(mxs_dma->dma_device.dev, MAX_XFER_BYTES); + dma_set_max_seg_size(dev, MAX_XFER_BYTES); mxs_dma->dma_device.device_alloc_chan_resources = mxs_dma_alloc_chan_resources; mxs_dma->dma_device.device_free_chan_resources = mxs_dma_free_chan_resources; @@ -815,18 +814,15 @@ static int mxs_dma_probe(struct platform_device *pdev) mxs_dma->dma_device.device_issue_pending = mxs_dma_enable_chan; ret = dmaenginem_async_device_register(&mxs_dma->dma_device); - if (ret) { - dev_err(mxs_dma->dma_device.dev, "unable to register\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "unable to register\n"); - ret = of_dma_controller_register(np, mxs_dma_xlate, mxs_dma); - if (ret) { - dev_err(mxs_dma->dma_device.dev, - "failed to register controller\n"); - } + ret = devm_of_dma_controller_register(dev, np, mxs_dma_xlate, mxs_dma); + if (ret) + return dev_err_probe(dev, ret, + "failed to register controller\n"); - dev_info(mxs_dma->dma_device.dev, "initialized\n"); + dev_info(dev, "initialized\n"); return 0; } @@ -840,3 +836,6 @@ static struct platform_driver mxs_dma_driver = { }; builtin_platform_driver(mxs_dma_driver); + +MODULE_DESCRIPTION("MXS DMA driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c index 334425faac00..05d7321629cc 100644 --- a/drivers/dma/nbpfaxi.c +++ b/drivers/dma/nbpfaxi.c @@ -1486,20 +1486,6 @@ static void nbpf_remove(struct platform_device *pdev) clk_disable_unprepare(nbpf->clk); } -static const struct platform_device_id nbpf_ids[] = { - {"nbpfaxi64dmac1b4", (kernel_ulong_t)&nbpf_cfg[NBPF1B4]}, - {"nbpfaxi64dmac1b8", (kernel_ulong_t)&nbpf_cfg[NBPF1B8]}, - {"nbpfaxi64dmac1b16", (kernel_ulong_t)&nbpf_cfg[NBPF1B16]}, - {"nbpfaxi64dmac4b4", (kernel_ulong_t)&nbpf_cfg[NBPF4B4]}, - {"nbpfaxi64dmac4b8", (kernel_ulong_t)&nbpf_cfg[NBPF4B8]}, - {"nbpfaxi64dmac4b16", (kernel_ulong_t)&nbpf_cfg[NBPF4B16]}, - {"nbpfaxi64dmac8b4", (kernel_ulong_t)&nbpf_cfg[NBPF8B4]}, - {"nbpfaxi64dmac8b8", (kernel_ulong_t)&nbpf_cfg[NBPF8B8]}, - {"nbpfaxi64dmac8b16", (kernel_ulong_t)&nbpf_cfg[NBPF8B16]}, - {}, -}; -MODULE_DEVICE_TABLE(platform, nbpf_ids); - static int nbpf_runtime_suspend(struct device *dev) { struct nbpf_device *nbpf = dev_get_drvdata(dev); @@ -1523,7 +1509,6 @@ static struct platform_driver nbpf_driver = { .of_match_table = nbpf_match, .pm = pm_ptr(&nbpf_pm_ops), }, - .id_table = nbpf_ids, .probe = nbpf_probe, .remove = nbpf_remove, }; diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index e9fbfd5a3d51..bf805f1024f6 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -970,6 +970,7 @@ static const struct pci_device_id pch_dma_id_table[] = { { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7831_DMA2_4CH), 4}, /* SPI */ { 0, }, }; +MODULE_DEVICE_TABLE(pci, pch_dma_id_table); static SIMPLE_DEV_PM_OPS(pch_dma_pm_ops, pch_dma_suspend, pch_dma_resume); @@ -987,4 +988,3 @@ MODULE_DESCRIPTION("Intel EG20T PCH / LAPIS Semicon ML7213/ML7223/ML7831 IOH " "DMA controller driver"); MODULE_AUTHOR("Yong Wang <yong.y.wang@intel.com>"); MODULE_LICENSE("GPL v2"); -MODULE_DEVICE_TABLE(pci, pch_dma_id_table); diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 25ba84b18704..6214d9000db8 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -502,6 +502,7 @@ struct pl330_dmac { struct dma_pl330_chan *peripherals; /* keep at end */ int quirks; + struct dentry *dbgfs; struct reset_control *rstc; struct reset_control *rstc_ocp; }; @@ -2952,14 +2953,24 @@ DEFINE_SHOW_ATTRIBUTE(pl330_debugfs); static inline void init_pl330_debugfs(struct pl330_dmac *pl330) { - debugfs_create_file(dev_name(pl330->ddma.dev), - S_IFREG | 0444, NULL, pl330, - &pl330_debugfs_fops); + pl330->dbgfs = debugfs_create_file(dev_name(pl330->ddma.dev), + S_IFREG | 0444, NULL, pl330, + &pl330_debugfs_fops); +} + +static inline void deinit_pl330_debugfs(struct pl330_dmac *pl330) +{ + debugfs_remove(pl330->dbgfs); + pl330->dbgfs = NULL; } #else static inline void init_pl330_debugfs(struct pl330_dmac *pl330) { } + +static inline void deinit_pl330_debugfs(struct pl330_dmac *pl330) +{ +} #endif /* @@ -3204,6 +3215,8 @@ static void pl330_remove(struct amba_device *adev) struct dma_pl330_chan *pch, *_p; int i, irq; + deinit_pl330_debugfs(pl330); + pm_runtime_get_noresume(pl330->ddma.dev); if (adev->dev.of_node) diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index fa2ee0b3e09f..fc43124fefa8 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -744,6 +744,7 @@ pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc) sw_desc = kzalloc_flex(*sw_desc, hw_desc, nb_hw_desc, GFP_NOWAIT); if (!sw_desc) return NULL; + sw_desc->nb_desc = nb_hw_desc; sw_desc->desc_pool = chan->desc_pool; for (i = 0; i < nb_hw_desc; i++) { @@ -752,10 +753,10 @@ pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc) dev_err(&chan->vc.chan.dev->device, "%s(): Couldn't allocate the %dth hw_desc from dma_pool %p\n", __func__, i, sw_desc->desc_pool); + sw_desc->nb_desc = i; goto err; } - sw_desc->nb_desc++; sw_desc->hw_desc[i] = desc; if (i == 0) diff --git a/drivers/dma/qcom/Kconfig b/drivers/dma/qcom/Kconfig index ace75d7b835a..c71b0b5d536b 100644 --- a/drivers/dma/qcom/Kconfig +++ b/drivers/dma/qcom/Kconfig @@ -11,7 +11,7 @@ config QCOM_ADM and on-chip peripheral devices. config QCOM_BAM_DMA - tristate "QCOM BAM DMA support" + tristate "Qualcomm BAM DMA support" depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM) select DMA_ENGINE select DMA_VIRTUAL_CHANNELS @@ -20,7 +20,7 @@ config QCOM_BAM_DMA provides DMA capabilities for a variety of on-chip devices. config QCOM_GPI_DMA - tristate "Qualcomm Technologies GPI DMA support" + tristate "Qualcomm GPI DMA support" depends on ARCH_QCOM select DMA_ENGINE select DMA_VIRTUAL_CHANNELS @@ -32,7 +32,7 @@ config QCOM_GPI_DMA transfer data between DDR and peripheral. config QCOM_HIDMA_MGMT - tristate "Qualcomm Technologies HIDMA Management support" + tristate "Qualcomm HIDMA Management support" depends on HAS_IOMEM select DMA_ENGINE help @@ -44,7 +44,7 @@ config QCOM_HIDMA_MGMT host would run the QCOM_HIDMA_MGMT management driver. config QCOM_HIDMA - tristate "Qualcomm Technologies HIDMA Channel support" + tristate "Qualcomm HIDMA Channel support" depends on HAS_IOMEM select DMA_ENGINE help diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 19116295f832..05a3b1f9e0c2 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -199,6 +199,35 @@ static const struct reg_offset_data bam_v1_7_reg_info[] = { [BAM_P_FIFO_SIZES] = { 0x13820, 0x00, 0x1000, 0x00 }, }; +static const struct reg_offset_data bam_v2_0_reg_info[] = { + [BAM_CTRL] = { 0x0000, 0x00, 0x00, 0x00 }, + [BAM_REVISION] = { 0x1000, 0x00, 0x00, 0x00 }, + [BAM_NUM_PIPES] = { 0x1008, 0x00, 0x00, 0x00 }, + [BAM_DESC_CNT_TRSHLD] = { 0x0008, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS] = { 0x3010, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_MSK] = { 0x3014, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_UNMASKED] = { 0x3018, 0x00, 0x00, 0x00 }, + [BAM_IRQ_STTS] = { 0x0014, 0x00, 0x00, 0x00 }, + [BAM_IRQ_CLR] = { 0x0018, 0x00, 0x00, 0x00 }, + [BAM_IRQ_EN] = { 0x001C, 0x00, 0x00, 0x00 }, + [BAM_CNFG_BITS] = { 0x007C, 0x00, 0x00, 0x00 }, + [BAM_IRQ_SRCS_EE] = { 0x3000, 0x00, 0x00, 0x1000 }, + [BAM_IRQ_SRCS_MSK_EE] = { 0x3004, 0x00, 0x00, 0x1000 }, + [BAM_P_CTRL] = { 0xC000, 0x1000, 0x00, 0x00 }, + [BAM_P_RST] = { 0xC004, 0x1000, 0x00, 0x00 }, + [BAM_P_HALT] = { 0xC008, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_STTS] = { 0xC010, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_CLR] = { 0xC014, 0x1000, 0x00, 0x00 }, + [BAM_P_IRQ_EN] = { 0xC018, 0x1000, 0x00, 0x00 }, + [BAM_P_EVNT_DEST_ADDR] = { 0xC82C, 0x00, 0x1000, 0x00 }, + [BAM_P_EVNT_REG] = { 0xC818, 0x00, 0x1000, 0x00 }, + [BAM_P_SW_OFSTS] = { 0xC800, 0x00, 0x1000, 0x00 }, + [BAM_P_DATA_FIFO_ADDR] = { 0xC824, 0x00, 0x1000, 0x00 }, + [BAM_P_DESC_FIFO_ADDR] = { 0xC81C, 0x00, 0x1000, 0x00 }, + [BAM_P_EVNT_GEN_TRSHLD] = { 0xC828, 0x00, 0x1000, 0x00 }, + [BAM_P_FIFO_SIZES] = { 0xC820, 0x00, 0x1000, 0x00 }, +}; + /* BAM CTRL */ #define BAM_SW_RST BIT(0) #define BAM_EN BIT(1) @@ -1208,6 +1237,7 @@ static const struct of_device_id bam_of_match[] = { { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info }, { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info }, { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info }, + { .compatible = "qcom,bam-v2.0.0", .data = &bam_v2_0_reg_info }, {} }; @@ -1371,6 +1401,7 @@ static void bam_dma_remove(struct platform_device *pdev) struct bam_device *bdev = platform_get_drvdata(pdev); u32 i; + pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_force_suspend(&pdev->dev); of_dma_controller_free(pdev->dev.of_node); diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index c9a6f610ffd9..a5055a6273af 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -2260,6 +2260,7 @@ static int gpi_probe(struct platform_device *pdev) /* clear and Set capabilities */ dma_cap_zero(gpi_dev->dma_device.cap_mask); dma_cap_set(DMA_SLAVE, gpi_dev->dma_device.cap_mask); + dma_cap_set(DMA_PRIVATE, gpi_dev->dma_device.cap_mask); /* configure dmaengine apis */ gpi_dev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index 5a8dca8db5ce..c939635be21d 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -45,7 +45,6 @@ #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include <linux/list.h> -#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_device.h> #include <linux/slab.h> @@ -624,12 +623,10 @@ static ssize_t hidma_show_values(struct device *dev, { struct hidma_dev *mdev = dev_get_drvdata(dev); - buf[0] = 0; - if (strcmp(attr->attr.name, "chid") == 0) - sprintf(buf, "%d\n", mdev->chidx); + return sysfs_emit(buf, "%d\n", mdev->chidx); - return strlen(buf); + return 0; } static inline void hidma_sysfs_uninit(struct hidma_dev *dev) diff --git a/drivers/dma/qcom/hidma_ll.c b/drivers/dma/qcom/hidma_ll.c index 53244e0e34a3..f081aa787f0c 100644 --- a/drivers/dma/qcom/hidma_ll.c +++ b/drivers/dma/qcom/hidma_ll.c @@ -677,11 +677,7 @@ int hidma_ll_setup(struct hidma_lldev *lldev) /* configure interrupts */ hidma_ll_setup_irq(lldev, lldev->msi_support); - rc = hidma_ll_enable(lldev); - if (rc) - return rc; - - return rc; + return hidma_ll_enable(lldev); } void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi) diff --git a/drivers/dma/qcom/hidma_mgmt_sys.c b/drivers/dma/qcom/hidma_mgmt_sys.c index 930eae0a6257..9672ef9ee8fc 100644 --- a/drivers/dma/qcom/hidma_mgmt_sys.c +++ b/drivers/dma/qcom/hidma_mgmt_sys.c @@ -102,15 +102,12 @@ static ssize_t show_values(struct device *dev, struct device_attribute *attr, struct hidma_mgmt_dev *mdev = dev_get_drvdata(dev); unsigned int i; - buf[0] = 0; - for (i = 0; i < ARRAY_SIZE(hidma_mgmt_files); i++) { - if (strcmp(attr->attr.name, hidma_mgmt_files[i].name) == 0) { - sprintf(buf, "%d\n", hidma_mgmt_files[i].get(mdev)); - break; - } + if (strcmp(attr->attr.name, hidma_mgmt_files[i].name) == 0) + return sysfs_emit(buf, "%d\n", + hidma_mgmt_files[i].get(mdev)); } - return strlen(buf); + return 0; } static ssize_t set_values(struct device *dev, struct device_attribute *attr, @@ -143,15 +140,15 @@ static ssize_t show_values_channel(struct kobject *kobj, struct hidma_chan_attr *chattr; struct hidma_mgmt_dev *mdev; - buf[0] = 0; chattr = container_of(attr, struct hidma_chan_attr, attr); mdev = chattr->mdev; + if (strcmp(attr->attr.name, "priority") == 0) - sprintf(buf, "%d\n", mdev->priority[chattr->index]); + return sysfs_emit(buf, "%d\n", mdev->priority[chattr->index]); else if (strcmp(attr->attr.name, "weight") == 0) - sprintf(buf, "%d\n", mdev->weight[chattr->index]); + return sysfs_emit(buf, "%d\n", mdev->weight[chattr->index]); - return strlen(buf); + return 0; } static ssize_t set_values_channel(struct kobject *kobj, diff --git a/drivers/dma/sf-pdma/sf-pdma.c b/drivers/dma/sf-pdma/sf-pdma.c index b3cba11b6203..6f79cc28703e 100644 --- a/drivers/dma/sf-pdma/sf-pdma.c +++ b/drivers/dma/sf-pdma/sf-pdma.c @@ -17,7 +17,6 @@ #include <linux/device.h> #include <linux/kernel.h> #include <linux/platform_device.h> -#include <linux/mod_devicetable.h> #include <linux/dma-mapping.h> #include <linux/of.h> #include <linux/of_dma.h> diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c index d84ca551b2bf..ca76f1bb45c4 100644 --- a/drivers/dma/sh/rz-dmac.c +++ b/drivers/dma/sh/rz-dmac.c @@ -10,12 +10,15 @@ */ #include <linux/bitfield.h> +#include <linux/cleanup.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/interrupt.h> #include <linux/iopoll.h> #include <linux/irqchip/irq-renesas-rzv2h.h> +#include <linux/irqchip/irq-renesas-rzt2h.h> #include <linux/list.h> +#include <linux/lockdep.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_dma.h> @@ -32,6 +35,7 @@ enum rz_dmac_prep_type { RZ_DMAC_DESC_MEMCPY, RZ_DMAC_DESC_SLAVE_SG, + RZ_DMAC_DESC_CYCLIC, }; struct rz_lmdesc { @@ -56,10 +60,23 @@ struct rz_dmac_desc { /* For slave sg */ struct scatterlist *sg; unsigned int sgcount; + struct rz_lmdesc *start_lmdesc; }; #define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd) +/** + * enum rz_dmac_chan_status: RZ DMAC channel status + * @RZ_DMAC_CHAN_STATUS_PAUSED: Channel is paused though DMA engine callbacks + * @RZ_DMAC_CHAN_STATUS_CYCLIC: Channel is cyclic + * @RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL: Channel is paused through driver internal logic + */ +enum rz_dmac_chan_status { + RZ_DMAC_CHAN_STATUS_PAUSED, + RZ_DMAC_CHAN_STATUS_CYCLIC, + RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL, +}; + struct rz_dmac_chan { struct virt_dma_chan vc; void __iomem *ch_base; @@ -71,13 +88,18 @@ struct rz_dmac_chan { dma_addr_t src_per_address; dma_addr_t dst_per_address; + unsigned long status; + u32 chcfg; u32 chctrl; int mid_rid; + int dmac_ack; + + struct { + u32 nxla; + } pm_state; struct list_head ld_free; - struct list_head ld_queue; - struct list_head ld_active; struct { struct rz_lmdesc *base; @@ -94,9 +116,19 @@ struct rz_dmac_icu { u8 dmac_index; }; +struct rz_dmac_info { + void (*icu_register_dma_req)(struct platform_device *icu_dev, + u8 dmac_index, u8 dmac_channel, u16 req_no); + void (*icu_register_dma_ack)(struct platform_device *icu_dev, + u8 dmac_index, u8 dmac_channel, u16 ack_no); + u16 default_dma_ack_no; + u16 default_dma_req_no; +}; + struct rz_dmac { struct dma_device engine; struct rz_dmac_icu icu; + const struct rz_dmac_info *info; struct device *dev; struct reset_control *rstc; void __iomem *base; @@ -105,8 +137,6 @@ struct rz_dmac { unsigned int n_channels; struct rz_dmac_chan *channels; - bool has_icu; - DECLARE_BITMAP(modules, 1024); }; @@ -117,10 +147,12 @@ struct rz_dmac { * Registers */ +#define CRTB 0x0020 #define CHSTAT 0x0024 #define CHCTRL 0x0028 #define CHCFG 0x002c #define NXLA 0x0038 +#define CRLA 0x003c #define DCTRL 0x0000 @@ -131,10 +163,12 @@ struct rz_dmac { #define CHANNEL_8_15_COMMON_BASE 0x0700 #define CHSTAT_ER BIT(4) +#define CHSTAT_SUS BIT(3) #define CHSTAT_EN BIT(0) #define CHCTRL_CLRINTMSK BIT(17) #define CHCTRL_CLRSUS BIT(9) +#define CHCTRL_SETSUS BIT(8) #define CHCTRL_CLRTC BIT(6) #define CHCTRL_CLREND BIT(5) #define CHCTRL_CLRRQ BIT(4) @@ -170,6 +204,8 @@ struct rz_dmac { /* LINK MODE DESCRIPTOR */ #define HEADER_LV BIT(0) +#define HEADER_LE BIT(1) +#define HEADER_WBD BIT(2) #define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16 #define RZ_DMAC_MAX_CHANNELS 16 @@ -248,6 +284,12 @@ static void rz_lmdesc_setup(struct rz_dmac_chan *channel, * Descriptors preparation */ +static u32 rz_dmac_lmdesc_addr(struct rz_dmac_chan *channel, struct rz_lmdesc *lmdesc) +{ + return channel->lmdesc.base_dma + + (sizeof(struct rz_lmdesc) * (lmdesc - channel->lmdesc.base)); +} + static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel) { struct rz_lmdesc *lmdesc = channel->lmdesc.head; @@ -261,48 +303,48 @@ static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel) channel->lmdesc.head = lmdesc; } +static bool rz_dmac_chan_is_enabled(struct rz_dmac_chan *channel) +{ + u32 val = rz_dmac_ch_readl(channel, CHSTAT, 1); + + return !!(val & CHSTAT_EN); +} + +static bool rz_dmac_chan_is_paused(struct rz_dmac_chan *channel) +{ + u32 val = rz_dmac_ch_readl(channel, CHSTAT, 1); + + return !!(val & CHSTAT_SUS); +} + static void rz_dmac_enable_hw(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); - unsigned long flags; u32 nxla; u32 chctrl; - u32 chstat; dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); - local_irq_save(flags); - rz_dmac_lmdesc_recycle(channel); - nxla = channel->lmdesc.base_dma + - (sizeof(struct rz_lmdesc) * (channel->lmdesc.head - - channel->lmdesc.base)); - - chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); - if (!(chstat & CHSTAT_EN)) { - chctrl = (channel->chctrl | CHCTRL_SETEN); - rz_dmac_ch_writel(channel, nxla, NXLA, 1); - rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); - rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); - rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1); - } + nxla = rz_dmac_lmdesc_addr(channel, channel->lmdesc.head); - local_irq_restore(flags); + chctrl = (channel->chctrl | CHCTRL_SETEN); + rz_dmac_ch_writel(channel, nxla, NXLA, 1); + rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); + rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); + rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1); } static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); - unsigned long flags; dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); - local_irq_save(flags); rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); - local_irq_restore(flags); } static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) @@ -318,6 +360,70 @@ static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) rz_dmac_ext_writel(dmac, dmars32, dmars_offset); } +static void rz_dmac_set_dma_req_no(struct rz_dmac *dmac, unsigned int index, + int req_no) +{ + if (dmac->info->icu_register_dma_req) + dmac->info->icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, + index, req_no); + else + rz_dmac_set_dmars_register(dmac, index, req_no); +} + +/* + * Map MID/RID request number (bits[0:9] of DMA specifier) to the ICU + * DMA ACK signal number, per RZ/G3E hardware manual Table 4.6-28. + * + * Three peripheral groups cover all ACK-capable peripherals: + * + * PFC external DMA pins (DREQ0..DREQ4): + * req_no 0x000-0x004 -> ACK No. 84-88 (ack = req_no + 84) + * + * SSIU BUSIFs (ssip00..ssip93): + * req_no 0x161-0x198 -> ACK No. 28-83 (ack = req_no - 0x145) + * + * SPDIF (CH0..CH2) + SCU SRC (sr0..sr9) + DVC (cmd0..cmd1): + * req_no 0x199-0x1b4 -> ACK No. 0-27 (ack = req_no - 0x199) + */ +static int rz_dmac_get_ack_no(const struct rz_dmac_info *info, u16 req_no) +{ + if (!info->icu_register_dma_ack) + return -EINVAL; + + switch (req_no) { + case 0x000 ... 0x004: + /* PFC external DMA pins: ACK No. 84-88 */ + return req_no + 84; + case 0x161 ... 0x198: + /* SSIU BUSIFs: ACK No. 28-83 */ + return req_no - 0x145; + case 0x199 ... 0x1b4: + /* SPDIF + SCU SRC + DVC: ACK No. 0-27 */ + return req_no - 0x199; + default: + return -EINVAL; + } +} + +static void rz_dmac_set_dma_ack_no(struct rz_dmac *dmac, unsigned int index, + int ack_no) +{ + if (ack_no < 0 || !dmac->info->icu_register_dma_ack) + return; + + dmac->info->icu_register_dma_ack(dmac->icu.pdev, dmac->icu.dmac_index, + index, ack_no); +} + +static void rz_dmac_reset_dma_ack_no(struct rz_dmac *dmac, int ack_no) +{ + if (ack_no < 0 || !dmac->info->icu_register_dma_ack) + return; + + dmac->info->icu_register_dma_ack(dmac->icu.pdev, dmac->icu.dmac_index, + dmac->info->default_dma_ack_no, ack_no); +} + static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; @@ -326,6 +432,8 @@ static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) struct rz_dmac_desc *d = channel->desc; u32 chcfg = CHCFG_MEM_COPY; + d->start_lmdesc = lmdesc; + /* prepare descriptor */ lmdesc->sa = d->src; lmdesc->da = d->dest; @@ -333,18 +441,12 @@ static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) lmdesc->chcfg = chcfg; lmdesc->chitvl = 0; lmdesc->chext = 0; - lmdesc->header = HEADER_LV; + lmdesc->header = HEADER_LV | HEADER_LE; - if (dmac->has_icu) { - rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, - channel->index, - RZV2H_ICU_DMAC_REQ_NO_DEFAULT); - } else { - rz_dmac_set_dmars_register(dmac, channel->index, 0); - } + rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no); channel->chcfg = chcfg; - channel->chctrl = CHCTRL_STG | CHCTRL_SETEN; + channel->chctrl = CHCTRL_STG; } static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) @@ -366,6 +468,7 @@ static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) } lmdesc = channel->lmdesc.tail; + d->start_lmdesc = lmdesc; for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) { if (d->direction == DMA_DEV_TO_MEM) { @@ -381,7 +484,7 @@ static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) lmdesc->chext = 0; if (i == (sg_len - 1)) { lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM); - lmdesc->header = HEADER_LV; + lmdesc->header = HEADER_LV | HEADER_LE; } else { lmdesc->chcfg = channel->chcfg; lmdesc->header = HEADER_LV; @@ -392,28 +495,78 @@ static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) channel->lmdesc.tail = lmdesc; - if (dmac->has_icu) { - rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, - channel->index, channel->mid_rid); + rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid); + rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack); + + channel->chctrl = 0; +} + +static void rz_dmac_prepare_descs_for_cyclic(struct rz_dmac_chan *channel) +{ + struct dma_chan *chan = &channel->vc.chan; + struct rz_dmac *dmac = to_rz_dmac(chan->device); + struct rz_dmac_desc *d = channel->desc; + size_t period_len = d->sgcount; + struct rz_lmdesc *lmdesc; + size_t buf_len = d->len; + size_t periods = buf_len / period_len; + + lockdep_assert_held(&channel->vc.lock); + + channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DMS; + + if (d->direction == DMA_DEV_TO_MEM) { + channel->chcfg |= CHCFG_SAD; + channel->chcfg &= ~CHCFG_REQD; } else { - rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid); + channel->chcfg |= CHCFG_DAD | CHCFG_REQD; } - channel->chctrl = CHCTRL_SETEN; + lmdesc = channel->lmdesc.tail; + d->start_lmdesc = lmdesc; + + for (size_t i = 0; i < periods; i++) { + if (d->direction == DMA_DEV_TO_MEM) { + lmdesc->sa = d->src; + lmdesc->da = d->dest + (i * period_len); + } else { + lmdesc->sa = d->src + (i * period_len); + lmdesc->da = d->dest; + } + + lmdesc->tb = period_len; + lmdesc->chitvl = 0; + lmdesc->chext = 0; + lmdesc->chcfg = channel->chcfg; + lmdesc->header = HEADER_LV | HEADER_WBD; + + if (i == periods - 1) + lmdesc->nxla = rz_dmac_lmdesc_addr(channel, d->start_lmdesc); + + if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) + lmdesc = channel->lmdesc.base; + } + + channel->lmdesc.tail = lmdesc; + + rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid); + rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack); } -static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan) +static void rz_dmac_xfer_desc(struct rz_dmac_chan *chan) { - struct rz_dmac_desc *d = chan->desc; struct virt_dma_desc *vd; vd = vchan_next_desc(&chan->vc); - if (!vd) - return 0; + if (!vd) { + chan->desc = NULL; + return; + } list_del(&vd->node); + chan->desc = to_rz_dmac_desc(vd); - switch (d->type) { + switch (chan->desc->type) { case RZ_DMAC_DESC_MEMCPY: rz_dmac_prepare_desc_for_memcpy(chan); break; @@ -422,13 +575,12 @@ static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan) rz_dmac_prepare_descs_for_slave_sg(chan); break; - default: - return -EINVAL; + case RZ_DMAC_DESC_CYCLIC: + rz_dmac_prepare_descs_for_cyclic(chan); + break; } rz_dmac_enable_hw(chan); - - return 0; } /* @@ -447,6 +599,7 @@ static int rz_dmac_alloc_chan_resources(struct dma_chan *chan) if (!desc) break; + /* No need to lock. This is called only for the 1st client. */ list_add_tail(&desc->node, &channel->ld_free); channel->descs_allocated++; } @@ -461,34 +614,49 @@ static void rz_dmac_free_chan_resources(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); - struct rz_lmdesc *lmdesc = channel->lmdesc.base; struct rz_dmac_desc *desc, *_desc; unsigned long flags; - unsigned int i; + int ret; + + PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) { + dev_err(dmac->dev, "RPM resume failed for channel %s, ret=%d\n!", + dma_chan_name(chan), ret); + } spin_lock_irqsave(&channel->vc.lock, flags); - for (i = 0; i < DMAC_NR_LMDESC; i++) - lmdesc[i].header = 0; + rz_lmdesc_setup(channel, channel->lmdesc.base); - rz_dmac_disable_hw(channel); - list_splice_tail_init(&channel->ld_active, &channel->ld_free); - list_splice_tail_init(&channel->ld_queue, &channel->ld_free); + /* Skip touching HW if RPM resume failed. Let the cleanup do its jobs. */ + if (!ret) + rz_dmac_disable_hw(channel); if (channel->mid_rid >= 0) { clear_bit(channel->mid_rid, dmac->modules); channel->mid_rid = -EINVAL; } + channel->status = 0; + rz_dmac_reset_dma_ack_no(dmac, channel->dmac_ack); + channel->dmac_ack = -EINVAL; + spin_unlock_irqrestore(&channel->vc.lock, flags); + vchan_free_chan_resources(&channel->vc); + + spin_lock_irqsave(&channel->vc.lock, flags); + list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) { + list_del(&desc->node); kfree(desc); channel->descs_allocated--; } INIT_LIST_HEAD(&channel->ld_free); - vchan_free_chan_resources(&channel->vc); + + spin_unlock_irqrestore(&channel->vc.lock, flags); } static struct dma_async_tx_descriptor * @@ -502,10 +670,13 @@ rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", __func__, channel->index, &src, &dest, len); - if (list_empty(&channel->ld_free)) - return NULL; + scoped_guard(spinlock_irqsave, &channel->vc.lock) { + desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node); + if (!desc) + return NULL; - desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); + list_del(&desc->node); + } desc->type = RZ_DMAC_DESC_MEMCPY; desc->src = src; @@ -513,7 +684,6 @@ rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, desc->len = len; desc->direction = DMA_MEM_TO_MEM; - list_move_tail(channel->ld_free.next, &channel->ld_queue); return vchan_tx_prep(&channel->vc, &desc->vd, flags); } @@ -529,14 +699,16 @@ rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, int dma_length = 0; int i = 0; - if (list_empty(&channel->ld_free)) - return NULL; + scoped_guard(spinlock_irqsave, &channel->vc.lock) { + desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node); + if (!desc) + return NULL; - desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); + list_del(&desc->node); + } - for_each_sg(sgl, sg, sg_len, i) { + for_each_sg(sgl, sg, sg_len, i) dma_length += sg_dma_len(sg); - } desc->type = RZ_DMAC_DESC_SLAVE_SG; desc->sg = sgl; @@ -549,54 +721,114 @@ rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, else desc->dest = channel->dst_per_address; - list_move_tail(channel->ld_free.next, &channel->ld_queue); + return vchan_tx_prep(&channel->vc, &desc->vd, flags); +} + +static struct dma_async_tx_descriptor * +rz_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction direction, + unsigned long flags) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_dmac_desc *desc; + size_t periods; + + if (!is_slave_direction(direction)) + return NULL; + + if (!period_len || !buf_len) + return NULL; + + periods = buf_len / period_len; + if (!periods || periods > DMAC_NR_LMDESC) + return NULL; + + scoped_guard(spinlock_irqsave, &channel->vc.lock) { + if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) + return NULL; + + desc = list_first_entry_or_null(&channel->ld_free, struct rz_dmac_desc, node); + if (!desc) + return NULL; + + list_del(&desc->node); + + channel->status |= BIT(RZ_DMAC_CHAN_STATUS_CYCLIC); + } + + desc->type = RZ_DMAC_DESC_CYCLIC; + desc->sgcount = period_len; + desc->len = buf_len; + desc->direction = direction; + + if (direction == DMA_DEV_TO_MEM) { + desc->src = channel->src_per_address; + desc->dest = buf_addr; + } else { + desc->src = buf_addr; + desc->dest = channel->dst_per_address; + } + return vchan_tx_prep(&channel->vc, &desc->vd, flags); } static int rz_dmac_terminate_all(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); - struct rz_lmdesc *lmdesc = channel->lmdesc.base; + struct rz_dmac *dmac = to_rz_dmac(chan->device); unsigned long flags; - unsigned int i; LIST_HEAD(head); + int ret; + + PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) { + dev_err(dmac->dev, "RPM resume failed for channel %s, ret=%d\n!", + dma_chan_name(chan), ret); + } - rz_dmac_disable_hw(channel); spin_lock_irqsave(&channel->vc.lock, flags); - for (i = 0; i < DMAC_NR_LMDESC; i++) - lmdesc[i].header = 0; + /* Don't return if RPM failed. Let the cleanup do its jobs. */ + if (!ret) + rz_dmac_disable_hw(channel); + rz_lmdesc_setup(channel, channel->lmdesc.base); + + if (channel->desc) { + vchan_terminate_vdesc(&channel->desc->vd); + channel->desc = NULL; + } - list_splice_tail_init(&channel->ld_active, &channel->ld_free); - list_splice_tail_init(&channel->ld_queue, &channel->ld_free); vchan_get_all_descriptors(&channel->vc, &head); + + channel->status = 0; + spin_unlock_irqrestore(&channel->vc.lock, flags); vchan_dma_desc_free_list(&channel->vc, &head); - return 0; + return ret; } static void rz_dmac_issue_pending(struct dma_chan *chan) { struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); struct rz_dmac *dmac = to_rz_dmac(chan->device); - struct rz_dmac_desc *desc; unsigned long flags; + int ret; + + PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return; spin_lock_irqsave(&channel->vc.lock, flags); - if (!list_empty(&channel->ld_queue)) { - desc = list_first_entry(&channel->ld_queue, - struct rz_dmac_desc, node); - channel->desc = desc; - if (vchan_issue_pending(&channel->vc)) { - if (rz_dmac_xfer_desc(channel) < 0) - dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n", - channel->index); - else - list_move_tail(channel->ld_queue.next, - &channel->ld_active); - } - } + /* + * Issue the descriptor. If another transfer is already in progress, the + * issued descriptor will be handled after the current transfer finishes. + */ + if (vchan_issue_pending(&channel->vc) && !channel->desc) + rz_dmac_xfer_desc(channel); spin_unlock_irqrestore(&channel->vc.lock, flags); } @@ -654,13 +886,13 @@ static int rz_dmac_config(struct dma_chan *chan, static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd) { - /* - * Place holder - * Descriptor allocation is done during alloc_chan_resources and - * get freed during free_chan_resources. - * list is used to manage the descriptors and avoid any memory - * allocation/free during DMA read/write. - */ + struct rz_dmac_chan *channel = to_rz_dmac_chan(vd->tx.chan); + struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); + struct rz_dmac_desc *desc = to_rz_dmac_desc(vd); + + guard(spinlock_irqsave)(&vc->lock); + + list_add_tail(&desc->node, &channel->ld_free); } static void rz_dmac_device_synchronize(struct dma_chan *chan) @@ -670,18 +902,275 @@ static void rz_dmac_device_synchronize(struct dma_chan *chan) u32 chstat; int ret; + vchan_synchronize(&channel->vc); + + PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return; + ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN), 100, 100000, false, channel, CHSTAT, 1); if (ret < 0) dev_warn(dmac->dev, "DMA Timeout"); - if (dmac->has_icu) { - rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, - channel->index, - RZV2H_ICU_DMAC_REQ_NO_DEFAULT); + rz_dmac_set_dma_req_no(dmac, channel->index, dmac->info->default_dma_req_no); + rz_dmac_reset_dma_ack_no(dmac, channel->dmac_ack); +} + +static struct rz_lmdesc * +rz_dmac_get_next_lmdesc(struct rz_lmdesc *base, struct rz_lmdesc *lmdesc) +{ + struct rz_lmdesc *next = ++lmdesc; + + if (next >= base + DMAC_NR_LMDESC) + next = base; + + return next; +} + +static u32 rz_dmac_calculate_residue_bytes_in_vd(struct rz_dmac_chan *channel, + struct rz_dmac_desc *desc, u32 crla) +{ + struct rz_lmdesc *lmdesc = desc->start_lmdesc; + struct dma_chan *chan = &channel->vc.chan; + struct rz_dmac *dmac = to_rz_dmac(chan->device); + u32 residue = 0, i = 0; + + while (lmdesc->nxla != crla) { + lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); + if (++i >= DMAC_NR_LMDESC) + return 0; + } + + /* Calculate residue from next lmdesc to end of virtual desc */ + if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) { + u32 start_lmdesc_addr = rz_dmac_lmdesc_addr(channel, desc->start_lmdesc); + + while (lmdesc->nxla != start_lmdesc_addr) { + residue += lmdesc->tb; + lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); + } } else { - rz_dmac_set_dmars_register(dmac, channel->index, 0); + while (lmdesc->chcfg & CHCFG_DEM) { + residue += lmdesc->tb; + lmdesc = rz_dmac_get_next_lmdesc(channel->lmdesc.base, lmdesc); + } + } + + dev_dbg(dmac->dev, "%s: VD residue is %u\n", __func__, residue); + + return residue; +} + +static int rz_dmac_chan_get_residue(struct device *dev, struct rz_dmac_chan *channel, + dma_cookie_t cookie, u32 *residue) +{ + struct rz_dmac_desc *desc = NULL; + struct virt_dma_desc *vd; + u32 crla, crtb, i; + int ret; + + vd = vchan_find_desc(&channel->vc, cookie); + if (vd) { + /* Descriptor has been issued but not yet processed. */ + desc = to_rz_dmac_desc(vd); + *residue = desc->len; + return 0; + } else if (channel->desc && channel->desc->vd.tx.cookie == cookie) { + /* Descriptor is currently processed. */ + desc = channel->desc; + } + + if (!desc) { + /* Descriptor was not found. May be already completed by now. */ + *residue = 0; + return 0; + } + + PM_RUNTIME_ACQUIRE_IF_ENABLED(dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + /* + * We need to read two registers. Make sure the hardware does not move + * to next lmdesc while reading the current lmdesc. Trying it 3 times + * should be enough: initial read, retry, retry for the paranoid. + */ + for (i = 0; i < 3; i++) { + crla = rz_dmac_ch_readl(channel, CRLA, 1); + crtb = rz_dmac_ch_readl(channel, CRTB, 1); + /* Still the same? */ + if (crla == rz_dmac_ch_readl(channel, CRLA, 1)) + break; + } + + WARN_ONCE(i >= 3, "residue might not be continuous!"); + + /* + * Calculate number of bytes transferred in processing virtual descriptor. + * One virtual descriptor can have many lmdesc. + */ + *residue = crtb + rz_dmac_calculate_residue_bytes_in_vd(channel, desc, crla); + + return 0; +} + +static enum dma_status rz_dmac_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_dmac *dmac = to_rz_dmac(chan->device); + enum dma_status status; + u32 residue; + + scoped_guard(spinlock_irqsave, &channel->vc.lock) { + int ret; + + status = dma_cookie_status(chan, cookie, txstate); + if (status == DMA_COMPLETE || !txstate) + return status; + + ret = rz_dmac_chan_get_residue(dmac->dev, channel, cookie, &residue); + if (ret) + return DMA_ERROR; + + if (status == DMA_IN_PROGRESS && rz_dmac_chan_is_paused(channel)) + status = DMA_PAUSED; } + + dma_set_residue(txstate, residue); + + return status; +} + +static int rz_dmac_device_pause_set(struct rz_dmac_chan *channel, + unsigned long set_bitmask) +{ + int ret = 0; + u32 val; + + lockdep_assert_held(&channel->vc.lock); + + if (!rz_dmac_chan_is_enabled(channel)) + return 0; + + if (rz_dmac_chan_is_paused(channel)) + goto set_bit; + + rz_dmac_ch_writel(channel, CHCTRL_SETSUS, CHCTRL, 1); + ret = read_poll_timeout_atomic(rz_dmac_ch_readl, val, + (val & CHSTAT_SUS), 1, 1024, false, + channel, CHSTAT, 1); + +set_bit: + channel->status |= set_bitmask; + + return ret; +} + +static int rz_dmac_device_pause(struct dma_chan *chan) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_dmac *dmac = to_rz_dmac(chan->device); + int ret; + + PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + guard(spinlock_irqsave)(&channel->vc.lock); + + return rz_dmac_device_pause_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED)); +} + +static int rz_dmac_device_pause_internal(struct rz_dmac_chan *channel) +{ + lockdep_assert_held(&channel->vc.lock); + + /* Skip channels explicitly paused by consummers or disabled. */ + if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED) || + !rz_dmac_chan_is_enabled(channel)) + return 0; + + return rz_dmac_device_pause_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL)); +} + +static int rz_dmac_device_resume_set(struct rz_dmac_chan *channel, + unsigned long clear_bitmask) +{ + u32 val; + int ret; + + lockdep_assert_held(&channel->vc.lock); + + /* + * We can be: + * + * 1/ after the channel was paused by a consummer and now it + * needs to be resummed + * 2/ after the channel was paused internally (as a result of + * a system suspend with power loss or not) + * 3/ after the channel was paused by a consummer, the system + * went through a system suspend (with power loss or not) + * and the consummer wants to resume the channel + * + * To cover all the above cases we set both CLRSUS and SETEN. + * + * In case 1/ setting SETEN while the channel is still enabled + * is harmless for the controller. + * + * In case 2/ the channel is disabled when calling this function + * and setting CLRSUS is harmless for the controller as the + * channel is disabled anyway. + * + * In case 3/ the channel is disabled/enabled if the system + * went though a suspend with power loss/or not and setting + * CLRSUS/SETEN is harmless for the controller as the channel + * is enabled/disabled anyway. + */ + + rz_dmac_ch_writel(channel, CHCTRL_CLRSUS | CHCTRL_SETEN, CHCTRL, 1); + + ret = read_poll_timeout_atomic(rz_dmac_ch_readl, val, + ((val & (CHSTAT_SUS | CHSTAT_EN)) == CHSTAT_EN), + 1, 1024, false, channel, CHSTAT, 1); + + channel->status &= ~clear_bitmask; + + return ret; +} + +static int rz_dmac_device_resume(struct dma_chan *chan) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_dmac *dmac = to_rz_dmac(chan->device); + int ret; + + PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + guard(spinlock_irqsave)(&channel->vc.lock); + + if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED))) + return 0; + + return rz_dmac_device_resume_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED)); +} + +static int rz_dmac_device_resume_internal(struct rz_dmac_chan *channel) +{ + lockdep_assert_held(&channel->vc.lock); + + if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL))) + return 0; + + return rz_dmac_device_resume_set(channel, BIT(RZ_DMAC_CHAN_STATUS_PAUSED_INTERNAL)); } /* @@ -693,20 +1182,23 @@ static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) { struct dma_chan *chan = &channel->vc.chan; struct rz_dmac *dmac = to_rz_dmac(chan->device); - u32 chstat, chctrl; + u32 chstat; chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); if (chstat & CHSTAT_ER) { dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", channel->index, chstat); - rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); - goto done; + + scoped_guard(spinlock_irqsave, &channel->vc.lock) + rz_dmac_disable_hw(channel); + return; } - chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1); - rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1); -done: - return; + /* + * No need to lock. This just clears the END interrupt. Writing + * zeros to CHCTRL is just ignored by HW. + */ + rz_dmac_ch_writel(channel, CHCTRL_CLREND, CHCTRL, 1); } static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id) @@ -724,28 +1216,22 @@ static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id) static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id) { struct rz_dmac_chan *channel = dev_id; - struct rz_dmac_desc *desc = NULL; - unsigned long flags; + struct rz_dmac_desc *desc; - spin_lock_irqsave(&channel->vc.lock, flags); + guard(spinlock_irqsave)(&channel->vc.lock); - if (list_empty(&channel->ld_active)) { - /* Someone might have called terminate all */ - goto out; - } + desc = channel->desc; + if (!desc) + return IRQ_HANDLED; + + if (channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC)) { + vchan_cyclic_callback(&desc->vd); + } else { + vchan_cookie_complete(&desc->vd); + channel->desc = NULL; - desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node); - vchan_cookie_complete(&desc->vd); - list_move_tail(channel->ld_active.next, &channel->ld_free); - if (!list_empty(&channel->ld_queue)) { - desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc, - node); - channel->desc = desc; - if (rz_dmac_xfer_desc(channel) == 0) - list_move_tail(channel->ld_queue.next, &channel->ld_active); + rz_dmac_xfer_desc(channel); } -out: - spin_unlock_irqrestore(&channel->vc.lock, flags); return IRQ_HANDLED; } @@ -767,6 +1253,8 @@ static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg) channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) | CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg); + channel->dmac_ack = rz_dmac_get_ack_no(dmac->info, channel->mid_rid); + return !test_and_set_bit(channel->mid_rid, dmac->modules); } @@ -803,25 +1291,7 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, channel->index = index; channel->mid_rid = -EINVAL; - - /* Request the channel interrupt. */ - scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); - irq = platform_get_irq_byname(pdev, pdev_irqname); - if (irq < 0) - return irq; - - irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", - dev_name(dmac->dev), index); - if (!irqname) - return -ENOMEM; - - ret = devm_request_threaded_irq(dmac->dev, irq, rz_dmac_irq_handler, - rz_dmac_irq_handler_thread, 0, - irqname, channel); - if (ret) { - dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret); - return ret; - } + channel->dmac_ack = -EINVAL; /* Set io base address for each channel */ if (index < 8) { @@ -835,25 +1305,40 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, } /* Allocate descriptors */ - lmdesc = dma_alloc_coherent(&pdev->dev, - sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, - &channel->lmdesc.base_dma, GFP_KERNEL); + lmdesc = dmam_alloc_coherent(&pdev->dev, + sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, + &channel->lmdesc.base_dma, GFP_KERNEL); if (!lmdesc) { dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n"); return -ENOMEM; } rz_lmdesc_setup(channel, lmdesc); - /* Initialize register for each channel */ - rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); - channel->vc.desc_free = rz_dmac_virt_desc_free; vchan_init(&channel->vc, &dmac->engine); - INIT_LIST_HEAD(&channel->ld_queue); INIT_LIST_HEAD(&channel->ld_free); - INIT_LIST_HEAD(&channel->ld_active); - return 0; + /* Initialize register for each channel */ + rz_dmac_disable_hw(channel); + + /* Request the channel interrupt. */ + scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); + irq = platform_get_irq_byname(pdev, pdev_irqname); + if (irq < 0) + return irq; + + irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", + dev_name(dmac->dev), index); + if (!irqname) + return -ENOMEM; + + ret = devm_request_threaded_irq(dmac->dev, irq, rz_dmac_irq_handler, + rz_dmac_irq_handler_thread, 0, + irqname, channel); + if (ret) + dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", irq, ret); + + return ret; } static void rz_dmac_put_device(void *_dev) @@ -870,14 +1355,13 @@ static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac) uint32_t dmac_index; int ret; - ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args); - if (ret == -ENOENT) + if (!dmac->info->icu_register_dma_req) return 0; + + ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args); if (ret) return ret; - dmac->has_icu = true; - dmac->icu.pdev = of_find_device_by_node(args.np); of_node_put(args.np); if (!dmac->icu.pdev) { @@ -923,7 +1407,6 @@ static int rz_dmac_probe(struct platform_device *pdev) const char *irqname = "error"; struct dma_device *engine; struct rz_dmac *dmac; - int channel_num; int ret; int irq; u8 i; @@ -932,6 +1415,7 @@ static int rz_dmac_probe(struct platform_device *pdev) if (!dmac) return -ENOMEM; + dmac->info = device_get_match_data(&pdev->dev); dmac->dev = &pdev->dev; platform_set_drvdata(pdev, dmac); @@ -949,25 +1433,12 @@ static int rz_dmac_probe(struct platform_device *pdev) if (IS_ERR(dmac->base)) return PTR_ERR(dmac->base); - if (!dmac->has_icu) { + if (!dmac->info->icu_register_dma_req) { dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(dmac->ext_base)) return PTR_ERR(dmac->ext_base); } - /* Register interrupt handler for error */ - irq = platform_get_irq_byname(pdev, irqname); - if (irq < 0) - return irq; - - ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, - irqname, NULL); - if (ret) { - dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", - irq, ret); - return ret; - } - /* Initialize the channels. */ INIT_LIST_HEAD(&dmac->engine.channels); @@ -976,6 +1447,7 @@ static int rz_dmac_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc), "failed to get resets\n"); + pm_runtime_irq_safe(&pdev->dev); pm_runtime_enable(&pdev->dev); ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { @@ -993,6 +1465,18 @@ static int rz_dmac_probe(struct platform_device *pdev) goto err; } + /* Register interrupt handler for error */ + irq = platform_get_irq_byname_optional(pdev, irqname); + if (irq > 0) { + ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, + irqname, NULL); + if (ret) { + dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", + irq, ret); + goto err; + } + } + /* Register the DMAC as a DMA provider for DT. */ ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate, NULL); @@ -1003,6 +1487,9 @@ static int rz_dmac_probe(struct platform_device *pdev) engine = &dmac->engine; dma_cap_set(DMA_SLAVE, engine->cap_mask); dma_cap_set(DMA_MEMCPY, engine->cap_mask); + dma_cap_set(DMA_CYCLIC, engine->cap_mask); + engine->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + engine->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); @@ -1010,13 +1497,16 @@ static int rz_dmac_probe(struct platform_device *pdev) engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources; engine->device_free_chan_resources = rz_dmac_free_chan_resources; - engine->device_tx_status = dma_cookie_status; + engine->device_tx_status = rz_dmac_tx_status; engine->device_prep_slave_sg = rz_dmac_prep_slave_sg; engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy; + engine->device_prep_dma_cyclic = rz_dmac_prep_dma_cyclic; engine->device_config = rz_dmac_config; engine->device_terminate_all = rz_dmac_terminate_all; engine->device_issue_pending = rz_dmac_issue_pending; engine->device_synchronize = rz_dmac_device_synchronize; + engine->device_pause = rz_dmac_device_pause; + engine->device_resume = rz_dmac_device_resume; engine->copy_align = DMAENGINE_ALIGN_1_BYTE; dma_set_max_seg_size(engine->dev, U32_MAX); @@ -1031,16 +1521,6 @@ static int rz_dmac_probe(struct platform_device *pdev) dma_register_err: of_dma_controller_free(pdev->dev.of_node); err: - channel_num = i ? i - 1 : 0; - for (i = 0; i < channel_num; i++) { - struct rz_dmac_chan *channel = &dmac->channels[i]; - - dma_free_coherent(&pdev->dev, - sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, - channel->lmdesc.base, - channel->lmdesc.base_dma); - } - reset_control_assert(dmac->rstc); err_pm_runtime_put: pm_runtime_put(&pdev->dev); @@ -1053,26 +1533,151 @@ err_pm_disable: static void rz_dmac_remove(struct platform_device *pdev) { struct rz_dmac *dmac = platform_get_drvdata(pdev); - unsigned int i; dma_async_device_unregister(&dmac->engine); of_dma_controller_free(pdev->dev.of_node); - for (i = 0; i < dmac->n_channels; i++) { - struct rz_dmac_chan *channel = &dmac->channels[i]; - - dma_free_coherent(&pdev->dev, - sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, - channel->lmdesc.base, - channel->lmdesc.base_dma); - } reset_control_assert(dmac->rstc); pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); } +static void rz_dmac_suspend_recover(struct rz_dmac *dmac) +{ + int ret; + + PM_RUNTIME_ACQUIRE_IF_ENABLED(dmac->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return; + + for (unsigned int i = 0; i < dmac->n_channels; i++) { + struct rz_dmac_chan *channel = &dmac->channels[i]; + + guard(spinlock_irqsave)(&channel->vc.lock); + + if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))) + continue; + + rz_dmac_device_resume_internal(channel); + } +} + +static int rz_dmac_suspend(struct device *dev) +{ + struct rz_dmac *dmac = dev_get_drvdata(dev); + int ret = 0; + + for (unsigned int i = 0; i < dmac->n_channels; i++) { + struct rz_dmac_chan *channel = &dmac->channels[i]; + + guard(spinlock_irqsave)(&channel->vc.lock); + + if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))) + continue; + + ret = rz_dmac_device_pause_internal(channel); + if (ret) { + dev_err(dev, "Failed to suspend channel %s\n", + dma_chan_name(&channel->vc.chan)); + break; + } + + channel->pm_state.nxla = rz_dmac_ch_readl(channel, NXLA, 1); + } + + if (ret) + goto suspend_recover; + + ret = reset_control_assert(dmac->rstc); + if (ret) + goto suspend_recover; + + ret = pm_runtime_put_sync(dev); + if (ret < 0) + goto reset_deassert; + + return 0; + +reset_deassert: + reset_control_deassert(dmac->rstc); +suspend_recover: + rz_dmac_suspend_recover(dmac); + return ret; +} + +static int rz_dmac_resume(struct device *dev) +{ + struct rz_dmac *dmac = dev_get_drvdata(dev); + int errors = 0, ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret) + return ret; + + ret = reset_control_deassert(dmac->rstc); + if (ret) { + /* + * Do not put runtime PM here and keep the same state as in + * probe. As subsequent suspend/resume cycles may follow, leave + * the runtime PM as is, here, to avoid imbalances. + */ + return ret; + } + + rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); + rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); + + for (unsigned int i = 0; i < dmac->n_channels; i++) { + struct rz_dmac_chan *channel = &dmac->channels[i]; + + guard(spinlock_irqsave)(&channel->vc.lock); + + rz_dmac_disable_hw(&dmac->channels[i]); + + if (!(channel->status & BIT(RZ_DMAC_CHAN_STATUS_CYCLIC))) + continue; + + rz_dmac_set_dma_req_no(dmac, channel->index, channel->mid_rid); + rz_dmac_set_dma_ack_no(dmac, channel->index, channel->dmac_ack); + + rz_dmac_ch_writel(channel, channel->pm_state.nxla, NXLA, 1); + rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); + rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); + rz_dmac_ch_writel(channel, channel->chctrl, CHCTRL, 1); + + ret = rz_dmac_device_resume_internal(channel); + if (ret) { + errors = ret; + dev_err(dev, "Failed to resume channel %s, ret=%d\n", + dma_chan_name(&channel->vc.chan), ret); + } + } + + return errors ? : 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(rz_dmac_pm_ops, rz_dmac_suspend, rz_dmac_resume); + +static const struct rz_dmac_info rz_dmac_v2h_info = { + .icu_register_dma_req = rzv2h_icu_register_dma_req, + .icu_register_dma_ack = rzv2h_icu_register_dma_ack, + .default_dma_ack_no = RZV2H_ICU_DMAC_ACK_NO_DEFAULT, + .default_dma_req_no = RZV2H_ICU_DMAC_REQ_NO_DEFAULT, +}; + +static const struct rz_dmac_info rz_dmac_t2h_info = { + .icu_register_dma_req = rzt2h_icu_register_dma_req, + .default_dma_req_no = RZT2H_ICU_DMAC_REQ_NO_DEFAULT, +}; + +static const struct rz_dmac_info rz_dmac_generic_info = { + .default_dma_req_no = 0, +}; + static const struct of_device_id of_rz_dmac_match[] = { - { .compatible = "renesas,r9a09g057-dmac", }, - { .compatible = "renesas,rz-dmac", }, + { .compatible = "renesas,r9a09g057-dmac", .data = &rz_dmac_v2h_info }, + { .compatible = "renesas,r9a09g077-dmac", .data = &rz_dmac_t2h_info }, + { .compatible = "renesas,rz-dmac", .data = &rz_dmac_generic_info }, { /* Sentinel */ } }; MODULE_DEVICE_TABLE(of, of_rz_dmac_match); @@ -1081,6 +1686,7 @@ static struct platform_driver rz_dmac_driver = { .driver = { .name = "rz-dmac", .of_match_table = of_rz_dmac_match, + .pm = pm_ptr(&rz_dmac_pm_ops), }, .probe = rz_dmac_probe, .remove = rz_dmac_remove, diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c index 087fea3af2e4..19b32a23c882 100644 --- a/drivers/dma/sprd-dma.c +++ b/drivers/dma/sprd-dma.c @@ -1212,7 +1212,7 @@ static int sprd_dma_probe(struct platform_device *pdev) ret = pm_runtime_get_sync(&pdev->dev); if (ret < 0) - goto err_rpm; + goto err_register; ret = dma_async_device_register(&sdev->dma_dev); if (ret < 0) { @@ -1234,7 +1234,6 @@ err_of_register: err_register: pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); -err_rpm: sprd_dma_disable(sdev); return ret; } diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 9b803c0aec25..0d9ffa3e2663 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -602,7 +602,6 @@ struct d40_base { struct dma_device dma_both; struct dma_device dma_slave; struct dma_device dma_memcpy; - struct d40_chan *phy_chans; struct d40_chan *log_chans; struct d40_chan **lookup_log_chans; struct d40_chan **lookup_phy_chans; @@ -621,6 +620,7 @@ struct d40_base { u32 *regs_interrupt; u16 gcc_pwr_off_mask; struct d40_gen_dmac gen_dmac; + struct d40_chan phy_chans[]; }; static struct device *chan2dev(struct d40_chan *d40c) @@ -3128,6 +3128,7 @@ static int __init d40_hw_detect_init(struct platform_device *pdev, struct clk *clk; void __iomem *virtbase; struct d40_base *base; + size_t alloc_size; int num_log_chans; int num_phy_chans; int num_memcpy_chans; @@ -3185,22 +3186,24 @@ static int __init d40_hw_detect_init(struct platform_device *pdev, else num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4; + num_phy_chans = min(num_phy_chans, STEDMA40_MAX_PHYS); + /* The number of channels used for memcpy */ if (plat_data->num_of_memcpy_chans) num_memcpy_chans = plat_data->num_of_memcpy_chans; else num_memcpy_chans = ARRAY_SIZE(dma40_memcpy_channels); + num_memcpy_chans = min(num_memcpy_chans, D40_MEMCPY_MAX_CHANS); num_log_chans = num_phy_chans * D40_MAX_LOG_CHAN_PER_PHY; dev_info(dev, "hardware rev: %d with %d physical and %d logical channels\n", rev, num_phy_chans, num_log_chans); - base = devm_kzalloc(dev, - ALIGN(sizeof(struct d40_base), 4) + - (num_phy_chans + num_log_chans + num_memcpy_chans) * - sizeof(struct d40_chan), GFP_KERNEL); + alloc_size = struct_size(base, phy_chans, num_phy_chans); + alloc_size += sizeof(*base->log_chans) * (num_log_chans + num_memcpy_chans); + base = devm_kzalloc(dev, alloc_size, GFP_KERNEL); if (!base) return -ENOMEM; @@ -3213,7 +3216,6 @@ static int __init d40_hw_detect_init(struct platform_device *pdev, base->virtbase = virtbase; base->plat_data = plat_data; base->dev = dev; - base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4); base->log_chans = &base->phy_chans[num_phy_chans]; if (base->plat_data->num_of_phy_chans == 14) { diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index a9a254dbf8cb..7704b016aed8 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -354,8 +354,10 @@ static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan) size_t bytes; dma_addr_t pos; - pos = readl(pchan->base + DMA_CHAN_LLI_ADDR); - bytes = readl(pchan->base + DMA_CHAN_CUR_CNT); + do { + pos = readl(pchan->base + DMA_CHAN_LLI_ADDR); + bytes = readl(pchan->base + DMA_CHAN_CUR_CNT); + } while (pos != readl(pchan->base + DMA_CHAN_LLI_ADDR)); if (pos == LLI_LAST_ITEM) return bytes; @@ -945,16 +947,13 @@ static int sun6i_dma_terminate_all(struct dma_chan *chan) spin_lock_irqsave(&vchan->vc.lock, flags); - if (vchan->cyclic) { - vchan->cyclic = false; - if (pchan && pchan->desc) { - struct virt_dma_desc *vd = &pchan->desc->vd; - struct virt_dma_chan *vc = &vchan->vc; + if (pchan && pchan->desc && pchan->desc != pchan->done) { + struct virt_dma_desc *vd = &pchan->desc->vd; - list_add_tail(&vd->node, &vc->desc_completed); - } + vchan_terminate_vdesc(vd); } + vchan->cyclic = false; vchan_get_all_descriptors(&vchan->vc, &head); if (pchan) { @@ -982,7 +981,6 @@ static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan, struct sun6i_pchan *pchan = vchan->phy; struct sun6i_dma_lli *lli; struct virt_dma_desc *vd; - struct sun6i_desc *txd; enum dma_status ret; unsigned long flags; size_t bytes = 0; @@ -994,9 +992,9 @@ static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan, spin_lock_irqsave(&vchan->vc.lock, flags); vd = vchan_find_desc(&vchan->vc, cookie); - txd = to_sun6i_desc(&vd->tx); if (vd) { + struct sun6i_desc *txd = to_sun6i_desc(&vd->tx); for (lli = txd->v_lli; lli != NULL; lli = lli->v_lli_next) bytes += lli->len; } else if (!pchan || !pchan->desc) { diff --git a/drivers/dma/switchtec_dma.c b/drivers/dma/switchtec_dma.c new file mode 100644 index 000000000000..c8e2169877e9 --- /dev/null +++ b/drivers/dma/switchtec_dma.c @@ -0,0 +1,1437 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip Switchtec(tm) DMA Controller Driver + * Copyright (c) 2025, Kelvin Cao <kelvin.cao@microchip.com> + * Copyright (c) 2025, Microchip Corporation + */ + +#include <linux/bitfield.h> +#include <linux/circ_buf.h> +#include <linux/dmaengine.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/delay.h> +#include <linux/iopoll.h> + +#include "dmaengine.h" + +MODULE_DESCRIPTION("Switchtec PCIe Switch DMA Engine"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Kelvin Cao"); + +#define SWITCHTEC_DMAC_CHAN_CTRL_OFFSET 0x1000 +#define SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET 0x160000 + +#define SWITCHTEC_DMA_CHAN_HW_REGS_SIZE 0x1000 +#define SWITCHTEC_DMA_CHAN_FW_REGS_SIZE 0x80 + +#define SWITCHTEC_REG_CAP 0x80 +#define SWITCHTEC_REG_CHAN_CNT 0x84 +#define SWITCHTEC_REG_TAG_LIMIT 0x90 +#define SWITCHTEC_REG_CHAN_STS_VEC 0x94 +#define SWITCHTEC_REG_SE_BUF_CNT 0x98 +#define SWITCHTEC_REG_SE_BUF_BASE 0x9a + +#define SWITCHTEC_DESC_MAX_SIZE 0x100000 + +#define SWITCHTEC_CHAN_CTRL_PAUSE BIT(0) +#define SWITCHTEC_CHAN_CTRL_HALT BIT(1) +#define SWITCHTEC_CHAN_CTRL_RESET BIT(2) +#define SWITCHTEC_CHAN_CTRL_ERR_PAUSE BIT(3) + +#define SWITCHTEC_CHAN_STS_PAUSED BIT(9) +#define SWITCHTEC_CHAN_STS_HALTED BIT(10) +#define SWITCHTEC_CHAN_STS_PAUSED_MASK GENMASK(29, 13) + +#define SWITCHTEC_INVALID_HFID 0xffff + +#define SWITCHTEC_DMA_SQ_SIZE SZ_32K +#define SWITCHTEC_DMA_CQ_SIZE SZ_32K + +#define SWITCHTEC_DMA_RING_SIZE SZ_32K + +static const char * const channel_status_str[] = { + [13] = "received a VDM with length error status", + [14] = "received a VDM or Cpl with Unsupported Request error status", + [15] = "received a VDM or Cpl with Completion Abort error status", + [16] = "received a VDM with ECRC error status", + [17] = "received a VDM with EP error status", + [18] = "received a VDM with Reserved Cpl error status", + [19] = "received only part of split SE CplD", + [20] = "the ISP_DMAC detected a Completion Time Out", + [21] = "received a Cpl with Unsupported Request status", + [22] = "received a Cpl with Completion Abort status", + [23] = "received a Cpl with a reserved status", + [24] = "received a TLP with ECRC error status in its metadata", + [25] = "received a TLP with the EP bit set in the header", + [26] = "the ISP_DMAC tried to process a SE with an invalid Connection ID", + [27] = "the ISP_DMAC tried to process a SE with an invalid Remote Host interrupt", + [28] = "a reserved opcode was detected in an SE", + [29] = "received a SE Cpl with error status", +}; + +struct chan_hw_regs { + u16 cq_head; + u16 rsvd1; + u16 sq_tail; + u16 rsvd2; + u8 ctrl; + u8 rsvd3[3]; + u16 status; + u16 rsvd4; +}; + +#define PERF_BURST_SCALE_MASK GENMASK_U32(3, 2) +#define PERF_MRRS_MASK GENMASK_U32(6, 4) +#define PERF_INTERVAL_MASK GENMASK_U32(10, 8) +#define PERF_BURST_SIZE_MASK GENMASK_U32(14, 12) +#define PERF_ARB_WEIGHT_MASK GENMASK_U32(31, 24) + +#define SE_BUF_BASE_MASK GENMASK_U32(10, 2) +#define SE_BUF_LEN_MASK GENMASK_U32(20, 12) +#define SE_THRESH_MASK GENMASK_U32(31, 23) + +#define SWITCHTEC_CHAN_ENABLE BIT(1) + +struct chan_fw_regs { + u32 valid_en_se; + u32 cq_base_lo; + u32 cq_base_hi; + u16 cq_size; + u16 rsvd1; + u32 sq_base_lo; + u32 sq_base_hi; + u16 sq_size; + u16 rsvd2; + u32 int_vec; + u32 perf_cfg; + u32 rsvd3; + u32 perf_latency_selector; + u32 perf_fetched_se_cnt_lo; + u32 perf_fetched_se_cnt_hi; + u32 perf_byte_cnt_lo; + u32 perf_byte_cnt_hi; + u32 rsvd4; + u16 perf_se_pending; + u16 perf_se_buf_empty; + u32 perf_chan_idle; + u32 perf_lat_max; + u32 perf_lat_min; + u32 perf_lat_last; + u16 sq_current; + u16 sq_phase; + u16 cq_current; + u16 cq_phase; +}; + +struct switchtec_dma_chan { + struct switchtec_dma_dev *swdma_dev; + struct dma_chan dma_chan; + struct chan_hw_regs __iomem *mmio_chan_hw; + struct chan_fw_regs __iomem *mmio_chan_fw; + + /* Serialize hardware control register access */ + spinlock_t hw_ctrl_lock; + + struct tasklet_struct desc_task; + + /* Serialize descriptor preparation */ + spinlock_t submit_lock; + bool ring_active; + int cid; + + /* Serialize completion processing */ + spinlock_t complete_lock; + bool comp_ring_active; + + /* channel index and irq */ + int index; + int irq; + + /* + * In driver context, head is advanced by producer while + * tail is advanced by consumer. + */ + + /* the head and tail for both desc_ring and hw_sq */ + int head; + int tail; + int phase_tag; + struct switchtec_dma_hw_se_desc *hw_sq; + dma_addr_t dma_addr_sq; + + /* the tail for hw_cq */ + int cq_tail; + struct switchtec_dma_hw_ce *hw_cq; + dma_addr_t dma_addr_cq; + + struct list_head list; + + struct switchtec_dma_desc *desc_ring[SWITCHTEC_DMA_RING_SIZE]; +}; + +struct switchtec_dma_dev { + struct dma_device dma_dev; + struct pci_dev __rcu *pdev; + void __iomem *bar; + + struct switchtec_dma_chan **swdma_chans; + int chan_cnt; + int chan_status_irq; +}; + +enum chan_op { + ENABLE_CHAN, + DISABLE_CHAN, +}; + +enum switchtec_dma_opcode { + SWITCHTEC_DMA_OPC_MEMCPY = 0, + SWITCHTEC_DMA_OPC_RDIMM = 0x1, + SWITCHTEC_DMA_OPC_WRIMM = 0x2, + SWITCHTEC_DMA_OPC_RHI = 0x6, + SWITCHTEC_DMA_OPC_NOP = 0x7, +}; + +struct switchtec_dma_hw_se_desc { + u8 opc; + u8 ctrl; + __le16 tlp_setting; + __le16 rsvd1; + __le16 cid; + __le32 byte_cnt; + __le32 addr_lo; /* SADDR_LO/WIADDR_LO */ + __le32 addr_hi; /* SADDR_HI/WIADDR_HI */ + __le32 daddr_lo; + __le32 daddr_hi; + __le16 dfid; + __le16 sfid; +}; + +#define SWITCHTEC_SE_DFM BIT(5) +#define SWITCHTEC_SE_LIOF BIT(6) +#define SWITCHTEC_SE_BRR BIT(7) +#define SWITCHTEC_SE_CID_MASK GENMASK(15, 0) + +#define SWITCHTEC_CE_SC_LEN_ERR BIT(0) +#define SWITCHTEC_CE_SC_UR BIT(1) +#define SWITCHTEC_CE_SC_CA BIT(2) +#define SWITCHTEC_CE_SC_RSVD_CPL BIT(3) +#define SWITCHTEC_CE_SC_ECRC_ERR BIT(4) +#define SWITCHTEC_CE_SC_EP_SET BIT(5) +#define SWITCHTEC_CE_SC_D_RD_CTO BIT(8) +#define SWITCHTEC_CE_SC_D_RIMM_UR BIT(9) +#define SWITCHTEC_CE_SC_D_RIMM_CA BIT(10) +#define SWITCHTEC_CE_SC_D_RIMM_RSVD_CPL BIT(11) +#define SWITCHTEC_CE_SC_D_ECRC BIT(12) +#define SWITCHTEC_CE_SC_D_EP_SET BIT(13) +#define SWITCHTEC_CE_SC_D_BAD_CONNID BIT(14) +#define SWITCHTEC_CE_SC_D_BAD_RHI_ADDR BIT(15) +#define SWITCHTEC_CE_SC_D_INVD_CMD BIT(16) +#define SWITCHTEC_CE_SC_MASK GENMASK(16, 0) + +struct switchtec_dma_hw_ce { + __le32 rdimm_cpl_dw0; + __le32 rdimm_cpl_dw1; + __le32 rsvd1; + __le32 cpl_byte_cnt; + __le16 sq_head; + __le16 rsvd2; + __le32 rsvd3; + __le32 sts_code; + __le16 cid; + __le16 phase_tag; +}; + +struct switchtec_dma_desc { + struct dma_async_tx_descriptor txd; + struct switchtec_dma_hw_se_desc *hw; + u32 orig_size; + bool completed; +}; + +static int wait_for_chan_status(struct chan_hw_regs __iomem *chan_hw, u32 mask, + bool set) +{ + u32 status; + + return readl_poll_timeout_atomic(&chan_hw->status, status, + (set && (status & mask)) || + (!set && !(status & mask)), + 10, 100 * USEC_PER_MSEC); +} + +static int halt_channel(struct switchtec_dma_chan *swdma_chan) +{ + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + int ret; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + ret = -ENODEV; + goto unlock_and_exit; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + writeb(SWITCHTEC_CHAN_CTRL_HALT, &chan_hw->ctrl); + ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, true); + spin_unlock(&swdma_chan->hw_ctrl_lock); + +unlock_and_exit: + rcu_read_unlock(); + return ret; +} + +static int unhalt_channel(struct switchtec_dma_chan *swdma_chan) +{ + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + u8 ctrl; + int ret; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + ret = -ENODEV; + goto unlock_and_exit; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + ctrl = readb(&chan_hw->ctrl); + ctrl &= ~SWITCHTEC_CHAN_CTRL_HALT; + writeb(ctrl, &chan_hw->ctrl); + ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, false); + spin_unlock(&swdma_chan->hw_ctrl_lock); + +unlock_and_exit: + rcu_read_unlock(); + return ret; +} + +static void flush_pci_write(struct chan_hw_regs __iomem *chan_hw) +{ + readl(&chan_hw->cq_head); +} + +static int reset_channel(struct switchtec_dma_chan *swdma_chan) +{ + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + rcu_read_unlock(); + return -ENODEV; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + writel(SWITCHTEC_CHAN_CTRL_RESET | SWITCHTEC_CHAN_CTRL_ERR_PAUSE, + &chan_hw->ctrl); + flush_pci_write(chan_hw); + + udelay(1000); + + writel(SWITCHTEC_CHAN_CTRL_ERR_PAUSE, &chan_hw->ctrl); + spin_unlock(&swdma_chan->hw_ctrl_lock); + flush_pci_write(chan_hw); + + rcu_read_unlock(); + return 0; +} + +static int pause_reset_channel(struct switchtec_dma_chan *swdma_chan) +{ + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + rcu_read_unlock(); + return -ENODEV; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl); + spin_unlock(&swdma_chan->hw_ctrl_lock); + + flush_pci_write(chan_hw); + + rcu_read_unlock(); + + /* wait 60ms to ensure no pending CEs */ + mdelay(60); + + return reset_channel(swdma_chan); +} + +static int channel_op(struct switchtec_dma_chan *swdma_chan, int op) +{ + struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw; + struct pci_dev *pdev; + u32 valid_en_se; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + rcu_read_unlock(); + return -ENODEV; + } + + valid_en_se = readl(&chan_fw->valid_en_se); + if (op == ENABLE_CHAN) + valid_en_se |= SWITCHTEC_CHAN_ENABLE; + else + valid_en_se &= ~SWITCHTEC_CHAN_ENABLE; + + writel(valid_en_se, &chan_fw->valid_en_se); + + rcu_read_unlock(); + return 0; +} + +static int enable_channel(struct switchtec_dma_chan *swdma_chan) +{ + return channel_op(swdma_chan, ENABLE_CHAN); +} + +static int disable_channel(struct switchtec_dma_chan *swdma_chan) +{ + return channel_op(swdma_chan, DISABLE_CHAN); +} + +static void +switchtec_dma_cleanup_completed(struct switchtec_dma_chan *swdma_chan) +{ + struct device *chan_dev = &swdma_chan->dma_chan.dev->device; + struct switchtec_dma_desc *desc; + struct switchtec_dma_hw_ce *ce; + struct dmaengine_result res; + int tail, cid, se_idx, i; + __le16 phase_tag; + u32 sts_code; + __le32 *p; + + do { + spin_lock_bh(&swdma_chan->complete_lock); + if (!swdma_chan->comp_ring_active) { + spin_unlock_bh(&swdma_chan->complete_lock); + break; + } + + ce = &swdma_chan->hw_cq[swdma_chan->cq_tail]; + /* + * phase_tag is updated by hardware, ensure the value is + * not from the cache + */ + phase_tag = smp_load_acquire(&ce->phase_tag); + if (le16_to_cpu(phase_tag) == swdma_chan->phase_tag) { + spin_unlock_bh(&swdma_chan->complete_lock); + break; + } + + cid = le16_to_cpu(ce->cid); + se_idx = cid & (SWITCHTEC_DMA_SQ_SIZE - 1); + desc = swdma_chan->desc_ring[se_idx]; + + tail = swdma_chan->tail; + + res.residue = desc->orig_size - le32_to_cpu(ce->cpl_byte_cnt); + + sts_code = le32_to_cpu(ce->sts_code); + + if (!(sts_code & SWITCHTEC_CE_SC_MASK)) { + res.result = DMA_TRANS_NOERROR; + } else { + if (sts_code & SWITCHTEC_CE_SC_D_RD_CTO) + res.result = DMA_TRANS_READ_FAILED; + else + res.result = DMA_TRANS_WRITE_FAILED; + + dev_err(chan_dev, "CID 0x%04x failed, SC 0x%08x\n", cid, + (u32)(sts_code & SWITCHTEC_CE_SC_MASK)); + + p = (__le32 *)ce; + for (i = 0; i < sizeof(*ce) / 4; i++) { + dev_err(chan_dev, "CE DW%d: 0x%08x\n", i, + le32_to_cpu(*p)); + p++; + } + } + + desc->completed = true; + + swdma_chan->cq_tail++; + swdma_chan->cq_tail &= SWITCHTEC_DMA_CQ_SIZE - 1; + + rcu_read_lock(); + if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) { + rcu_read_unlock(); + spin_unlock_bh(&swdma_chan->complete_lock); + return; + } + writew(swdma_chan->cq_tail, &swdma_chan->mmio_chan_hw->cq_head); + rcu_read_unlock(); + + if (swdma_chan->cq_tail == 0) + swdma_chan->phase_tag = !swdma_chan->phase_tag; + + /* Out of order CE */ + if (se_idx != tail) { + spin_unlock_bh(&swdma_chan->complete_lock); + continue; + } + + do { + dma_cookie_complete(&desc->txd); + dma_descriptor_unmap(&desc->txd); + dmaengine_desc_get_callback_invoke(&desc->txd, &res); + desc->txd.callback = NULL; + desc->txd.callback_result = NULL; + desc->completed = false; + + tail++; + tail &= SWITCHTEC_DMA_SQ_SIZE - 1; + + /* + * Ensure the desc updates are visible before updating + * the tail index + */ + smp_store_release(&swdma_chan->tail, tail); + desc = swdma_chan->desc_ring[swdma_chan->tail]; + if (!desc->completed) + break; + } while (CIRC_CNT(READ_ONCE(swdma_chan->head), swdma_chan->tail, + SWITCHTEC_DMA_SQ_SIZE)); + + spin_unlock_bh(&swdma_chan->complete_lock); + } while (1); +} + +static void +switchtec_dma_abort_desc(struct switchtec_dma_chan *swdma_chan, int force) +{ + struct switchtec_dma_desc *desc; + struct dmaengine_result res; + + if (!force) + switchtec_dma_cleanup_completed(swdma_chan); + + spin_lock_bh(&swdma_chan->complete_lock); + + while (CIRC_CNT(swdma_chan->head, swdma_chan->tail, + SWITCHTEC_DMA_SQ_SIZE) >= 1) { + desc = swdma_chan->desc_ring[swdma_chan->tail]; + + res.residue = desc->orig_size; + res.result = DMA_TRANS_ABORTED; + + dma_cookie_complete(&desc->txd); + dma_descriptor_unmap(&desc->txd); + if (!force) + dmaengine_desc_get_callback_invoke(&desc->txd, &res); + desc->txd.callback = NULL; + desc->txd.callback_result = NULL; + + swdma_chan->tail++; + swdma_chan->tail &= SWITCHTEC_DMA_SQ_SIZE - 1; + } + + spin_unlock_bh(&swdma_chan->complete_lock); +} + +static void switchtec_dma_chan_stop(struct switchtec_dma_chan *swdma_chan) +{ + int rc; + + rc = halt_channel(swdma_chan); + if (rc) + return; + + rcu_read_lock(); + if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) { + rcu_read_unlock(); + return; + } + + writel(0, &swdma_chan->mmio_chan_fw->sq_base_lo); + writel(0, &swdma_chan->mmio_chan_fw->sq_base_hi); + writel(0, &swdma_chan->mmio_chan_fw->cq_base_lo); + writel(0, &swdma_chan->mmio_chan_fw->cq_base_hi); + + rcu_read_unlock(); +} + +static int switchtec_dma_terminate_all(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + + spin_lock_bh(&swdma_chan->complete_lock); + swdma_chan->comp_ring_active = false; + spin_unlock_bh(&swdma_chan->complete_lock); + + return pause_reset_channel(swdma_chan); +} + +static void switchtec_dma_synchronize(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + + int rc; + + switchtec_dma_abort_desc(swdma_chan, 1); + + rc = enable_channel(swdma_chan); + if (rc) + return; + + rc = reset_channel(swdma_chan); + if (rc) + return; + + rc = unhalt_channel(swdma_chan); + if (rc) + return; + + spin_lock_bh(&swdma_chan->submit_lock); + swdma_chan->head = 0; + spin_unlock_bh(&swdma_chan->submit_lock); + + spin_lock_bh(&swdma_chan->complete_lock); + swdma_chan->comp_ring_active = true; + swdma_chan->phase_tag = 0; + swdma_chan->tail = 0; + swdma_chan->cq_tail = 0; + swdma_chan->cid = 0; + dma_cookie_init(chan); + spin_unlock_bh(&swdma_chan->complete_lock); +} + +static struct dma_async_tx_descriptor * +switchtec_dma_prep_desc(struct dma_chan *c, u16 dst_fid, dma_addr_t dma_dst, + u16 src_fid, dma_addr_t dma_src, u64 data, + size_t len, unsigned long flags) + __acquires(swdma_chan->submit_lock) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(c, struct switchtec_dma_chan, dma_chan); + struct switchtec_dma_desc *desc; + int head, tail; + + spin_lock_bh(&swdma_chan->submit_lock); + + if (!swdma_chan->ring_active) + goto err_unlock; + + tail = READ_ONCE(swdma_chan->tail); + head = swdma_chan->head; + + if (!CIRC_SPACE(head, tail, SWITCHTEC_DMA_RING_SIZE)) + goto err_unlock; + + desc = swdma_chan->desc_ring[head]; + + if (src_fid != SWITCHTEC_INVALID_HFID && + dst_fid != SWITCHTEC_INVALID_HFID) + desc->hw->ctrl |= SWITCHTEC_SE_DFM; + + if (flags & DMA_PREP_INTERRUPT) + desc->hw->ctrl |= SWITCHTEC_SE_LIOF; + + if (flags & DMA_PREP_FENCE) + desc->hw->ctrl |= SWITCHTEC_SE_BRR; + + desc->txd.flags = flags; + + desc->completed = false; + desc->hw->opc = SWITCHTEC_DMA_OPC_MEMCPY; + desc->hw->addr_lo = cpu_to_le32(lower_32_bits(dma_src)); + desc->hw->addr_hi = cpu_to_le32(upper_32_bits(dma_src)); + desc->hw->daddr_lo = cpu_to_le32(lower_32_bits(dma_dst)); + desc->hw->daddr_hi = cpu_to_le32(upper_32_bits(dma_dst)); + desc->hw->byte_cnt = cpu_to_le32(len); + desc->hw->tlp_setting = 0; + desc->hw->dfid = cpu_to_le16(dst_fid); + desc->hw->sfid = cpu_to_le16(src_fid); + swdma_chan->cid &= SWITCHTEC_SE_CID_MASK; + desc->hw->cid = cpu_to_le16(swdma_chan->cid++); + desc->orig_size = len; + + /* return with the lock held, it will be released in tx_submit */ + + return &desc->txd; + +err_unlock: + /* + * Keep sparse happy by restoring an even lock count on + * this lock. + */ + __acquire(swdma_chan->submit_lock); + + spin_unlock_bh(&swdma_chan->submit_lock); + return NULL; +} + +static struct dma_async_tx_descriptor * +switchtec_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dst, + dma_addr_t dma_src, size_t len, unsigned long flags) + __acquires(swdma_chan->submit_lock) +{ + if (len > SWITCHTEC_DESC_MAX_SIZE) { + /* + * Keep sparse happy by restoring an even lock count on + * this lock. + */ + __acquire(swdma_chan->submit_lock); + return NULL; + } + + return switchtec_dma_prep_desc(c, SWITCHTEC_INVALID_HFID, dma_dst, + SWITCHTEC_INVALID_HFID, dma_src, 0, len, + flags); +} + +static dma_cookie_t +switchtec_dma_tx_submit(struct dma_async_tx_descriptor *desc) + __releases(swdma_chan->submit_lock) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(desc->chan, struct switchtec_dma_chan, dma_chan); + dma_cookie_t cookie; + int head; + + head = swdma_chan->head + 1; + head &= SWITCHTEC_DMA_RING_SIZE - 1; + + /* + * Ensure the desc updates are visible before updating the head index + */ + smp_store_release(&swdma_chan->head, head); + + cookie = dma_cookie_assign(desc); + + spin_unlock_bh(&swdma_chan->submit_lock); + + return cookie; +} + +static enum dma_status switchtec_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + enum dma_status ret; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + /* + * For jobs where the interrupts are disabled, this is the only place + * to process the completions returned by the hardware. Callers that + * disable interrupts must call tx_status() to determine when a job + * is done, so it is safe to process completions here. If a job has + * interrupts enabled, then the completions will normally be processed + * in the tasklet that is triggered by the interrupt and tx_status() + * does not need to be called. + */ + switchtec_dma_cleanup_completed(swdma_chan); + + return dma_cookie_status(chan, cookie, txstate); +} + +static void switchtec_dma_issue_pending(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; + + /* + * The sq_tail register is actually for the head of the + * submisssion queue. Chip has the opposite define of head/tail + * to the Linux kernel. + */ + + rcu_read_lock(); + if (!rcu_dereference(swdma_dev->pdev)) { + rcu_read_unlock(); + return; + } + + spin_lock_bh(&swdma_chan->submit_lock); + writew(swdma_chan->head, &swdma_chan->mmio_chan_hw->sq_tail); + spin_unlock_bh(&swdma_chan->submit_lock); + + rcu_read_unlock(); +} + +static int switchtec_dma_pause(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + int ret; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + ret = -ENODEV; + goto unlock_and_exit; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl); + ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_PAUSED, true); + spin_unlock(&swdma_chan->hw_ctrl_lock); + +unlock_and_exit: + rcu_read_unlock(); + return ret; +} + +static int switchtec_dma_resume(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw; + struct pci_dev *pdev; + int ret; + + rcu_read_lock(); + pdev = rcu_dereference(swdma_chan->swdma_dev->pdev); + if (!pdev) { + ret = -ENODEV; + goto unlock_and_exit; + } + + spin_lock(&swdma_chan->hw_ctrl_lock); + writeb(0, &chan_hw->ctrl); + ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_PAUSED, false); + spin_unlock(&swdma_chan->hw_ctrl_lock); + +unlock_and_exit: + rcu_read_unlock(); + return ret; +} + +static void switchtec_dma_desc_task(unsigned long data) +{ + struct switchtec_dma_chan *swdma_chan = (void *)data; + + switchtec_dma_cleanup_completed(swdma_chan); +} + +static irqreturn_t switchtec_dma_isr(int irq, void *chan) +{ + struct switchtec_dma_chan *swdma_chan = chan; + + if (swdma_chan->comp_ring_active) + tasklet_schedule(&swdma_chan->desc_task); + + return IRQ_HANDLED; +} + +static irqreturn_t switchtec_dma_chan_status_isr(int irq, void *dma) +{ + struct switchtec_dma_dev *swdma_dev = dma; + struct dma_device *dma_dev = &swdma_dev->dma_dev; + struct switchtec_dma_chan *swdma_chan; + struct chan_hw_regs __iomem *chan_hw; + struct device *chan_dev; + struct dma_chan *chan; + u32 chan_status; + int bit; + + list_for_each_entry(chan, &dma_dev->channels, device_node) { + swdma_chan = container_of(chan, struct switchtec_dma_chan, + dma_chan); + chan_dev = &swdma_chan->dma_chan.dev->device; + chan_hw = swdma_chan->mmio_chan_hw; + + rcu_read_lock(); + if (!rcu_dereference(swdma_dev->pdev)) { + rcu_read_unlock(); + goto out; + } + + chan_status = readl(&chan_hw->status); + chan_status &= SWITCHTEC_CHAN_STS_PAUSED_MASK; + rcu_read_unlock(); + + bit = ffs(chan_status); + if (!bit) + dev_dbg(chan_dev, "No pause bit set.\n"); + else + dev_err(chan_dev, "Paused, %s\n", + channel_status_str[bit - 1]); + } + +out: + return IRQ_HANDLED; +} + +static void switchtec_dma_free_desc(struct switchtec_dma_chan *swdma_chan) +{ + struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; + size_t size; + int i; + + size = SWITCHTEC_DMA_SQ_SIZE * sizeof(*swdma_chan->hw_sq); + if (swdma_chan->hw_sq) + dma_free_coherent(swdma_dev->dma_dev.dev, size, + swdma_chan->hw_sq, swdma_chan->dma_addr_sq); + + size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq); + if (swdma_chan->hw_cq) + dma_free_coherent(swdma_dev->dma_dev.dev, size, + swdma_chan->hw_cq, swdma_chan->dma_addr_cq); + + for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++) + kfree(swdma_chan->desc_ring[i]); +} + +static int switchtec_dma_alloc_desc(struct switchtec_dma_chan *swdma_chan) +{ + struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; + struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw; + struct switchtec_dma_desc *desc; + struct pci_dev *pdev; + size_t size; + int rc, i; + + swdma_chan->head = 0; + swdma_chan->tail = 0; + swdma_chan->cq_tail = 0; + + size = SWITCHTEC_DMA_SQ_SIZE * sizeof(*swdma_chan->hw_sq); + swdma_chan->hw_sq = dma_alloc_coherent(swdma_dev->dma_dev.dev, size, + &swdma_chan->dma_addr_sq, + GFP_NOWAIT); + if (!swdma_chan->hw_sq) { + rc = -ENOMEM; + goto free_and_exit; + } + + size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq); + swdma_chan->hw_cq = dma_alloc_coherent(swdma_dev->dma_dev.dev, size, + &swdma_chan->dma_addr_cq, + GFP_NOWAIT); + if (!swdma_chan->hw_cq) { + rc = -ENOMEM; + goto free_and_exit; + } + + /* reset host phase tag */ + swdma_chan->phase_tag = 0; + + for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++) { + desc = kzalloc_obj(*desc, GFP_NOWAIT); + if (!desc) { + rc = -ENOMEM; + goto free_and_exit; + } + + dma_async_tx_descriptor_init(&desc->txd, &swdma_chan->dma_chan); + desc->txd.tx_submit = switchtec_dma_tx_submit; + desc->hw = &swdma_chan->hw_sq[i]; + desc->completed = true; + + swdma_chan->desc_ring[i] = desc; + } + + rcu_read_lock(); + pdev = rcu_dereference(swdma_dev->pdev); + if (!pdev) { + rcu_read_unlock(); + rc = -ENODEV; + goto free_and_exit; + } + + /* set sq/cq */ + writel(lower_32_bits(swdma_chan->dma_addr_sq), &chan_fw->sq_base_lo); + writel(upper_32_bits(swdma_chan->dma_addr_sq), &chan_fw->sq_base_hi); + writel(lower_32_bits(swdma_chan->dma_addr_cq), &chan_fw->cq_base_lo); + writel(upper_32_bits(swdma_chan->dma_addr_cq), &chan_fw->cq_base_hi); + + writew(SWITCHTEC_DMA_SQ_SIZE, &swdma_chan->mmio_chan_fw->sq_size); + writew(SWITCHTEC_DMA_CQ_SIZE, &swdma_chan->mmio_chan_fw->cq_size); + + rcu_read_unlock(); + return 0; + +free_and_exit: + switchtec_dma_free_desc(swdma_chan); + return rc; +} + +static int switchtec_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev; + u32 perf_cfg; + int rc; + + rc = switchtec_dma_alloc_desc(swdma_chan); + if (rc) + return rc; + + rc = enable_channel(swdma_chan); + if (rc) + return rc; + + rc = reset_channel(swdma_chan); + if (rc) + return rc; + + rc = unhalt_channel(swdma_chan); + if (rc) + return rc; + + swdma_chan->ring_active = true; + swdma_chan->comp_ring_active = true; + swdma_chan->cid = 0; + + dma_cookie_init(chan); + + rcu_read_lock(); + if (!rcu_dereference(swdma_dev->pdev)) { + rcu_read_unlock(); + return -ENODEV; + } + + perf_cfg = readl(&swdma_chan->mmio_chan_fw->perf_cfg); + rcu_read_unlock(); + + dev_dbg(&chan->dev->device, "Burst Size: 0x%x\n", + FIELD_GET(PERF_BURST_SIZE_MASK, perf_cfg)); + + dev_dbg(&chan->dev->device, "Burst Scale: 0x%x\n", + FIELD_GET(PERF_BURST_SCALE_MASK, perf_cfg)); + + dev_dbg(&chan->dev->device, "Interval: 0x%x\n", + FIELD_GET(PERF_INTERVAL_MASK, perf_cfg)); + + dev_dbg(&chan->dev->device, "Arb Weight: 0x%x\n", + FIELD_GET(PERF_ARB_WEIGHT_MASK, perf_cfg)); + + dev_dbg(&chan->dev->device, "MRRS: 0x%x\n", + FIELD_GET(PERF_MRRS_MASK, perf_cfg)); + + return SWITCHTEC_DMA_SQ_SIZE; +} + +static void switchtec_dma_free_chan_resources(struct dma_chan *chan) +{ + struct switchtec_dma_chan *swdma_chan = + container_of(chan, struct switchtec_dma_chan, dma_chan); + + spin_lock_bh(&swdma_chan->submit_lock); + swdma_chan->ring_active = false; + spin_unlock_bh(&swdma_chan->submit_lock); + + spin_lock_bh(&swdma_chan->complete_lock); + swdma_chan->comp_ring_active = false; + spin_unlock_bh(&swdma_chan->complete_lock); + + switchtec_dma_chan_stop(swdma_chan); + switchtec_dma_abort_desc(swdma_chan, 0); + switchtec_dma_free_desc(swdma_chan); + + disable_channel(swdma_chan); +} + +static int switchtec_dma_chan_init(struct switchtec_dma_dev *swdma_dev, + struct pci_dev *pdev, int i) +{ + struct dma_device *dma = &swdma_dev->dma_dev; + struct switchtec_dma_chan *swdma_chan; + u32 valid_en_se, thresh; + int se_buf_len, irq, rc; + struct dma_chan *chan; + + swdma_chan = kzalloc_obj(*swdma_chan); + if (!swdma_chan) + return -ENOMEM; + + swdma_chan->phase_tag = 0; + swdma_chan->index = i; + swdma_chan->swdma_dev = swdma_dev; + + spin_lock_init(&swdma_chan->hw_ctrl_lock); + spin_lock_init(&swdma_chan->submit_lock); + spin_lock_init(&swdma_chan->complete_lock); + tasklet_init(&swdma_chan->desc_task, switchtec_dma_desc_task, + (unsigned long)swdma_chan); + + swdma_chan->mmio_chan_fw = + swdma_dev->bar + SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET + + i * SWITCHTEC_DMA_CHAN_FW_REGS_SIZE; + swdma_chan->mmio_chan_hw = + swdma_dev->bar + SWITCHTEC_DMAC_CHAN_CTRL_OFFSET + + i * SWITCHTEC_DMA_CHAN_HW_REGS_SIZE; + + swdma_dev->swdma_chans[i] = swdma_chan; + + rc = pause_reset_channel(swdma_chan); + if (rc) + goto free_and_exit; + + /* init perf tuner */ + writel(FIELD_PREP(PERF_BURST_SCALE_MASK, 1) | + FIELD_PREP(PERF_MRRS_MASK, 3) | + FIELD_PREP(PERF_BURST_SIZE_MASK, 6) | + FIELD_PREP(PERF_ARB_WEIGHT_MASK, 1), + &swdma_chan->mmio_chan_fw->perf_cfg); + + valid_en_se = readl(&swdma_chan->mmio_chan_fw->valid_en_se); + + dev_dbg(&pdev->dev, "Channel %d: SE buffer base %d\n", i, + FIELD_GET(SE_BUF_BASE_MASK, valid_en_se)); + + se_buf_len = FIELD_GET(SE_BUF_LEN_MASK, valid_en_se); + dev_dbg(&pdev->dev, "Channel %d: SE buffer count %d\n", i, se_buf_len); + + thresh = se_buf_len / 2; + valid_en_se |= FIELD_PREP(SE_THRESH_MASK, thresh); + writel(valid_en_se, &swdma_chan->mmio_chan_fw->valid_en_se); + + /* request irqs */ + irq = readl(&swdma_chan->mmio_chan_fw->int_vec); + dev_dbg(&pdev->dev, "Channel %d: CE irq vector %d\n", i, irq); + + rc = pci_request_irq(pdev, irq, switchtec_dma_isr, NULL, swdma_chan, + KBUILD_MODNAME); + if (rc) + goto free_and_exit; + + swdma_chan->irq = irq; + + chan = &swdma_chan->dma_chan; + chan->device = dma; + dma_cookie_init(chan); + + list_add_tail(&chan->device_node, &dma->channels); + + return 0; + +free_and_exit: + kfree(swdma_chan); + return rc; +} + +static int switchtec_dma_chan_free(struct pci_dev *pdev, + struct switchtec_dma_chan *swdma_chan) +{ + spin_lock_bh(&swdma_chan->submit_lock); + swdma_chan->ring_active = false; + spin_unlock_bh(&swdma_chan->submit_lock); + + spin_lock_bh(&swdma_chan->complete_lock); + swdma_chan->comp_ring_active = false; + spin_unlock_bh(&swdma_chan->complete_lock); + + pci_free_irq(pdev, swdma_chan->irq, swdma_chan); + tasklet_kill(&swdma_chan->desc_task); + + switchtec_dma_chan_stop(swdma_chan); + + return 0; +} + +static int switchtec_dma_chans_release(struct pci_dev *pdev, + struct switchtec_dma_dev *swdma_dev) +{ + int i; + + for (i = 0; i < swdma_dev->chan_cnt; i++) + switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]); + + return 0; +} + +static int switchtec_dma_chans_enumerate(struct switchtec_dma_dev *swdma_dev, + struct pci_dev *pdev, int chan_cnt) +{ + struct dma_device *dma = &swdma_dev->dma_dev; + int base, cnt, rc, i; + + swdma_dev->swdma_chans = kzalloc_objs(*swdma_dev->swdma_chans, chan_cnt); + + if (!swdma_dev->swdma_chans) + return -ENOMEM; + + base = readw(swdma_dev->bar + SWITCHTEC_REG_SE_BUF_BASE); + cnt = readw(swdma_dev->bar + SWITCHTEC_REG_SE_BUF_CNT); + + dev_dbg(&pdev->dev, "EP SE buffer base %d\n", base); + dev_dbg(&pdev->dev, "EP SE buffer count %d\n", cnt); + + INIT_LIST_HEAD(&dma->channels); + + for (i = 0; i < chan_cnt; i++) { + rc = switchtec_dma_chan_init(swdma_dev, pdev, i); + if (rc) { + dev_err(&pdev->dev, "Channel %d: init channel failed\n", + i); + chan_cnt = i; + goto err_exit; + } + } + + return chan_cnt; + +err_exit: + for (i = 0; i < chan_cnt; i++) + switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]); + + kfree(swdma_dev->swdma_chans); + + return rc; +} + +static void switchtec_dma_release(struct dma_device *dma_dev) +{ + struct switchtec_dma_dev *swdma_dev = + container_of(dma_dev, struct switchtec_dma_dev, dma_dev); + int i; + + for (i = 0; i < swdma_dev->chan_cnt; i++) + kfree(swdma_dev->swdma_chans[i]); + + kfree(swdma_dev->swdma_chans); + + put_device(dma_dev->dev); + kfree(swdma_dev); +} + +static int switchtec_dma_create(struct pci_dev *pdev) +{ + struct switchtec_dma_dev *swdma_dev; + int chan_cnt, nr_vecs, irq, rc; + struct dma_device *dma; + struct dma_chan *chan; + + /* + * Create the switchtec dma device + */ + swdma_dev = kzalloc_obj(*swdma_dev); + if (!swdma_dev) + return -ENOMEM; + + swdma_dev->bar = ioremap(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); + + RCU_INIT_POINTER(swdma_dev->pdev, pdev); + + nr_vecs = pci_msix_vec_count(pdev); + rc = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX); + if (rc < 0) + goto err_exit; + + irq = readw(swdma_dev->bar + SWITCHTEC_REG_CHAN_STS_VEC); + pci_dbg(pdev, "Channel pause irq vector %d\n", irq); + + rc = pci_request_irq(pdev, irq, NULL, switchtec_dma_chan_status_isr, + swdma_dev, KBUILD_MODNAME); + if (rc) + goto err_exit; + + swdma_dev->chan_status_irq = irq; + + chan_cnt = readl(swdma_dev->bar + SWITCHTEC_REG_CHAN_CNT); + if (!chan_cnt) { + pci_err(pdev, "No channel configured.\n"); + rc = -ENXIO; + goto err_exit; + } + + chan_cnt = switchtec_dma_chans_enumerate(swdma_dev, pdev, chan_cnt); + if (chan_cnt < 0) { + pci_err(pdev, "Failed to enumerate dma channels: %d\n", + chan_cnt); + rc = -ENXIO; + goto err_exit; + } + + swdma_dev->chan_cnt = chan_cnt; + + dma = &swdma_dev->dma_dev; + dma->copy_align = DMAENGINE_ALIGN_8_BYTES; + dma_cap_set(DMA_MEMCPY, dma->cap_mask); + dma_cap_set(DMA_PRIVATE, dma->cap_mask); + dma->dev = get_device(&pdev->dev); + + dma->device_alloc_chan_resources = switchtec_dma_alloc_chan_resources; + dma->device_free_chan_resources = switchtec_dma_free_chan_resources; + dma->device_prep_dma_memcpy = switchtec_dma_prep_memcpy; + dma->device_tx_status = switchtec_dma_tx_status; + dma->device_issue_pending = switchtec_dma_issue_pending; + dma->device_pause = switchtec_dma_pause; + dma->device_resume = switchtec_dma_resume; + dma->device_terminate_all = switchtec_dma_terminate_all; + dma->device_synchronize = switchtec_dma_synchronize; + dma->device_release = switchtec_dma_release; + + rc = dma_async_device_register(dma); + if (rc) { + pci_err(pdev, "Failed to register dma device: %d\n", rc); + goto err_chans_release_exit; + } + + pci_dbg(pdev, "Channel count: %d\n", chan_cnt); + + list_for_each_entry(chan, &dma->channels, device_node) + pci_dbg(pdev, "%s\n", dma_chan_name(chan)); + + pci_set_drvdata(pdev, swdma_dev); + + return 0; + +err_chans_release_exit: + switchtec_dma_chans_release(pdev, swdma_dev); + +err_exit: + if (swdma_dev->chan_status_irq) + free_irq(swdma_dev->chan_status_irq, swdma_dev); + + iounmap(swdma_dev->bar); + kfree(swdma_dev); + return rc; +} + +static int switchtec_dma_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + int rc; + + rc = pci_enable_device(pdev); + if (rc) + return rc; + + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + + rc = pci_request_mem_regions(pdev, KBUILD_MODNAME); + if (rc) + goto err_disable; + + pci_set_master(pdev); + + rc = switchtec_dma_create(pdev); + if (rc) + goto err_free; + + return 0; + +err_free: + pci_free_irq_vectors(pdev); + pci_release_mem_regions(pdev); + +err_disable: + pci_disable_device(pdev); + + return rc; +} + +static void switchtec_dma_remove(struct pci_dev *pdev) +{ + struct switchtec_dma_dev *swdma_dev = pci_get_drvdata(pdev); + + switchtec_dma_chans_release(pdev, swdma_dev); + + rcu_assign_pointer(swdma_dev->pdev, NULL); + synchronize_rcu(); + + pci_free_irq(pdev, swdma_dev->chan_status_irq, swdma_dev); + + pci_free_irq_vectors(pdev); + + dma_async_device_unregister(&swdma_dev->dma_dev); + + iounmap(swdma_dev->bar); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +/* + * Also use the class code to identify the devices, as some of the + * device IDs are also used for other devices with other classes by + * Microsemi. + */ +#define SW_ID(vendor_id, device_id) \ + { \ + .vendor = vendor_id, \ + .device = device_id, \ + .subvendor = PCI_ANY_ID, \ + .subdevice = PCI_ANY_ID, \ + .class = PCI_CLASS_SYSTEM_OTHER << 8, \ + .class_mask = 0xffffffff, \ + } + +static const struct pci_device_id switchtec_dma_pci_tbl[] = { + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4000), /* PFX 100XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4084), /* PFX 84XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4068), /* PFX 68XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4052), /* PFX 52XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4036), /* PFX 36XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4028), /* PFX 28XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4100), /* PSX 100XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4184), /* PSX 84XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4168), /* PSX 68XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4152), /* PSX 52XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4136), /* PSX 36XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4128), /* PSX 28XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4352), /* PFXA 52XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4336), /* PFXA 36XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4328), /* PFXA 28XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4452), /* PSXA 52XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4436), /* PSXA 36XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4428), /* PSXA 28XG4 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5000), /* PFX 100XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5084), /* PFX 84XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5068), /* PFX 68XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5052), /* PFX 52XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5036), /* PFX 36XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5028), /* PFX 28XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5100), /* PSX 100XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5184), /* PSX 84XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5168), /* PSX 68XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5152), /* PSX 52XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5136), /* PSX 36XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5128), /* PSX 28XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5300), /* PFXA 100XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5384), /* PFXA 84XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5368), /* PFXA 68XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5352), /* PFXA 52XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5336), /* PFXA 36XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5328), /* PFXA 28XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5400), /* PSXA 100XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5484), /* PSXA 84XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5468), /* PSXA 68XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5452), /* PSXA 52XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5436), /* PSXA 36XG5 */ + SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5428), /* PSXA 28XG5 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1001), /* PCI1001 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1002), /* PCI1002 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1003), /* PCI1003 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1004), /* PCI1004 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1005), /* PCI1005 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1006), /* PCI1006 16XG4 */ + SW_ID(PCI_VENDOR_ID_EFAR, 0x1008), /* PCI1008 16XG4 */ + {0} +}; +MODULE_DEVICE_TABLE(pci, switchtec_dma_pci_tbl); + +static struct pci_driver switchtec_dma_pci_driver = { + .name = KBUILD_MODNAME, + .id_table = switchtec_dma_pci_tbl, + .probe = switchtec_dma_probe, + .remove = switchtec_dma_remove, +}; +module_pci_driver(switchtec_dma_pci_driver); diff --git a/drivers/dma/tegra186-gpc-dma.c b/drivers/dma/tegra186-gpc-dma.c index 5948fbf32c21..64cedef1050a 100644 --- a/drivers/dma/tegra186-gpc-dma.c +++ b/drivers/dma/tegra186-gpc-dma.c @@ -15,6 +15,7 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/of_dma.h> +#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/reset.h> #include <linux/slab.h> @@ -22,7 +23,6 @@ #include "virt-dma.h" /* CSR register */ -#define TEGRA_GPCDMA_CHAN_CSR 0x00 #define TEGRA_GPCDMA_CSR_ENB BIT(31) #define TEGRA_GPCDMA_CSR_IE_EOC BIT(30) #define TEGRA_GPCDMA_CSR_ONCE BIT(27) @@ -58,7 +58,6 @@ #define TEGRA_GPCDMA_CSR_WEIGHT GENMASK(13, 10) /* STATUS register */ -#define TEGRA_GPCDMA_CHAN_STATUS 0x004 #define TEGRA_GPCDMA_STATUS_BUSY BIT(31) #define TEGRA_GPCDMA_STATUS_ISE_EOC BIT(30) #define TEGRA_GPCDMA_STATUS_PING_PONG BIT(28) @@ -70,22 +69,13 @@ #define TEGRA_GPCDMA_STATUS_IRQ_STA BIT(21) #define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA BIT(20) -#define TEGRA_GPCDMA_CHAN_CSRE 0x008 #define TEGRA_GPCDMA_CHAN_CSRE_PAUSE BIT(31) -/* Source address */ -#define TEGRA_GPCDMA_CHAN_SRC_PTR 0x00C - -/* Destination address */ -#define TEGRA_GPCDMA_CHAN_DST_PTR 0x010 - /* High address pointer */ -#define TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR 0x014 #define TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR GENMASK(7, 0) #define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR GENMASK(23, 16) /* MC sequence register */ -#define TEGRA_GPCDMA_CHAN_MCSEQ 0x18 #define TEGRA_GPCDMA_MCSEQ_DATA_SWAP BIT(31) #define TEGRA_GPCDMA_MCSEQ_REQ_COUNT GENMASK(30, 25) #define TEGRA_GPCDMA_MCSEQ_BURST GENMASK(24, 23) @@ -101,7 +91,6 @@ #define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK GENMASK(6, 0) /* MMIO sequence register */ -#define TEGRA_GPCDMA_CHAN_MMIOSEQ 0x01c #define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF BIT(31) #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH GENMASK(30, 28) #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8 \ @@ -120,17 +109,7 @@ #define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD GENMASK(18, 16) #define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT GENMASK(8, 7) -/* Channel WCOUNT */ -#define TEGRA_GPCDMA_CHAN_WCOUNT 0x20 - -/* Transfer count */ -#define TEGRA_GPCDMA_CHAN_XFER_COUNT 0x24 - -/* DMA byte count status */ -#define TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS 0x28 - /* Error Status Register */ -#define TEGRA_GPCDMA_CHAN_ERR_STATUS 0x30 #define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT 8 #define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK 0xF #define TEGRA_GPCDMA_CHAN_ERR_TYPE(err) ( \ @@ -143,16 +122,6 @@ #define TEGRA_DMA_MC_SLAVE_ERR 0xB #define TEGRA_DMA_MMIO_SLAVE_ERR 0xA -/* Fixed Pattern */ -#define TEGRA_GPCDMA_CHAN_FIXED_PATTERN 0x34 - -#define TEGRA_GPCDMA_CHAN_TZ 0x38 -#define TEGRA_GPCDMA_CHAN_TZ_MMIO_PROT_1 BIT(0) -#define TEGRA_GPCDMA_CHAN_TZ_MC_PROT_1 BIT(1) - -#define TEGRA_GPCDMA_CHAN_SPARE 0x3c -#define TEGRA_GPCDMA_CHAN_SPARE_EN_LEGACY_FC BIT(16) - /* * If any burst is in flight and DMA paused then this is the time to complete * on-flight burst and update DMA status register. @@ -178,21 +147,30 @@ struct tegra_dma_channel; */ struct tegra_dma_chip_data { bool hw_support_pause; + unsigned int addr_bits; unsigned int nr_channels; unsigned int channel_reg_size; unsigned int max_dma_count; + const struct tegra_dma_channel_regs *channel_regs; int (*terminate)(struct tegra_dma_channel *tdc); }; /* DMA channel registers */ struct tegra_dma_channel_regs { u32 csr; - u32 src_ptr; - u32 dst_ptr; - u32 high_addr_ptr; + u32 status; + u32 csre; + u32 src; + u32 dst; + u32 high_addr; + u32 src_high; + u32 dst_high; u32 mc_seq; u32 mmio_seq; u32 wcount; + u32 wxfer; + u32 wstatus; + u32 err_status; u32 fixed_pattern; }; @@ -205,7 +183,13 @@ struct tegra_dma_channel_regs { */ struct tegra_dma_sg_req { unsigned int len; - struct tegra_dma_channel_regs ch_regs; + dma_addr_t src; + dma_addr_t dst; + u32 csr; + u32 mc_seq; + u32 mmio_seq; + u32 wcount; + u32 fixed_pattern; }; /* @@ -228,19 +212,20 @@ struct tegra_dma_desc { * tegra_dma_channel: Channel specific information */ struct tegra_dma_channel { - bool config_init; - char name[30]; - enum dma_transfer_direction sid_dir; - enum dma_status status; - int id; - int irq; - int slave_id; + const struct tegra_dma_channel_regs *regs; struct tegra_dma *tdma; struct virt_dma_chan vc; struct tegra_dma_desc *dma_desc; struct dma_slave_config dma_sconfig; + enum dma_transfer_direction sid_dir; + enum dma_status status; unsigned int stream_id; unsigned long chan_base_offset; + bool config_init; + char name[30]; + int id; + int irq; + int slave_id; }; /* @@ -284,26 +269,55 @@ static inline struct device *tdc2dev(struct tegra_dma_channel *tdc) return tdc->vc.chan.device->dev; } +static void tegra_dma_program_addr(struct tegra_dma_channel *tdc, + struct tegra_dma_sg_req *sg_req) +{ + tdc_write(tdc, tdc->regs->src, lower_32_bits(sg_req->src)); + tdc_write(tdc, tdc->regs->dst, lower_32_bits(sg_req->dst)); + + if (tdc->tdma->chip_data->addr_bits > 39) { + tdc_write(tdc, tdc->regs->src_high, upper_32_bits(sg_req->src)); + tdc_write(tdc, tdc->regs->dst_high, upper_32_bits(sg_req->dst)); + } else { + u32 src_high = FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, + upper_32_bits(sg_req->src)); + u32 dst_high = FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, + upper_32_bits(sg_req->dst)); + + tdc_write(tdc, tdc->regs->high_addr, src_high | dst_high); + } +} + static void tegra_dma_dump_chan_regs(struct tegra_dma_channel *tdc) { dev_dbg(tdc2dev(tdc), "DMA Channel %d name %s register dump:\n", tdc->id, tdc->name); - dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x SRC %x DST %x\n", - tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR), - tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS), - tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE), - tdc_read(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR), - tdc_read(tdc, TEGRA_GPCDMA_CHAN_DST_PTR) - ); - dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x BSTA %x\n", - tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ), - tdc_read(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ), - tdc_read(tdc, TEGRA_GPCDMA_CHAN_WCOUNT), - tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT), - tdc_read(tdc, TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS) - ); + dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x\n", + tdc_read(tdc, tdc->regs->csr), + tdc_read(tdc, tdc->regs->status), + tdc_read(tdc, tdc->regs->csre)); + + if (tdc->tdma->chip_data->addr_bits > 39) { + dev_dbg(tdc2dev(tdc), "SRC %x SRC HI %x DST %x DST HI %x\n", + tdc_read(tdc, tdc->regs->src), + tdc_read(tdc, tdc->regs->src_high), + tdc_read(tdc, tdc->regs->dst), + tdc_read(tdc, tdc->regs->dst_high)); + } else { + dev_dbg(tdc2dev(tdc), "SRC %x DST %x HI ADDR %x\n", + tdc_read(tdc, tdc->regs->src), + tdc_read(tdc, tdc->regs->dst), + tdc_read(tdc, tdc->regs->high_addr)); + } + + dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x WSTA %x\n", + tdc_read(tdc, tdc->regs->mc_seq), + tdc_read(tdc, tdc->regs->mmio_seq), + tdc_read(tdc, tdc->regs->wcount), + tdc_read(tdc, tdc->regs->wxfer), + tdc_read(tdc, tdc->regs->wstatus)); dev_dbg(tdc2dev(tdc), "DMA ERR_STA %x\n", - tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS)); + tdc_read(tdc, tdc->regs->err_status)); } static int tegra_dma_sid_reserve(struct tegra_dma_channel *tdc, @@ -377,13 +391,13 @@ static int tegra_dma_pause(struct tegra_dma_channel *tdc) int ret; u32 val; - val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE); + val = tdc_read(tdc, tdc->regs->csre); val |= TEGRA_GPCDMA_CHAN_CSRE_PAUSE; - tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val); + tdc_write(tdc, tdc->regs->csre, val); /* Wait until busy bit is de-asserted */ ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr + - tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS, + tdc->chan_base_offset + tdc->regs->status, val, !(val & TEGRA_GPCDMA_STATUS_BUSY), TEGRA_GPCDMA_BURST_COMPLETE_TIME, @@ -419,9 +433,9 @@ static void tegra_dma_resume(struct tegra_dma_channel *tdc) { u32 val; - val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE); + val = tdc_read(tdc, tdc->regs->csre); val &= ~TEGRA_GPCDMA_CHAN_CSRE_PAUSE; - tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val); + tdc_write(tdc, tdc->regs->csre, val); tdc->status = DMA_IN_PROGRESS; } @@ -456,27 +470,27 @@ static void tegra_dma_disable(struct tegra_dma_channel *tdc) { u32 csr, status; - csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR); + csr = tdc_read(tdc, tdc->regs->csr); /* Disable interrupts */ csr &= ~TEGRA_GPCDMA_CSR_IE_EOC; /* Disable DMA */ csr &= ~TEGRA_GPCDMA_CSR_ENB; - tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr); + tdc_write(tdc, tdc->regs->csr, csr); /* Clear interrupt status if it is there */ - status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); + status = tdc_read(tdc, tdc->regs->status); if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) { dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, status); + tdc_write(tdc, tdc->regs->status, status); } } static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc) { struct tegra_dma_desc *dma_desc = tdc->dma_desc; - struct tegra_dma_channel_regs *ch_regs; + struct tegra_dma_sg_req *sg_req; int ret; u32 val; @@ -488,29 +502,27 @@ static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc) /* Configure next transfer immediately after DMA is busy */ ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr + - tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS, + tdc->chan_base_offset + tdc->regs->status, val, (val & TEGRA_GPCDMA_STATUS_BUSY), 0, TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT); if (ret) return; - ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs; + sg_req = &dma_desc->sg_req[dma_desc->sg_idx]; - tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr); + tdc_write(tdc, tdc->regs->wcount, sg_req->wcount); + tegra_dma_program_addr(tdc, sg_req); /* Start DMA */ - tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, - ch_regs->csr | TEGRA_GPCDMA_CSR_ENB); + tdc_write(tdc, tdc->regs->csr, + sg_req->csr | TEGRA_GPCDMA_CSR_ENB); } static void tegra_dma_start(struct tegra_dma_channel *tdc) { struct tegra_dma_desc *dma_desc = tdc->dma_desc; - struct tegra_dma_channel_regs *ch_regs; + struct tegra_dma_sg_req *sg_req; struct virt_dma_desc *vdesc; if (!dma_desc) { @@ -526,21 +538,19 @@ static void tegra_dma_start(struct tegra_dma_channel *tdc) tegra_dma_resume(tdc); } - ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs; + sg_req = &dma_desc->sg_req[dma_desc->sg_idx]; - tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, 0); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_FIXED_PATTERN, ch_regs->fixed_pattern); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ, ch_regs->mmio_seq); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, ch_regs->mc_seq); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, ch_regs->csr); + tegra_dma_program_addr(tdc, sg_req); + tdc_write(tdc, tdc->regs->wcount, sg_req->wcount); + tdc_write(tdc, tdc->regs->csr, 0); + tdc_write(tdc, tdc->regs->fixed_pattern, sg_req->fixed_pattern); + tdc_write(tdc, tdc->regs->mmio_seq, sg_req->mmio_seq); + tdc_write(tdc, tdc->regs->mc_seq, sg_req->mc_seq); + tdc_write(tdc, tdc->regs->csr, sg_req->csr); /* Start DMA */ - tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, - ch_regs->csr | TEGRA_GPCDMA_CSR_ENB); + tdc_write(tdc, tdc->regs->csr, + sg_req->csr | TEGRA_GPCDMA_CSR_ENB); } static void tegra_dma_xfer_complete(struct tegra_dma_channel *tdc) @@ -601,19 +611,19 @@ static irqreturn_t tegra_dma_isr(int irq, void *dev_id) u32 status; /* Check channel error status register */ - status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS); + status = tdc_read(tdc, tdc->regs->err_status); if (status) { tegra_dma_chan_decode_error(tdc, status); tegra_dma_dump_chan_regs(tdc); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS, 0xFFFFFFFF); + tdc_write(tdc, tdc->regs->err_status, 0xFFFFFFFF); } spin_lock(&tdc->vc.lock); - status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); + status = tdc_read(tdc, tdc->regs->status); if (!(status & TEGRA_GPCDMA_STATUS_ISE_EOC)) goto irq_done; - tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, + tdc_write(tdc, tdc->regs->status, TEGRA_GPCDMA_STATUS_ISE_EOC); if (!dma_desc) @@ -673,10 +683,10 @@ static int tegra_dma_stop_client(struct tegra_dma_channel *tdc) * to stop DMA engine from starting any more bursts for * the given client and wait for in flight bursts to complete */ - csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR); + csr = tdc_read(tdc, tdc->regs->csr); csr &= ~(TEGRA_GPCDMA_CSR_REQ_SEL_MASK); csr |= TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED; - tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr); + tdc_write(tdc, tdc->regs->csr, csr); /* Wait for in flight data transfer to finish */ udelay(TEGRA_GPCDMA_BURST_COMPLETE_TIME); @@ -687,7 +697,7 @@ static int tegra_dma_stop_client(struct tegra_dma_channel *tdc) ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr + tdc->chan_base_offset + - TEGRA_GPCDMA_CHAN_STATUS, + tdc->regs->status, status, !(status & (TEGRA_GPCDMA_STATUS_CHANNEL_TX | TEGRA_GPCDMA_STATUS_CHANNEL_RX)), @@ -739,14 +749,14 @@ static int tegra_dma_get_residual(struct tegra_dma_channel *tdc) unsigned int bytes_xfer, residual; u32 wcount = 0, status; - wcount = tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT); + wcount = tdc_read(tdc, tdc->regs->wxfer); /* * Set wcount = 0 if EOC bit is set. The transfer would have * already completed and the CHAN_XFER_COUNT could have updated * for the next transfer, specifically in case of cyclic transfers. */ - status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); + status = tdc_read(tdc, tdc->regs->status); if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) wcount = 0; @@ -825,6 +835,13 @@ static unsigned int get_burst_size(struct tegra_dma_channel *tdc, * len to calculate the optimum burst size */ burst_byte = burst_size ? burst_size * slave_bw : len; + + /* + * Find the largest burst size that evenly divides the transfer length. + * The hardware requires the transfer length to be a multiple of the + * burst size - partial bursts are not supported. + */ + burst_byte = min(burst_byte, 1U << __ffs(len)); burst_mmio_width = burst_byte / 4; if (burst_mmio_width < TEGRA_GPCDMA_MMIOSEQ_BURST_MIN) @@ -837,7 +854,7 @@ static unsigned int get_burst_size(struct tegra_dma_channel *tdc, static int get_transfer_param(struct tegra_dma_channel *tdc, enum dma_transfer_direction direction, - u32 *apb_addr, + dma_addr_t *apb_addr, u32 *mmio_seq, u32 *csr, unsigned int *burst_size, @@ -893,7 +910,7 @@ tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value, /* Configure default priority weight for the channel */ csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1); - mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); + mc_seq = tdc_read(tdc, tdc->regs->mc_seq); /* retain stream-id and clean rest */ mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK; @@ -915,17 +932,15 @@ tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value, dma_desc->bytes_req = len; dma_desc->sg_count = 1; sg_req = dma_desc->sg_req; + sg_req[0].src = 0; + sg_req[0].dst = dest; - sg_req[0].ch_regs.src_ptr = 0; - sg_req[0].ch_regs.dst_ptr = dest; - sg_req[0].ch_regs.high_addr_ptr = - FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32)); - sg_req[0].ch_regs.fixed_pattern = value; + sg_req[0].fixed_pattern = value; /* Word count reg takes value as (N +1) words */ - sg_req[0].ch_regs.wcount = ((len - 4) >> 2); - sg_req[0].ch_regs.csr = csr; - sg_req[0].ch_regs.mmio_seq = 0; - sg_req[0].ch_regs.mc_seq = mc_seq; + sg_req[0].wcount = ((len - 4) >> 2); + sg_req[0].csr = csr; + sg_req[0].mmio_seq = 0; + sg_req[0].mc_seq = mc_seq; sg_req[0].len = len; dma_desc->cyclic = false; @@ -961,7 +976,7 @@ tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest, /* Configure default priority weight for the channel */ csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1); - mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); + mc_seq = tdc_read(tdc, tdc->regs->mc_seq); /* retain stream-id and clean rest */ mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK) | (TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK); @@ -985,17 +1000,14 @@ tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest, dma_desc->sg_count = 1; sg_req = dma_desc->sg_req; - sg_req[0].ch_regs.src_ptr = src; - sg_req[0].ch_regs.dst_ptr = dest; - sg_req[0].ch_regs.high_addr_ptr = - FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (src >> 32)); - sg_req[0].ch_regs.high_addr_ptr |= - FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32)); + sg_req[0].src = src; + sg_req[0].dst = dest; + /* Word count reg takes value as (N +1) words */ - sg_req[0].ch_regs.wcount = ((len - 4) >> 2); - sg_req[0].ch_regs.csr = csr; - sg_req[0].ch_regs.mmio_seq = 0; - sg_req[0].ch_regs.mc_seq = mc_seq; + sg_req[0].wcount = ((len - 4) >> 2); + sg_req[0].csr = csr; + sg_req[0].mmio_seq = 0; + sg_req[0].mc_seq = mc_seq; sg_req[0].len = len; dma_desc->cyclic = false; @@ -1010,7 +1022,8 @@ tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl, struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count; enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED; - u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0; + u32 csr, mc_seq, mmio_seq = 0; + dma_addr_t apb_ptr = 0; struct tegra_dma_sg_req *sg_req; struct tegra_dma_desc *dma_desc; struct scatterlist *sg; @@ -1049,7 +1062,7 @@ tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl, if (flags & DMA_PREP_INTERRUPT) csr |= TEGRA_GPCDMA_CSR_IE_EOC; - mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); + mc_seq = tdc_read(tdc, tdc->regs->mc_seq); /* retain stream-id and clean rest */ mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK; @@ -1096,25 +1109,21 @@ tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl, dma_desc->bytes_req += len; if (direction == DMA_MEM_TO_DEV) { - sg_req[i].ch_regs.src_ptr = mem; - sg_req[i].ch_regs.dst_ptr = apb_ptr; - sg_req[i].ch_regs.high_addr_ptr = - FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32)); + sg_req[i].src = mem; + sg_req[i].dst = apb_ptr; } else if (direction == DMA_DEV_TO_MEM) { - sg_req[i].ch_regs.src_ptr = apb_ptr; - sg_req[i].ch_regs.dst_ptr = mem; - sg_req[i].ch_regs.high_addr_ptr = - FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32)); + sg_req[i].src = apb_ptr; + sg_req[i].dst = mem; } /* * Word count register takes input in words. Writing a value * of N into word count register means a req of (N+1) words. */ - sg_req[i].ch_regs.wcount = ((len - 4) >> 2); - sg_req[i].ch_regs.csr = csr; - sg_req[i].ch_regs.mmio_seq = mmio_seq; - sg_req[i].ch_regs.mc_seq = mc_seq; + sg_req[i].wcount = ((len - 4) >> 2); + sg_req[i].csr = csr; + sg_req[i].mmio_seq = mmio_seq; + sg_req[i].mc_seq = mc_seq; sg_req[i].len = len; } @@ -1128,7 +1137,8 @@ tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_l unsigned long flags) { enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED; - u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0, burst_size; + u32 csr, mc_seq, mmio_seq = 0, burst_size; + dma_addr_t apb_ptr = 0; unsigned int max_dma_count, len, period_count, i; struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma_desc *dma_desc; @@ -1186,7 +1196,7 @@ tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_l mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1); - mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); + mc_seq = tdc_read(tdc, tdc->regs->mc_seq); /* retain stream-id and clean rest */ mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK; @@ -1217,24 +1227,20 @@ tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_l for (i = 0; i < period_count; i++) { mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len); if (direction == DMA_MEM_TO_DEV) { - sg_req[i].ch_regs.src_ptr = mem; - sg_req[i].ch_regs.dst_ptr = apb_ptr; - sg_req[i].ch_regs.high_addr_ptr = - FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32)); + sg_req[i].src = mem; + sg_req[i].dst = apb_ptr; } else if (direction == DMA_DEV_TO_MEM) { - sg_req[i].ch_regs.src_ptr = apb_ptr; - sg_req[i].ch_regs.dst_ptr = mem; - sg_req[i].ch_regs.high_addr_ptr = - FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32)); + sg_req[i].src = apb_ptr; + sg_req[i].dst = mem; } /* * Word count register takes input in words. Writing a value * of N into word count register means a req of (N+1) words. */ - sg_req[i].ch_regs.wcount = ((len - 4) >> 2); - sg_req[i].ch_regs.csr = csr; - sg_req[i].ch_regs.mmio_seq = mmio_seq; - sg_req[i].ch_regs.mc_seq = mc_seq; + sg_req[i].wcount = ((len - 4) >> 2); + sg_req[i].csr = csr; + sg_req[i].mmio_seq = mmio_seq; + sg_req[i].mc_seq = mc_seq; sg_req[i].len = len; mem += len; @@ -1304,27 +1310,76 @@ static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, return chan; } +static const struct tegra_dma_channel_regs tegra186_reg_offsets = { + .csr = 0x0, + .status = 0x4, + .csre = 0x8, + .src = 0xc, + .dst = 0x10, + .high_addr = 0x14, + .mc_seq = 0x18, + .mmio_seq = 0x1c, + .wcount = 0x20, + .wxfer = 0x24, + .wstatus = 0x28, + .err_status = 0x30, + .fixed_pattern = 0x34, +}; + +static const struct tegra_dma_channel_regs tegra264_reg_offsets = { + .csr = 0x0, + .status = 0x4, + .csre = 0x8, + .src = 0xc, + .dst = 0x10, + .src_high = 0x14, + .dst_high = 0x18, + .mc_seq = 0x1c, + .mmio_seq = 0x20, + .wcount = 0x24, + .wxfer = 0x28, + .wstatus = 0x2c, + .err_status = 0x34, + .fixed_pattern = 0x38, +}; + static const struct tegra_dma_chip_data tegra186_dma_chip_data = { .nr_channels = 32, + .addr_bits = 39, .channel_reg_size = SZ_64K, .max_dma_count = SZ_1G, .hw_support_pause = false, + .channel_regs = &tegra186_reg_offsets, .terminate = tegra_dma_stop_client, }; static const struct tegra_dma_chip_data tegra194_dma_chip_data = { .nr_channels = 32, + .addr_bits = 39, .channel_reg_size = SZ_64K, .max_dma_count = SZ_1G, .hw_support_pause = true, + .channel_regs = &tegra186_reg_offsets, .terminate = tegra_dma_pause, }; static const struct tegra_dma_chip_data tegra234_dma_chip_data = { .nr_channels = 32, + .addr_bits = 39, .channel_reg_size = SZ_64K, .max_dma_count = SZ_1G, .hw_support_pause = true, + .channel_regs = &tegra186_reg_offsets, + .terminate = tegra_dma_pause_noerr, +}; + +static const struct tegra_dma_chip_data tegra264_dma_chip_data = { + .nr_channels = 32, + .addr_bits = 41, + .channel_reg_size = SZ_64K, + .max_dma_count = SZ_1G, + .hw_support_pause = true, + .channel_regs = &tegra264_reg_offsets, .terminate = tegra_dma_pause_noerr, }; @@ -1339,13 +1394,16 @@ static const struct of_device_id tegra_dma_of_match[] = { .compatible = "nvidia,tegra234-gpcdma", .data = &tegra234_dma_chip_data, }, { + .compatible = "nvidia,tegra264-gpcdma", + .data = &tegra264_dma_chip_data, + }, { }, }; MODULE_DEVICE_TABLE(of, tegra_dma_of_match); static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id) { - unsigned int reg_val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); + unsigned int reg_val = tdc_read(tdc, tdc->regs->mc_seq); reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK); reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK); @@ -1353,16 +1411,20 @@ static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id) reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK, stream_id); reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK, stream_id); - tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, reg_val); + tdc_write(tdc, tdc->regs->mc_seq, reg_val); return 0; } static int tegra_dma_probe(struct platform_device *pdev) { const struct tegra_dma_chip_data *cdata = NULL; + struct tegra_dma_channel *tdc; + struct tegra_dma *tdma; + struct dma_chan *chan; + struct device *chdev; + bool use_iommu_map = false; unsigned int i; u32 stream_id; - struct tegra_dma *tdma; int ret; cdata = of_device_get_match_data(&pdev->dev); @@ -1381,18 +1443,19 @@ static int tegra_dma_probe(struct platform_device *pdev) if (IS_ERR(tdma->base_addr)) return PTR_ERR(tdma->base_addr); - tdma->rst = devm_reset_control_get_exclusive(&pdev->dev, "gpcdma"); + tdma->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, "gpcdma"); if (IS_ERR(tdma->rst)) { return dev_err_probe(&pdev->dev, PTR_ERR(tdma->rst), - "Missing controller reset\n"); + "Failed to get controller reset\n"); } reset_control_reset(tdma->rst); tdma->dma_dev.dev = &pdev->dev; - if (!tegra_dev_iommu_get_stream_id(&pdev->dev, &stream_id)) { - dev_err(&pdev->dev, "Missing iommu stream-id\n"); - return -EINVAL; + use_iommu_map = of_property_present(pdev->dev.of_node, "iommu-map"); + if (!use_iommu_map) { + if (!tegra_dev_iommu_get_stream_id(&pdev->dev, &stream_id)) + return dev_err_probe(&pdev->dev, -EINVAL, "Missing iommu stream-id\n"); } ret = device_property_read_u32(&pdev->dev, "dma-channel-mask", @@ -1404,9 +1467,10 @@ static int tegra_dma_probe(struct platform_device *pdev) tdma->chan_mask = TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK; } + /* Initialize vchan for each channel and populate the channels list */ INIT_LIST_HEAD(&tdma->dma_dev.channels); for (i = 0; i < cdata->nr_channels; i++) { - struct tegra_dma_channel *tdc = &tdma->channels[i]; + tdc = &tdma->channels[i]; /* Check for channel mask */ if (!(tdma->chan_mask & BIT(i))) @@ -1419,18 +1483,17 @@ static int tegra_dma_probe(struct platform_device *pdev) tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET + i * cdata->channel_reg_size; snprintf(tdc->name, sizeof(tdc->name), "gpcdma.%d", i); + tdc->regs = cdata->channel_regs; tdc->tdma = tdma; tdc->id = i; tdc->slave_id = -1; vchan_init(&tdc->vc, &tdma->dma_dev); tdc->vc.desc_free = tegra_dma_desc_free; - - /* program stream-id for this channel */ - tegra_dma_program_sid(tdc, stream_id); - tdc->stream_id = stream_id; } + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(cdata->addr_bits)); + dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); dma_cap_set(DMA_MEMCPY, tdma->dma_dev.cap_mask); @@ -1460,37 +1523,59 @@ static int tegra_dma_probe(struct platform_device *pdev) tdma->dma_dev.device_synchronize = tegra_dma_chan_synchronize; tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; - ret = dma_async_device_register(&tdma->dma_dev); + /* Register the DMA device and the channels */ + ret = dmaenginem_async_device_register(&tdma->dma_dev); if (ret < 0) { dev_err_probe(&pdev->dev, ret, "GPC DMA driver registration failed\n"); return ret; } - ret = of_dma_controller_register(pdev->dev.of_node, - tegra_dma_of_xlate, tdma); + /* + * Configure stream ID for each channel from the channels registered + * above. This is done in a separate iteration to ensure that only + * the channels available and registered for the DMA device are used. + */ + list_for_each_entry(chan, &tdma->dma_dev.channels, device_node) { + chdev = &chan->dev->device; + tdc = to_tegra_dma_chan(chan); + + if (use_iommu_map) { + chdev->bus = pdev->dev.bus; + dma_coerce_mask_and_coherent(chdev, DMA_BIT_MASK(cdata->addr_bits)); + + ret = of_dma_configure_id(chdev, pdev->dev.of_node, + true, &tdc->id); + if (ret) + return dev_err_probe(chdev, ret, + "Failed to configure IOMMU for channel %d\n", tdc->id); + + if (!tegra_dev_iommu_get_stream_id(chdev, &stream_id)) + return dev_err_probe(chdev, -EINVAL, + "Failed to get stream ID for channel %d\n", tdc->id); + + chan->dev->chan_dma_dev = true; + } + + /* program stream-id for this channel */ + tegra_dma_program_sid(tdc, stream_id); + tdc->stream_id = stream_id; + } + + ret = devm_of_dma_controller_register(&pdev->dev, pdev->dev.of_node, + tegra_dma_of_xlate, tdma); if (ret < 0) { dev_err_probe(&pdev->dev, ret, "GPC DMA OF registration failed\n"); - - dma_async_device_unregister(&tdma->dma_dev); return ret; } - dev_info(&pdev->dev, "GPC DMA driver register %lu channels\n", + dev_info(&pdev->dev, "GPC DMA driver registered %lu channels\n", hweight_long(tdma->chan_mask)); return 0; } -static void tegra_dma_remove(struct platform_device *pdev) -{ - struct tegra_dma *tdma = platform_get_drvdata(pdev); - - of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&tdma->dma_dev); -} - static int __maybe_unused tegra_dma_pm_suspend(struct device *dev) { struct tegra_dma *tdma = dev_get_drvdata(dev); @@ -1541,7 +1626,6 @@ static struct platform_driver tegra_dma_driver = { .of_match_table = tegra_dma_of_match, }, .probe = tegra_dma_probe, - .remove = tegra_dma_remove, }; module_platform_driver(tegra_dma_driver); diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index 14e0c408ed1e..21a381d022cf 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -335,8 +335,16 @@ static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc, struct tegra_adma *tdma = tdc->tdma; unsigned int sreq_index = tdc->sreq_index; - if (tdc->sreq_reserved) - return tdc->sreq_dir == direction ? 0 : -EINVAL; + if (tdc->sreq_reserved) { + if (tdc->sreq_dir != direction) { + dev_err(tdma->dev, + "DMA request direction mismatch: reserved=%s, requested=%s\n", + dmaengine_get_direction_text(tdc->sreq_dir), + dmaengine_get_direction_text(direction)); + return -EINVAL; + } + return 0; + } if (sreq_index > tdma->cdata->ch_req_max) { dev_err(tdma->dev, "invalid DMA request\n"); @@ -665,8 +673,11 @@ static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc, const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata; unsigned int burst_size, adma_dir, fifo_size_shift; - if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS) + if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS) { + dev_err(tdc2dev(tdc), "invalid DMA periods %zu (max %u)\n", + desc->num_periods, ADMA_CH_CONFIG_MAX_BUFS); return -EINVAL; + } switch (direction) { case DMA_MEM_TO_DEV: @@ -1029,8 +1040,8 @@ static int tegra_adma_probe(struct platform_device *pdev) cdata = of_device_get_match_data(&pdev->dev); if (!cdata) { - dev_err(&pdev->dev, "device match data not found\n"); - return -ENODEV; + return dev_err_probe(&pdev->dev, -ENODEV, + "device match data not found\n"); } tdma = devm_kzalloc(&pdev->dev, @@ -1056,7 +1067,8 @@ static int tegra_adma_probe(struct platform_device *pdev) unsigned int ch_base_offset; if (res_page->start < res_base->start) - return -EINVAL; + return dev_err_probe(&pdev->dev, -EINVAL, + "invalid page/global resource order\n"); page_offset = res_page->start - res_base->start; ch_base_offset = cdata->ch_base_offset; if (!ch_base_offset) @@ -1064,7 +1076,9 @@ static int tegra_adma_probe(struct platform_device *pdev) page_no = div_u64(page_offset, ch_base_offset); if (!page_no || page_no > INT_MAX) - return -EINVAL; + return dev_err_probe(&pdev->dev, -EINVAL, + "invalid page number %llu\n", + (unsigned long long)page_no); tdma->ch_page_no = page_no - 1; tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); @@ -1073,22 +1087,17 @@ static int tegra_adma_probe(struct platform_device *pdev) } } else { /* If no 'page' property found, then reg DT binding would be legacy */ - res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res_base) { - tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); - if (IS_ERR(tdma->base_addr)) - return PTR_ERR(tdma->base_addr); - } else { - return -ENODEV; - } + tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(tdma->base_addr)) + return PTR_ERR(tdma->base_addr); tdma->ch_base_addr = tdma->base_addr + cdata->ch_base_offset; } tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio"); if (IS_ERR(tdma->ahub_clk)) { - dev_err(&pdev->dev, "Error: Missing ahub controller clock\n"); - return PTR_ERR(tdma->ahub_clk); + return dev_err_probe(&pdev->dev, PTR_ERR(tdma->ahub_clk), + "failed to get ahub clock\n"); } tdma->dma_chan_mask = devm_kzalloc(&pdev->dev, @@ -1104,8 +1113,8 @@ static int tegra_adma_probe(struct platform_device *pdev) (u32 *)tdma->dma_chan_mask, BITS_TO_U32(tdma->nr_channels)); if (ret < 0 && (ret != -EINVAL)) { - dev_err(&pdev->dev, "dma-channel-mask is not complete.\n"); - return ret; + return dev_err_probe(&pdev->dev, ret, + "dma-channel-mask is not complete.\n"); } INIT_LIST_HEAD(&tdma->dma_dev.channels); @@ -1127,11 +1136,13 @@ static int tegra_adma_probe(struct platform_device *pdev) cdata->global_ch_config_base + (4 * i); } - tdc->irq = of_irq_get(pdev->dev.of_node, i); - if (tdc->irq <= 0) { - ret = tdc->irq ?: -ENXIO; + ret = of_irq_get(pdev->dev.of_node, i); + if (ret <= 0) { + ret = dev_err_probe(&pdev->dev, ret ?: -ENXIO, + "failed to get IRQ for channel %d\n", i); goto irq_dispose; } + tdc->irq = ret; vchan_init(&tdc->vc, &tdma->dma_dev); tdc->vc.desc_free = tegra_adma_desc_free; @@ -1141,12 +1152,18 @@ static int tegra_adma_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); ret = pm_runtime_resume_and_get(&pdev->dev); - if (ret < 0) + if (ret < 0) { + ret = dev_err_probe(&pdev->dev, ret, + "runtime PM resume failed\n"); goto rpm_disable; + } ret = tegra_adma_init(tdma); - if (ret) + if (ret) { + ret = dev_err_probe(&pdev->dev, ret, + "failed to initialize ADMA\n"); goto rpm_put; + } dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); @@ -1172,14 +1189,16 @@ static int tegra_adma_probe(struct platform_device *pdev) ret = dma_async_device_register(&tdma->dma_dev); if (ret < 0) { - dev_err(&pdev->dev, "ADMA registration failed: %d\n", ret); + ret = dev_err_probe(&pdev->dev, ret, + "ADMA registration failed\n"); goto rpm_put; } ret = of_dma_controller_register(pdev->dev.of_node, tegra_dma_of_xlate, tdma); if (ret < 0) { - dev_err(&pdev->dev, "ADMA OF registration failed %d\n", ret); + ret = dev_err_probe(&pdev->dev, ret, + "ADMA OF registration failed\n"); goto dma_remove; } diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index f87d244cc2d6..70eaf7ee57e6 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -312,7 +312,7 @@ k3_udma_glue_request_tx_chn_common(struct device *dev, if (xudma_is_pktdma(tx_chn->common.udmax)) { /* prepare the channel device as coherent */ - tx_chn->common.chan_dev.dma_coherent = true; + dev_set_dma_coherent(&tx_chn->common.chan_dev); dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev, DMA_BIT_MASK(48)); } @@ -1003,7 +1003,7 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, if (xudma_is_pktdma(rx_chn->common.udmax)) { /* prepare the channel device as coherent */ - rx_chn->common.chan_dev.dma_coherent = true; + dev_set_dma_coherent(&rx_chn->common.chan_dev); dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, DMA_BIT_MASK(48)); } @@ -1104,7 +1104,7 @@ k3_udma_glue_request_remote_rx_chn_common(struct k3_udma_glue_rx_channel *rx_chn if (xudma_is_pktdma(rx_chn->common.udmax)) { /* prepare the channel device as coherent */ - rx_chn->common.chan_dev.dma_coherent = true; + dev_set_dma_coherent(&rx_chn->common.chan_dev); dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, DMA_BIT_MASK(48)); rx_chn->single_fdq = false; @@ -1243,8 +1243,9 @@ void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn) rx_chn->psil_paired = false; } - for (i = 0; i < rx_chn->flow_num; i++) - k3_udma_glue_release_rx_flow(rx_chn, i); + if (rx_chn->flows) + for (i = 0; i < rx_chn->flow_num; i++) + k3_udma_glue_release_rx_flow(rx_chn, i); if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base)) xudma_free_gp_rflow_range(rx_chn->common.udmax, diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index c964ebfcf3b6..fb21e0df5ab7 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -428,18 +428,18 @@ static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel) /* No special handling for the channel */ chan->dev->chan_dma_dev = false; - chan_dev->dma_coherent = false; + dev_clear_dma_coherent(chan_dev); chan_dev->dma_parms = NULL; } else if (asel == 14 || asel == 15) { chan->dev->chan_dma_dev = true; - chan_dev->dma_coherent = true; + dev_set_dma_coherent(chan_dev); dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48)); chan_dev->dma_parms = chan_dev->parent->dma_parms; } else { dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel); - chan_dev->dma_coherent = false; + dev_clear_dma_coherent(chan_dev); chan_dev->dma_parms = NULL; } } @@ -3408,7 +3408,7 @@ static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc, return 0; } -static struct dma_descriptor_metadata_ops metadata_ops = { +static const struct dma_descriptor_metadata_ops metadata_ops = { .attach = udma_attach_metadata, .get_ptr = udma_get_metadata_ptr, .set_len = udma_set_metadata_len, diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c index f64624ea44ad..fa1173e49900 100644 --- a/drivers/dma/xgene-dma.c +++ b/drivers/dma/xgene-dma.c @@ -18,7 +18,6 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/irq.h> -#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_device.h> diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index d02a4dac2291..8d4a5d14e8db 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -20,7 +20,6 @@ * user interrupt wires that generate interrupts to the host. */ -#include <linux/mod_devicetable.h> #include <linux/bitfield.h> #include <linux/dmapool.h> #include <linux/regmap.h> @@ -61,6 +60,8 @@ struct xdma_desc_block { * @dir: Transferring direction of the channel * @cfg: Transferring config of the channel * @irq: IRQ assigned to the channel + * @last_interrupt: task for comppleting last interrupt + * @stop_requested: stop request flag */ struct xdma_chan { struct virt_dma_chan vchan; @@ -1234,8 +1235,8 @@ static int xdma_probe(struct platform_device *pdev) xdev->rmap = devm_regmap_init_mmio(&pdev->dev, reg_base, &xdma_regmap_config); - if (!xdev->rmap) { - xdma_err(xdev, "config regmap failed: %d", ret); + if (IS_ERR(xdev->rmap)) { + xdma_err(xdev, "config regmap failed: %pe", xdev->rmap); goto failed; } INIT_LIST_HEAD(&xdev->dma_dev.channels); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index b53292e02448..cffe7c6fa640 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -165,8 +165,10 @@ #define XILINX_DMA_FLUSH_MM2S 2 #define XILINX_DMA_FLUSH_BOTH 1 -/* Delay loop counter to prevent hardware failure */ -#define XILINX_DMA_LOOP_COUNT 1000000 +/* Timeout for polling various registers */ +#define XILINX_DMA_POLL_TIMEOUT_US 1000000 +/* Delay between polls (avoid a delay of 0 to prevent CPU stalls) */ +#define XILINX_DMA_POLL_DELAY_US 10 /* AXI DMA Specific Registers/Offsets */ #define XILINX_DMA_REG_SRCDSTADDR 0x18 @@ -653,7 +655,7 @@ static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx, return seg->hw.app; } -static struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = { +static const struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = { .get_ptr = xilinx_dma_get_metadata_ptr, }; @@ -754,15 +756,25 @@ xilinx_aximcdma_alloc_tx_segment(struct xilinx_dma_chan *chan) return segment; } -static void xilinx_dma_clean_hw_desc(struct xilinx_axidma_desc_hw *hw) +static void xilinx_dma_clean_hw_desc(struct xilinx_dma_chan *chan, + struct xilinx_axidma_tx_segment *segment) { - u32 next_desc = hw->next_desc; - u32 next_desc_msb = hw->next_desc_msb; + dma_addr_t next; + u32 i; - memset(hw, 0, sizeof(struct xilinx_axidma_desc_hw)); + /* + * Restore the buffer descriptor's next descriptor pointer to the value + * set up in xilinx_dma_alloc_chan_resources(). Otherwise using the DMA + * in cyclic mode leaves the next descriptor pointer altered and + * prevents subsequent non-cyclic transfers. + */ + i = segment - chan->seg_v; + next = chan->seg_p + + sizeof(*chan->seg_v) * ((i + 1) % XILINX_DMA_NUM_DESCS); - hw->next_desc = next_desc; - hw->next_desc_msb = next_desc_msb; + memset(&segment->hw, 0, sizeof(segment->hw)); + segment->hw.next_desc = lower_32_bits(next); + segment->hw.next_desc_msb = upper_32_bits(next); } static void xilinx_mcdma_clean_hw_desc(struct xilinx_aximcdma_desc_hw *hw) @@ -784,7 +796,7 @@ static void xilinx_mcdma_clean_hw_desc(struct xilinx_aximcdma_desc_hw *hw) static void xilinx_dma_free_tx_segment(struct xilinx_dma_chan *chan, struct xilinx_axidma_tx_segment *segment) { - xilinx_dma_clean_hw_desc(&segment->hw); + xilinx_dma_clean_hw_desc(chan, segment); list_add_tail(&segment->node, &chan->free_seg_list); } @@ -918,9 +930,9 @@ static void xilinx_dma_free_descriptors(struct xilinx_dma_chan *chan) spin_lock_irqsave(&chan->lock, flags); - xilinx_dma_free_desc_list(chan, &chan->pending_list); xilinx_dma_free_desc_list(chan, &chan->done_list); xilinx_dma_free_desc_list(chan, &chan->active_list); + xilinx_dma_free_desc_list(chan, &chan->pending_list); spin_unlock_irqrestore(&chan->lock, flags); } @@ -997,16 +1009,16 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan, struct xilinx_cdma_tx_segment, node); cdma_hw = &cdma_seg->hw; - residue += (cdma_hw->control - cdma_hw->status) & - chan->xdev->max_buffer_len; + residue += (cdma_hw->control & chan->xdev->max_buffer_len) - + (cdma_hw->status & chan->xdev->max_buffer_len); } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { axidma_seg = list_entry(entry, struct xilinx_axidma_tx_segment, node); axidma_hw = &axidma_seg->hw; - residue += (axidma_hw->control - axidma_hw->status) & - chan->xdev->max_buffer_len; + residue += (axidma_hw->control & chan->xdev->max_buffer_len) - + (axidma_hw->status & chan->xdev->max_buffer_len); } else { aximcdma_seg = list_entry(entry, @@ -1014,8 +1026,8 @@ static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan, node); aximcdma_hw = &aximcdma_seg->hw; residue += - (aximcdma_hw->control - aximcdma_hw->status) & - chan->xdev->max_buffer_len; + (aximcdma_hw->control & chan->xdev->max_buffer_len) - + (aximcdma_hw->status & chan->xdev->max_buffer_len); } } @@ -1235,14 +1247,6 @@ static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan) dma_cookie_init(dchan); - if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { - /* For AXI DMA resetting once channel will reset the - * other channel as well so enable the interrupts here. - */ - dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, - XILINX_DMA_DMAXR_ALL_IRQ_MASK); - } - if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg) dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_CDMA_CR_SGMODE); @@ -1332,8 +1336,9 @@ static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan) /* Wait for the hardware to halt */ return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - val & XILINX_DMA_DMASR_HALTED, 0, - XILINX_DMA_LOOP_COUNT); + val & XILINX_DMA_DMASR_HALTED, + XILINX_DMA_POLL_DELAY_US, + XILINX_DMA_POLL_TIMEOUT_US); } /** @@ -1347,8 +1352,9 @@ static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan) u32 val; return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - val & XILINX_DMA_DMASR_IDLE, 0, - XILINX_DMA_LOOP_COUNT); + val & XILINX_DMA_DMASR_IDLE, + XILINX_DMA_POLL_DELAY_US, + XILINX_DMA_POLL_TIMEOUT_US); } /** @@ -1364,8 +1370,9 @@ static void xilinx_dma_start(struct xilinx_dma_chan *chan) /* Wait for the hardware to start */ err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - !(val & XILINX_DMA_DMASR_HALTED), 0, - XILINX_DMA_LOOP_COUNT); + !(val & XILINX_DMA_DMASR_HALTED), + XILINX_DMA_POLL_DELAY_US, + XILINX_DMA_POLL_TIMEOUT_US); if (err) { dev_err(chan->dev, "Cannot start channel %p: %x\n", @@ -1564,10 +1571,38 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) if (chan->err) return; - if (list_empty(&chan->pending_list)) + if (list_empty(&chan->pending_list)) { + if (chan->cyclic) { + struct xilinx_dma_tx_descriptor *desc; + struct list_head *entry; + + desc = list_last_entry(&chan->done_list, + struct xilinx_dma_tx_descriptor, node); + list_for_each(entry, &desc->segments) { + struct xilinx_axidma_tx_segment *axidma_seg; + struct xilinx_axidma_desc_hw *axidma_hw; + axidma_seg = list_entry(entry, + struct xilinx_axidma_tx_segment, + node); + axidma_hw = &axidma_seg->hw; + axidma_hw->status = 0; + } + + list_splice_tail_init(&chan->done_list, &chan->active_list); + chan->desc_pendingcount = 0; + chan->idle = false; + } return; + } - if (!chan->idle) + /* + * Direct (non-SG) mode has no descriptor queue: writing the BTT + * register launches a transfer immediately, so a new transfer must + * not be programmed while one is in flight. Keep such transfers + * serialized. SG mode supports chaining onto a running transfer via + * tail-pointer extension, so it is allowed to proceed when busy. + */ + if (!chan->has_sg && !chan->idle) return; head_desc = list_first_entry(&chan->pending_list, @@ -1583,17 +1618,18 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) reg &= ~XILINX_DMA_CR_COALESCE_MAX; reg |= chan->desc_pendingcount << XILINX_DMA_CR_COALESCE_SHIFT; - dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); } - if (chan->has_sg) + if (chan->has_sg && list_empty(&chan->active_list)) xilinx_write(chan, XILINX_DMA_REG_CURDESC, head_desc->async_tx.phys); reg &= ~XILINX_DMA_CR_DELAY_MAX; reg |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT; + reg |= XILINX_DMA_DMAXR_ALL_IRQ_MASK; dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); - xilinx_dma_start(chan); + if (chan->idle) + xilinx_dma_start(chan); if (chan->err) return; @@ -1646,9 +1682,6 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan) if (chan->err) return; - if (!chan->idle) - return; - if (list_empty(&chan->pending_list)) return; @@ -1671,8 +1704,9 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan) dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg); /* Program current descriptor */ - xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest), - head_desc->async_tx.phys); + if (chan->has_sg && list_empty(&chan->active_list)) + xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest), + head_desc->async_tx.phys); /* Program channel enable register */ reg = dma_ctrl_read(chan, XILINX_MCDMA_CHEN_OFFSET); @@ -1684,7 +1718,8 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan) reg |= XILINX_MCDMA_CR_RUNSTOP_MASK; dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg); - xilinx_dma_start(chan); + if (chan->idle) + xilinx_dma_start(chan); if (chan->err) return; @@ -1780,8 +1815,9 @@ static int xilinx_dma_reset(struct xilinx_dma_chan *chan) /* Wait for the hardware to finish reset */ err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp, - !(tmp & XILINX_DMA_DMACR_RESET), 0, - XILINX_DMA_LOOP_COUNT); + !(tmp & XILINX_DMA_DMACR_RESET), + XILINX_DMA_POLL_DELAY_US, + XILINX_DMA_POLL_TIMEOUT_US); if (err) { dev_err(chan->dev, "reset timeout, cr %x, sr %x\n", @@ -1879,8 +1915,10 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) if (status & XILINX_MCDMA_IRQ_IOC_MASK) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); - chan->idle = true; - chan->start_transfer(chan); + if (list_empty(&chan->active_list)) { + chan->idle = true; + chan->start_transfer(chan); + } spin_unlock(&chan->lock); } @@ -1936,8 +1974,10 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) XILINX_DMA_DMASR_DLY_CNT_IRQ)) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); - chan->idle = true; - chan->start_transfer(chan); + if (list_empty(&chan->active_list)) { + chan->idle = true; + chan->start_transfer(chan); + } spin_unlock(&chan->lock); } @@ -3024,7 +3064,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, return -EINVAL; } - xdev->common.directions |= chan->direction; + xdev->common.directions |= BIT(chan->direction); /* Request the interrupt */ chan->irq = of_irq_get(node, chan->tdest); @@ -3180,7 +3220,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) = axivdma_clk_init; struct device_node *node = pdev->dev.of_node; struct xilinx_dma_device *xdev; - struct device_node *child, *np = pdev->dev.of_node; + struct device_node *np = pdev->dev.of_node; u32 num_frames, addr_width = XILINX_DMA_DFAULT_ADDRWIDTH, len_width; int i, err; @@ -3320,12 +3360,10 @@ static int xilinx_dma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, xdev); /* Initialize the channels */ - for_each_child_of_node(node, child) { + for_each_child_of_node_scoped(node, child) { err = xilinx_dma_child_probe(xdev, child); - if (err < 0) { - of_node_put(child); + if (err < 0) goto error; - } } if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c index f6a812e49ddc..2b03584102ac 100644 --- a/drivers/dma/xilinx/zynqmp_dma.c +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -1159,8 +1159,6 @@ err_disable_pm: /** * zynqmp_dma_remove - Driver remove function * @pdev: Pointer to the platform_device structure - * - * Return: Always '0' */ static void zynqmp_dma_remove(struct platform_device *pdev) { @@ -1170,9 +1168,9 @@ static void zynqmp_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&zdev->common); zynqmp_dma_chan_remove(zdev->chan); - if (pm_runtime_active(zdev->dev)) - zynqmp_dma_runtime_suspend(zdev->dev); pm_runtime_disable(zdev->dev); + if (!pm_runtime_status_suspended(zdev->dev)) + zynqmp_dma_runtime_suspend(zdev->dev); } static const struct of_device_id zynqmp_dma_of_match[] = { |