diff options
| -rw-r--r-- | Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt | 2 | ||||
| -rw-r--r-- | drivers/dma/imx-sdma.c | 899 | ||||
| -rw-r--r-- | include/linux/dmaengine.h | 2 | ||||
| -rw-r--r-- | include/linux/platform_data/dma-imx.h | 36 |
4 files changed, 743 insertions, 196 deletions
diff --git a/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt b/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt index 12c316ff4834..e7b5f734a2d8 100644 --- a/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt +++ b/Documentation/devicetree/bindings/dma/fsl-imx-sdma.txt @@ -9,6 +9,7 @@ Required properties: "fsl,imx53-sdma" "fsl,imx6q-sdma" "fsl,imx7d-sdma" + "fsl,imx6sx-sdma" "fsl,imx6ul-sdma" "fsl,imx8mq-sdma" "fsl,imx8mm-sdma" @@ -55,6 +56,7 @@ The full ID of peripheral types can be found below. 22 SSI Dual FIFO (needs firmware ver >= 2) 23 Shared ASRC 24 SAI + 25 HDMI Audio The third cell specifies the transfer priority as below. diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index cacc725ca545..2659cd2c5c4c 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -23,10 +23,12 @@ #include <linux/semaphore.h> #include <linux/spinlock.h> #include <linux/device.h> +#include <linux/genalloc.h> #include <linux/dma-mapping.h> #include <linux/firmware.h> #include <linux/slab.h> #include <linux/platform_device.h> +#include <linux/pm_runtime.h> #include <linux/dmaengine.h> #include <linux/of.h> #include <linux/of_address.h> @@ -73,6 +75,9 @@ #define SDMA_CHNENBL0_IMX35 0x200 #define SDMA_CHNENBL0_IMX31 0x080 #define SDMA_CHNPRI_0 0x100 +#define SDMA_DONE0_CONFIG 0x1000 +#define SDMA_DONE0_CONFIG_DONE_SEL 0x7 +#define SDMA_DONE0_CONFIG_DONE_DIS 0x6 /* * Buffer descriptor status values. @@ -167,6 +172,8 @@ #define SDMA_WATERMARK_LEVEL_SPDIF BIT(10) #define SDMA_WATERMARK_LEVEL_SP BIT(11) #define SDMA_WATERMARK_LEVEL_DP BIT(12) +#define SDMA_WATERMARK_LEVEL_SD BIT(13) +#define SDMA_WATERMARK_LEVEL_DD BIT(14) #define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16) #define SDMA_WATERMARK_LEVEL_LWE BIT(28) #define SDMA_WATERMARK_LEVEL_HWE BIT(29) @@ -174,6 +181,7 @@ #define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) #define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \ @@ -234,11 +242,19 @@ struct sdma_script_start_addrs { s32 sai_2_mcu_addr; s32 uart_2_mcu_addr; s32 uartsh_2_mcu_addr; + s32 i2c_2_mcu_addr; + s32 mcu_2_i2c_addr; /* End of v3 array */ s32 mcu_2_zqspi_addr; /* End of v4 array */ }; +#define SDMA_WATERMARK_LEVEL_FIFOS_OFF 12 +#define SDMA_WATERMARK_LEVEL_FIFO_OFF_OFF 16 +#define SDMA_WATERMARK_LEVEL_SW_DONE BIT(23) +#define SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF 24 +#define SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO_OFF 28 + /* * Mode/Count of data node descriptors - IPCv2 */ @@ -422,8 +438,9 @@ struct sdma_channel { struct sdma_desc *desc; struct sdma_engine *sdma; unsigned int channel; - enum dma_transfer_direction direction; + enum dma_transfer_direction direction; struct dma_slave_config slave_config; + struct sdma_audio_config *audio_config; enum sdma_peripheral_type peripheral_type; unsigned int event_id0; unsigned int event_id1; @@ -441,6 +458,7 @@ struct sdma_channel { struct work_struct terminate_worker; struct list_head terminated; bool is_ram_script; + int prio; }; #define IMX_DMA_SG_LOOP BIT(0) @@ -450,6 +468,15 @@ struct sdma_channel { #define MXC_SDMA_MIN_PRIORITY 1 #define MXC_SDMA_MAX_PRIORITY 7 +/* + * 0x78(SDMA_XTRIG_CONF2+4)~0x100(SDMA_CHNPRI_O) registers are reserved and + * can't be accessed. Skip these register touch in suspend/resume. Also below + * two macros are only used on i.mx6sx. + */ +#define MXC_SDMA_RESERVED_REG (SDMA_CHNPRI_0 - SDMA_XTRIG_CONF2 - 4) +#define MXC_SDMA_SAVED_REG_NUM (((SDMA_CHNENBL0_IMX35 + 4 * 48) - \ + MXC_SDMA_RESERVED_REG) / 4) + #define SDMA_FIRMWARE_MAGIC 0x414d4453 /** @@ -488,15 +515,22 @@ struct sdma_driver_data { * https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf */ bool ecspi_fixed; + bool has_done0; + bool pm_runtime; }; struct sdma_engine { struct device *dev; struct sdma_channel channel[MAX_DMA_CHANNELS]; struct sdma_channel_control *channel_control; + u32 save_regs[MXC_SDMA_SAVED_REG_NUM]; + u32 save_done0_regs[2]; + const char *fw_name; void __iomem *regs; struct sdma_context_data *context; dma_addr_t context_phys; + dma_addr_t ccb_phys; + bool is_on; struct dma_device dma_device; struct clk *clk_ipg; struct clk *clk_ahb; @@ -512,6 +546,10 @@ struct sdma_engine { /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/ bool clk_ratio; bool fw_loaded; + struct gen_pool *iram_pool; + u32 fw_fail; + u8 *fw_data; + unsigned short ram_code_start; }; static int sdma_config_write(struct dma_chan *chan, @@ -608,6 +646,12 @@ static struct sdma_driver_data sdma_imx6q = { .script_addrs = &sdma_script_imx6q, }; +static struct sdma_driver_data sdma_imx6sx = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx6q, +}; + static struct sdma_driver_data sdma_imx6ul = { .chnenbl0 = SDMA_CHNENBL0_IMX35, .num_events = 48, @@ -640,6 +684,16 @@ static struct sdma_driver_data sdma_imx8mq = { .check_ratio = 1, }; +static struct sdma_driver_data sdma_imx8mp = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx7d, + .check_ratio = 1, + .ecspi_fixed = true, + .has_done0 = true, + .pm_runtime = true, +}; + static const struct of_device_id sdma_dt_ids[] = { { .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, }, { .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, }, @@ -648,8 +702,10 @@ static const struct of_device_id sdma_dt_ids[] = { { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, }, { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, }, { .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, }, + { .compatible = "fsl,imx6sx-sdma", .data = &sdma_imx6sx, }, { .compatible = "fsl,imx6ul-sdma", .data = &sdma_imx6ul, }, { .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, }, + { .compatible = "fsl,imx8mp-sdma", .data = &sdma_imx8mp, }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, sdma_dt_ids); @@ -717,7 +773,7 @@ static int sdma_run_channel0(struct sdma_engine *sdma) sdma_enable_channel(sdma, 0); ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP, - reg, !(reg & 1), 1, 500); + reg, !(reg & 1), 1, 5000); if (ret) dev_err(sdma->dev, "Timeout waiting for CH0 ready\n"); @@ -731,36 +787,80 @@ static int sdma_run_channel0(struct sdma_engine *sdma) return ret; } -static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, - u32 address) +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 47 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 48 + +static void sdma_add_scripts(struct sdma_engine *sdma, + const struct sdma_script_start_addrs *addr) +{ + s32 *addr_arr = (u32 *)addr; + s32 *saddr_arr = (u32 *)sdma->script_addrs; + int i; + + /* use the default firmware in ROM if missing external firmware */ + if (!sdma->script_number) + sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; + + if (sdma->script_number > sizeof(struct sdma_script_start_addrs) + / sizeof(s32)) { + dev_err(sdma->dev, + "SDMA script number %d not match with firmware.\n", + sdma->script_number); + return; + } + + for (i = 0; i < sdma->script_number; i++) + if (addr_arr[i] > 0) + saddr_arr[i] = addr_arr[i]; +} + +static int sdma_load_script(struct sdma_engine *sdma) { struct sdma_buffer_descriptor *bd0 = sdma->bd0; + const struct sdma_script_start_addrs *addr; + struct sdma_firmware_header *header; + unsigned short *ram_code; void *buf_virt; dma_addr_t buf_phys; int ret; unsigned long flags; - buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL); - if (!buf_virt) { + header = (struct sdma_firmware_header *)sdma->fw_data; + + addr = (void *)header + header->script_addrs_start; + ram_code = (void *)header + header->ram_code_start; + sdma->ram_code_start = header->ram_code_start; + + buf_virt = dma_alloc_coherent(sdma->dev, header->ram_code_size, + &buf_phys, GFP_KERNEL); + if (!buf_virt) return -ENOMEM; - } spin_lock_irqsave(&sdma->channel_0_lock, flags); bd0->mode.command = C0_SETPM; bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; - bd0->mode.count = size / 2; + bd0->mode.count = header->ram_code_size / 2; bd0->buffer_addr = buf_phys; - bd0->ext_buffer_addr = address; + bd0->ext_buffer_addr = addr->ram_code_start_addr; - memcpy(buf_virt, buf, size); + memcpy(buf_virt, ram_code, header->ram_code_size); ret = sdma_run_channel0(sdma); spin_unlock_irqrestore(&sdma->channel_0_lock, flags); - dma_free_coherent(sdma->dev, size, buf_virt, buf_phys); + dma_free_coherent(sdma->dev, header->ram_code_size, buf_virt, buf_phys); + + sdma_add_scripts(sdma, addr); + + sdma->fw_loaded = true; + dev_info_once(sdma->dev, "loaded firmware %d.%d\n", + header->version_major, + header->version_minor); return ret; } @@ -774,6 +874,38 @@ static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event) val = readl_relaxed(sdma->regs + chnenbl); __set_bit(channel, &val); writel_relaxed(val, sdma->regs + chnenbl); + + /* Set SDMA_DONEx_CONFIG is sw_done enabled */ + if (sdmac->audio_config && sdmac->audio_config->sw_done_sel & BIT(31)) { + u32 sw_done_sel = sdmac->audio_config->sw_done_sel & 0xff; + u32 offset = SDMA_DONE0_CONFIG + sw_done_sel / 4; + u32 done_sel = SDMA_DONE0_CONFIG_DONE_SEL + + ((sw_done_sel % 4) << 3); + u32 sw_done_dis = SDMA_DONE0_CONFIG_DONE_DIS + + ((sw_done_sel % 4) << 3); + + val = readl_relaxed(sdma->regs + offset); + __set_bit(done_sel, &val); + __clear_bit(sw_done_dis, &val); + writel_relaxed(val, sdma->regs + offset); + } + +} + +static int sdma_set_channel_priority(struct sdma_channel *sdmac, + unsigned int priority) +{ + struct sdma_engine *sdma = sdmac->sdma; + int channel = sdmac->channel; + + if (priority < MXC_SDMA_MIN_PRIORITY + || priority > MXC_SDMA_MAX_PRIORITY) { + return -EINVAL; + } + + writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel); + + return 0; } static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event) @@ -818,6 +950,7 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) struct sdma_buffer_descriptor *bd; int error = 0; enum dma_status old_status = sdmac->status; + int count = 0; /* * loop mode. Iterate over descriptors, re-setup them and @@ -828,6 +961,17 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) bd = &desc->bd[desc->buf_tail]; + /* + * re-enable HSTART_HE if all bds consumed at the last time, + * that happens in high loading case which sdma_handle_channel_ + * loop can't be handled in time while all bds run out in sdma + * side, then sdma script clear HE and cause channel stop. + */ + if (count == desc->num_bd) { + dev_warn(sdmac->sdma->dev, "All bds consumed,restart now.\n"); + sdma_enable_channel(sdmac->sdma, sdmac->channel); + } + if (bd->mode.status & BD_DONE) break; @@ -840,13 +984,20 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) /* * We use bd->mode.count to calculate the residue, since contains * the number of bytes present in the current buffer descriptor. + * Note: in IMX_DMATYPE_MULTI_SAI case, bd->mode.count used as + * remaining bytes instead so that one register could be saved. + * so chn_real_count = desc->period_len - bd->mode.count. */ + if (sdmac->peripheral_type == IMX_DMATYPE_MULTI_SAI) + desc->chn_real_count = desc->period_len - bd->mode.count; + else + desc->chn_real_count = bd->mode.count; - desc->chn_real_count = bd->mode.count; bd->mode.status |= BD_DONE; bd->mode.count = desc->period_len; desc->buf_ptail = desc->buf_tail; desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd; + count++; /* * The callback is called from the interrupt context in order @@ -893,6 +1044,13 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) struct sdma_engine *sdma = dev_id; unsigned long stat; + if (sdma->drvdata->pm_runtime) + pm_runtime_get_sync(sdma->dev); + else { + clk_enable(sdma->clk_ipg); + clk_enable(sdma->clk_ahb); + } + stat = readl_relaxed(sdma->regs + SDMA_H_INTR); writel_relaxed(stat, sdma->regs + SDMA_H_INTR); /* channel 0 is special and not handled here, see run_channel0() */ @@ -907,7 +1065,10 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) desc = sdmac->desc; if (desc) { if (sdmac->flags & IMX_DMA_SG_LOOP) { - sdma_update_channel_loop(sdmac); + if (sdmac->peripheral_type != IMX_DMATYPE_HDMI) + sdma_update_channel_loop(sdmac); + else + vchan_cyclic_callback(&desc->vd); } else { mxc_sdma_handle_channel_normal(sdmac); vchan_cookie_complete(&desc->vd); @@ -919,6 +1080,14 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) __clear_bit(channel, &stat); } + if (sdma->drvdata->pm_runtime) { + pm_runtime_mark_last_busy(sdma->dev); + pm_runtime_put_autosuspend(sdma->dev); + } else { + clk_disable(sdma->clk_ipg); + clk_disable(sdma->clk_ahb); + } + return IRQ_HANDLED; } @@ -1023,6 +1192,19 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_IPU_MEMORY: emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr; break; + case IMX_DMATYPE_HDMI: + emi_2_per = sdma->script_addrs->hdmi_dma_addr; + sdmac->is_ram_script = true; + break; + case IMX_DMATYPE_MULTI_SAI: + per_2_emi = sdma->script_addrs->sai_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_sai_addr; + sdmac->is_ram_script = true; + break; + case IMX_DMATYPE_I2C: + per_2_emi = sdma->script_addrs->i2c_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_i2c_addr; + sdmac->is_ram_script = true; default: break; } @@ -1031,6 +1213,13 @@ static void sdma_get_pc(struct sdma_channel *sdmac, sdmac->pc_to_device = emi_2_per; sdmac->device_to_device = per_2_per; sdmac->pc_to_pc = emi_2_emi; + + if (sdma->ram_code_start && + ((sdmac->pc_from_device >= sdma->ram_code_start) || + (sdmac->pc_to_device >= sdma->ram_code_start) || + (sdmac->device_to_device >= sdma->ram_code_start || + (sdmac->pc_to_pc >= sdma->ram_code_start)))) + sdmac->is_ram_script = true; } static int sdma_load_context(struct sdma_channel *sdmac) @@ -1070,11 +1259,16 @@ static int sdma_load_context(struct sdma_channel *sdmac) /* Send by context the event mask,base address for peripheral * and watermark level */ - context->gReg[0] = sdmac->event_mask[1]; - context->gReg[1] = sdmac->event_mask[0]; - context->gReg[2] = sdmac->per_addr; - context->gReg[6] = sdmac->shp_addr; - context->gReg[7] = sdmac->watermark_level; + if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { + context->gReg[4] = sdmac->per_addr; + context->gReg[6] = sdmac->shp_addr; + } else { + context->gReg[0] = sdmac->event_mask[1]; + context->gReg[1] = sdmac->event_mask[0]; + context->gReg[2] = sdmac->per_addr; + context->gReg[6] = sdmac->shp_addr; + context->gReg[7] = sdmac->watermark_level; + } bd0->mode.command = C0_SETDM; bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; @@ -1088,6 +1282,31 @@ static int sdma_load_context(struct sdma_channel *sdmac) return ret; } +static int sdma_save_restore_context(struct sdma_engine *sdma, bool save) +{ + struct sdma_context_data *context = sdma->context; + struct sdma_buffer_descriptor *bd0 = sdma->bd0; + unsigned long flags; + int ret; + + spin_lock_irqsave(&sdma->channel_0_lock, flags); + + if (save) + bd0->mode.command = C0_GETDM; + else + bd0->mode.command = C0_SETDM; + + bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; + bd0->mode.count = MAX_DMA_CHANNELS * sizeof(*context) / 4; + bd0->buffer_addr = sdma->context_phys; + bd0->ext_buffer_addr = 2048; + ret = sdma_run_channel0(sdma); + + spin_unlock_irqrestore(&sdma->channel_0_lock, flags); + + return ret; +} + static struct sdma_channel *to_sdma_chan(struct dma_chan *chan) { return container_of(chan, struct sdma_channel, vc.chan); @@ -1124,6 +1343,9 @@ static int sdma_terminate_all(struct dma_chan *chan) struct sdma_channel *sdmac = to_sdma_chan(chan); unsigned long flags; + if (sdmac->sdma->drvdata->pm_runtime) + pm_runtime_get_sync(sdmac->sdma->dev); + spin_lock_irqsave(&sdmac->vc.lock, flags); sdma_disable_channel(chan); @@ -1143,6 +1365,11 @@ static int sdma_terminate_all(struct dma_chan *chan) spin_unlock_irqrestore(&sdmac->vc.lock, flags); + if (sdmac->sdma->drvdata->pm_runtime) { + pm_runtime_mark_last_busy(sdmac->sdma->dev); + pm_runtime_put_autosuspend(sdmac->sdma->dev); + } + return 0; } @@ -1193,6 +1420,49 @@ static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP; sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT; + + if (sdmac->audio_config->src_fifo_num > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SD; + if (sdmac->audio_config->dst_fifo_num > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD; +} + +static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac) +{ + u8 fifo_num = sdmac->audio_config->src_fifo_num | + sdmac->audio_config->dst_fifo_num; + u8 fifo_offset = sdmac->audio_config->src_fifo_off | + sdmac->audio_config->dst_fifo_off; + u8 words_per_fifo = sdmac->audio_config->words_per_fifo; + + sdmac->watermark_level &= ~(0xFF << SDMA_WATERMARK_LEVEL_FIFOS_OFF | + SDMA_WATERMARK_LEVEL_SW_DONE | + 0xf << SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF | + 0xf << SDMA_WATERMARK_LEVEL_FIFO_OFF_OFF | + 0xf << SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO_OFF); + + if (sdmac->audio_config->sw_done_sel & BIT(31)) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SW_DONE | + (sdmac->audio_config->sw_done_sel & 0xff) << + SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF; + + /* For fifo_num + * bit 12-15 is the fifo number; + * bit 16-19 is the fifo offset, + * bit 28-31 is the channels per fifo. + * so here only need to shift left fifo_num 12 bit for watermake_level + */ + if (fifo_num) + sdmac->watermark_level |= fifo_num << + SDMA_WATERMARK_LEVEL_FIFOS_OFF; + + if (fifo_offset) + sdmac->watermark_level |= fifo_offset << + SDMA_WATERMARK_LEVEL_FIFO_OFF_OFF; + + if (words_per_fifo) + sdmac->watermark_level |= (words_per_fifo - 1) << + SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO_OFF; } static int sdma_config_channel(struct dma_chan *chan) @@ -1218,6 +1488,12 @@ static int sdma_config_channel(struct dma_chan *chan) break; } + sdma_set_channel_priority(sdmac, sdmac->prio); + + sdma_event_enable(sdmac, sdmac->event_id0); + if (sdmac->event_id1) + sdma_event_enable(sdmac, sdmac->event_id1); + sdma_get_pc(sdmac, sdmac->peripheral_type); if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) && @@ -1227,8 +1503,12 @@ static int sdma_config_channel(struct dma_chan *chan) if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP || sdmac->peripheral_type == IMX_DMATYPE_ASRC) sdma_set_watermarklevel_for_p2p(sdmac); - } else + } else { + if (sdmac->peripheral_type == IMX_DMATYPE_MULTI_SAI) + sdma_set_watermarklevel_for_sais(sdmac); + __set_bit(sdmac->event_id0, sdmac->event_mask); + } /* Address */ sdmac->shp_addr = sdmac->per_address; @@ -1240,28 +1520,18 @@ static int sdma_config_channel(struct dma_chan *chan) return 0; } -static int sdma_set_channel_priority(struct sdma_channel *sdmac, - unsigned int priority) -{ - struct sdma_engine *sdma = sdmac->sdma; - int channel = sdmac->channel; - - if (priority < MXC_SDMA_MIN_PRIORITY - || priority > MXC_SDMA_MAX_PRIORITY) { - return -EINVAL; - } - - writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel); - - return 0; -} - static int sdma_request_channel0(struct sdma_engine *sdma) { int ret = -EBUSY; - sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys, - GFP_NOWAIT); + if (sdma->iram_pool) + sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys); + else + sdma->bd0 = dma_alloc_coherent(sdma->dev, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys, GFP_NOWAIT); if (!sdma->bd0) { ret = -ENOMEM; goto out; @@ -1281,10 +1551,15 @@ out: static int sdma_alloc_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; int ret = 0; - desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size, - &desc->bd_phys, GFP_NOWAIT); + if (sdma->iram_pool) + desc->bd = gen_pool_dma_alloc(sdma->iram_pool, bd_size, + &desc->bd_phys); + else + desc->bd = dma_alloc_coherent(sdma->dev, bd_size, + &desc->bd_phys, GFP_NOWAIT); if (!desc->bd) { ret = -ENOMEM; goto out; @@ -1296,9 +1571,14 @@ out: static void sdma_free_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; - dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, - desc->bd_phys); + if (sdma->iram_pool) + gen_pool_free(sdma->iram_pool, (unsigned long)desc->bd, + bd_size); + else + dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, + desc->bd_phys); } static void sdma_desc_free(struct virt_dma_desc *vd) @@ -1309,12 +1589,122 @@ static void sdma_desc_free(struct virt_dma_desc *vd) kfree(desc); } +static int sdma_runtime_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sdma_engine *sdma = platform_get_drvdata(pdev); + + if (!sdma->is_on) + return 0; + + sdma->fw_loaded = false; + sdma->is_on = false; + + clk_disable(sdma->clk_ipg); + clk_disable(sdma->clk_ahb); + + /* free channel0 bd */ + if (sdma->iram_pool) + gen_pool_free(sdma->iram_pool, (unsigned long)sdma->bd0, + sizeof(struct sdma_buffer_descriptor)); + else + dma_free_coherent(sdma->dev, + sizeof(struct sdma_buffer_descriptor), + sdma->bd0, sdma->bd0_phys); + + return 0; +} + +static int sdma_runtime_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sdma_engine *sdma = platform_get_drvdata(pdev); + int i, ret = 0; + + ret = clk_enable(sdma->clk_ipg); + if (ret) + return ret; + ret = clk_enable(sdma->clk_ahb); + if (ret) + goto disable_clk_ipg; + + /* Do nothing at HW level if audiomix which shared with audio driver + * not off indeed. + */ + if (readl_relaxed(sdma->regs + SDMA_H_C0PTR)) { + if (sdma->iram_pool) + sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys); + else + sdma->bd0 = dma_alloc_coherent(sdma->dev, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys, GFP_NOWAIT); + if (!sdma->bd0) + ret = -ENOMEM; + + sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys; + sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys; + + sdma->is_on = true; + sdma->fw_loaded = true; + + return ret; + } + + /* Be sure SDMA has not started yet */ + writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); + + /* disable all channels */ + for (i = 0; i < sdma->drvdata->num_events; i++) + writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i)); + + /* All channels have priority 0 */ + for (i = 0; i < MAX_DMA_CHANNELS; i++) + writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4); + + ret = sdma_request_channel0(sdma); + if (ret) + return ret; + + sdma_config_ownership(&sdma->channel[0], false, true, false); + + /* Set Command Channel (Channel Zero) */ + writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR); + + /* Set bits of CONFIG register but with static context switching */ + if (sdma->clk_ratio) + writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG); + else + writel_relaxed(0, sdma->regs + SDMA_H_CONFIG); + + writel_relaxed(sdma->ccb_phys, sdma->regs + SDMA_H_C0PTR); + + /* Initializes channel's priorities */ + sdma_set_channel_priority(&sdma->channel[0], 7); + + if (!sdma->fw_data) + dev_dbg(sdma->dev, "firmware not ready.\n"); + else if (sdma_load_script(sdma)) + dev_warn(sdma->dev, "failed to load script.\n"); + + sdma->is_on = true; + + return 0; + +disable_clk_ipg: + clk_disable(sdma->clk_ipg); + dev_err(sdma->dev, "initialisation failed with %d\n", ret); + + return ret; +} + static int sdma_alloc_chan_resources(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); struct imx_dma_data *data = chan->private; struct imx_dma_data mem_data; - int prio, ret; + int prio; /* * MEMCPY may never setup chan->private by filter function such as @@ -1352,31 +1742,31 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) sdmac->peripheral_type = data->peripheral_type; sdmac->event_id0 = data->dma_request; sdmac->event_id1 = data->dma_request2; - - ret = clk_enable(sdmac->sdma->clk_ipg); - if (ret) - return ret; - ret = clk_enable(sdmac->sdma->clk_ahb); - if (ret) - goto disable_clk_ipg; - - ret = sdma_set_channel_priority(sdmac, prio); - if (ret) - goto disable_clk_ahb; + sdmac->prio = prio; return 0; - -disable_clk_ahb: - clk_disable(sdmac->sdma->clk_ahb); -disable_clk_ipg: - clk_disable(sdmac->sdma->clk_ipg); - return ret; } static void sdma_free_chan_resources(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); - struct sdma_engine *sdma = sdmac->sdma; + + /* + * Per fw_data is null which means firmware not loaded and sdma + * not initialized, directly return. This happens in below case: + * + * -- driver request dma chan in probe phase, after that driver + * fall into -EPROBE_DEFER and free channel again without + * anything else about dma, so just return directly, otherwise + * kernel could hang since dma hardware not ready if drvdata-> + * pm_runtime is false. + * + */ + if (unlikely(!sdmac->sdma->fw_data)) + return; + + if (sdmac->sdma->drvdata->pm_runtime) + pm_runtime_get_sync(sdmac->sdma->dev); sdma_terminate_all(chan); @@ -1391,8 +1781,13 @@ static void sdma_free_chan_resources(struct dma_chan *chan) sdma_set_channel_priority(sdmac, 0); - clk_disable(sdma->clk_ipg); - clk_disable(sdma->clk_ahb); + kfree(sdmac->audio_config); + sdmac->audio_config = NULL; + + if (sdmac->sdma->drvdata->pm_runtime) { + pm_runtime_mark_last_busy(sdmac->sdma->dev); + pm_runtime_put_autosuspend(sdmac->sdma->dev); + } } static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, @@ -1420,7 +1815,7 @@ static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, desc->sdmac = sdmac; desc->num_bd = bds; - if (sdma_alloc_bd(desc)) + if (bds && sdma_alloc_bd(desc)) goto err_desc_out; /* No slave_config called in MEMCPY case, so do here */ @@ -1456,10 +1851,16 @@ static struct dma_async_tx_descriptor *sdma_prep_memcpy( dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n", &dma_src, &dma_dst, len, channel); + if (sdma->drvdata->pm_runtime) + pm_runtime_get_sync(sdmac->sdma->dev); + desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM, len / SDMA_BD_MAX_CNT + 1); - if (!desc) + if (!desc) { + if (sdma->drvdata->pm_runtime) + pm_runtime_put_sync_suspend(sdmac->sdma->dev); return NULL; + } do { count = min_t(size_t, len, SDMA_BD_MAX_CNT); @@ -1491,6 +1892,11 @@ static struct dma_async_tx_descriptor *sdma_prep_memcpy( bd->mode.status = param; } while (len); + if (sdmac->sdma->drvdata->pm_runtime) { + pm_runtime_mark_last_busy(sdmac->sdma->dev); + pm_runtime_put_autosuspend(sdmac->sdma->dev); + } + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); } @@ -1506,6 +1912,9 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( struct scatterlist *sg; struct sdma_desc *desc; + if (sdma->drvdata->pm_runtime) + pm_runtime_get_sync(sdmac->sdma->dev); + sdma_config_write(chan, &sdmac->slave_config, direction); desc = sdma_transfer_init(sdmac, direction, sg_len); @@ -1541,6 +1950,9 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( if (count & 3 || sg->dma_address & 3) goto err_bd_out; break; + case DMA_SLAVE_BUSWIDTH_3_BYTES: + bd->mode.command = 3; + break; case DMA_SLAVE_BUSWIDTH_2_BYTES: bd->mode.command = 2; if (count & 1 || sg->dma_address & 1) @@ -1569,12 +1981,19 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( bd->mode.status = param; } + if (sdmac->sdma->drvdata->pm_runtime) { + pm_runtime_mark_last_busy(sdmac->sdma->dev); + pm_runtime_put_autosuspend(sdmac->sdma->dev); + } + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); err_bd_out: sdma_free_bd(desc); kfree(desc); err_out: sdmac->status = DMA_ERROR; + if (sdma->drvdata->pm_runtime) + pm_runtime_put_sync_suspend(sdmac->sdma->dev); return NULL; } @@ -1585,13 +2004,19 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( { struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; - int num_periods = buf_len / period_len; + int num_periods = 0; int channel = sdmac->channel; int i = 0, buf = 0; struct sdma_desc *desc; dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel); + if (sdma->drvdata->pm_runtime) + pm_runtime_get_sync(sdmac->sdma->dev); + + if (sdmac->peripheral_type != IMX_DMATYPE_HDMI) + num_periods = buf_len / period_len; + sdma_config_write(chan, &sdmac->slave_config, direction); desc = sdma_transfer_init(sdmac, direction, num_periods); @@ -1608,6 +2033,9 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( goto err_bd_out; } + if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); + while (buf < buf_len) { struct sdma_buffer_descriptor *bd = &desc->bd[i]; int param; @@ -1640,12 +2068,19 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( i++; } + if (sdmac->sdma->drvdata->pm_runtime) { + pm_runtime_mark_last_busy(sdmac->sdma->dev); + pm_runtime_put_autosuspend(sdmac->sdma->dev); + } + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); err_bd_out: sdma_free_bd(desc); kfree(desc); err_out: sdmac->status = DMA_ERROR; + if (sdma->drvdata->pm_runtime) + pm_runtime_put_sync_suspend(sdmac->sdma->dev); return NULL; } @@ -1655,6 +2090,8 @@ static int sdma_config_write(struct dma_chan *chan, { struct sdma_channel *sdmac = to_sdma_chan(chan); + sdmac->watermark_level = 0; + if (direction == DMA_DEV_TO_MEM) { sdmac->per_address = dmaengine_cfg->src_addr; sdmac->watermark_level = dmaengine_cfg->src_maxburst * @@ -1668,6 +2105,10 @@ static int sdma_config_write(struct dma_chan *chan, sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) & SDMA_WATERMARK_LEVEL_HWML; sdmac->word_size = dmaengine_cfg->dst_addr_width; + } else if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { + sdmac->per_address = dmaengine_cfg->dst_addr; + sdmac->per_address2 = dmaengine_cfg->src_addr; + sdmac->watermark_level = 0; } else { sdmac->per_address = dmaengine_cfg->dst_addr; sdmac->watermark_level = dmaengine_cfg->dst_maxburst * @@ -1682,19 +2123,31 @@ static int sdma_config(struct dma_chan *chan, struct dma_slave_config *dmaengine_cfg) { struct sdma_channel *sdmac = to_sdma_chan(chan); + void *tmp; memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg)); + /* Allocate special sdma_audio_config if it's used */ + if (dmaengine_cfg->peripheral_config) { + tmp = krealloc(sdmac->audio_config, + dmaengine_cfg->peripheral_size, GFP_NOWAIT); + if (!tmp) + return -ENOMEM; + + sdmac->audio_config = (struct sdma_audio_config *)tmp; + + memcpy(tmp, dmaengine_cfg->peripheral_config, + dmaengine_cfg->peripheral_size); + } + /* Set ENBLn earlier to make sure dma request triggered after that */ if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) return -EINVAL; - sdma_event_enable(sdmac, sdmac->event_id0); - if (sdmac->event_id1) { - if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) - return -EINVAL; - sdma_event_enable(sdmac, sdmac->event_id1); - } + + if (sdmac->event_id1 && + sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) + return -EINVAL; return 0; } @@ -1745,68 +2198,41 @@ static void sdma_issue_pending(struct dma_chan *chan) struct sdma_channel *sdmac = to_sdma_chan(chan); unsigned long flags; + if (sdmac->sdma->drvdata->pm_runtime) + pm_runtime_get_sync(sdmac->sdma->dev); + spin_lock_irqsave(&sdmac->vc.lock, flags); if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc) sdma_start_desc(sdmac); spin_unlock_irqrestore(&sdmac->vc.lock, flags); -} - -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 45 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 46 - -static void sdma_add_scripts(struct sdma_engine *sdma, - const struct sdma_script_start_addrs *addr) -{ - s32 *addr_arr = (u32 *)addr; - s32 *saddr_arr = (u32 *)sdma->script_addrs; - int i; - /* use the default firmware in ROM if missing external firmware */ - if (!sdma->script_number) - sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; - - if (sdma->script_number > sizeof(struct sdma_script_start_addrs) - / sizeof(s32)) { - dev_err(sdma->dev, - "SDMA script number %d not match with firmware.\n", - sdma->script_number); - return; + if (sdmac->sdma->drvdata->pm_runtime) { + pm_runtime_mark_last_busy(sdmac->sdma->dev); + pm_runtime_put_autosuspend(sdmac->sdma->dev); } - - for (i = 0; i < sdma->script_number; i++) - if (addr_arr[i] > 0) - saddr_arr[i] = addr_arr[i]; - - /* - * get uart_2_mcu_addr/uartsh_2_mcu_addr rom script specially because - * they are now replaced by uart_2_mcu_ram_addr/uartsh_2_mcu_ram_addr - * to be compatible with legacy freescale/nxp sdma firmware, and they - * are located in the bottom part of sdma_script_start_addrs which are - * beyond the SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1. - */ - if (addr->uart_2_mcu_addr) - sdma->script_addrs->uart_2_mcu_addr = addr->uart_2_mcu_addr; - if (addr->uartsh_2_mcu_addr) - sdma->script_addrs->uartsh_2_mcu_addr = addr->uartsh_2_mcu_addr; - } static void sdma_load_firmware(const struct firmware *fw, void *context) { struct sdma_engine *sdma = context; const struct sdma_firmware_header *header; - const struct sdma_script_start_addrs *addr; - unsigned short *ram_code; if (!fw) { - dev_info(sdma->dev, "external firmware not found, using ROM firmware\n"); - /* In this case we just use the ROM firmware. */ + /* Load firmware once more time if timeout */ + if (sdma->fw_fail) + dev_info(sdma->dev, "external firmware not found, using ROM firmware\n"); + else { + request_firmware_nowait(THIS_MODULE, + FW_ACTION_UEVENT, sdma->fw_name, + sdma->dev, GFP_KERNEL, sdma, + sdma_load_firmware); + sdma->fw_fail++; + } + return; } - if (fw->size < sizeof(*header)) + if (fw->size < sizeof(*header) || sdma->fw_loaded) goto err_firmware; header = (struct sdma_firmware_header *)fw->data; @@ -1833,25 +2259,18 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) goto err_firmware; } - addr = (void *)header + header->script_addrs_start; - ram_code = (void *)header + header->ram_code_start; + dev_info(sdma->dev, "firmware found.\n"); - clk_enable(sdma->clk_ipg); - clk_enable(sdma->clk_ahb); - /* download the RAM image for SDMA */ - sdma_load_script(sdma, ram_code, - header->ram_code_size, - addr->ram_code_start_addr); - clk_disable(sdma->clk_ipg); - clk_disable(sdma->clk_ahb); + if (!sdma->fw_data) { + sdma->fw_data = kmalloc(fw->size, GFP_KERNEL); + if (!sdma->fw_data) + goto err_firmware; - sdma_add_scripts(sdma, addr); + memcpy(sdma->fw_data, fw->data, fw->size); - sdma->fw_loaded = true; - - dev_info(sdma->dev, "loaded firmware %d.%d\n", - header->version_major, - header->version_minor); + if (!sdma->drvdata->pm_runtime) + pm_runtime_get_sync(sdma->dev); + } err_firmware: release_firmware(fw); @@ -1935,79 +2354,34 @@ static int sdma_get_firmware(struct sdma_engine *sdma, return ret; } -static int sdma_init(struct sdma_engine *sdma) +static int sdma_init_sw(struct sdma_engine *sdma) { - int i, ret; - dma_addr_t ccb_phys; - - ret = clk_enable(sdma->clk_ipg); - if (ret) - return ret; - ret = clk_enable(sdma->clk_ahb); - if (ret) - goto disable_clk_ipg; + int ret, ccbsize; if (sdma->drvdata->check_ratio && (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg))) sdma->clk_ratio = 1; - /* Be sure SDMA has not started yet */ - writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); - - sdma->channel_control = dma_alloc_coherent(sdma->dev, - MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) + - sizeof(struct sdma_context_data), - &ccb_phys, GFP_KERNEL); + ccbsize = MAX_DMA_CHANNELS * (sizeof(struct sdma_channel_control) + + sizeof(struct sdma_context_data)); + if (sdma->iram_pool) + sdma->channel_control = gen_pool_dma_alloc(sdma->iram_pool, + ccbsize, &sdma->ccb_phys); + else + sdma->channel_control = dma_alloc_coherent(sdma->dev, ccbsize, + &sdma->ccb_phys, GFP_KERNEL); if (!sdma->channel_control) { ret = -ENOMEM; - goto err_dma_alloc; + return ret; } sdma->context = (void *)sdma->channel_control + MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control); - sdma->context_phys = ccb_phys + + sdma->context_phys = sdma->ccb_phys + MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control); - /* disable all channels */ - for (i = 0; i < sdma->drvdata->num_events; i++) - writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i)); - - /* All channels have priority 0 */ - for (i = 0; i < MAX_DMA_CHANNELS; i++) - writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4); - - ret = sdma_request_channel0(sdma); - if (ret) - goto err_dma_alloc; - - sdma_config_ownership(&sdma->channel[0], false, true, false); - - /* Set Command Channel (Channel Zero) */ - writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR); - - /* Set bits of CONFIG register but with static context switching */ - if (sdma->clk_ratio) - writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG); - else - writel_relaxed(0, sdma->regs + SDMA_H_CONFIG); - - writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR); - - /* Initializes channel's priorities */ - sdma_set_channel_priority(&sdma->channel[0], 7); - - clk_disable(sdma->clk_ipg); - clk_disable(sdma->clk_ahb); - return 0; - -err_dma_alloc: - clk_disable(sdma->clk_ahb); -disable_clk_ipg: - clk_disable(sdma->clk_ipg); - dev_err(sdma->dev, "initialisation failed with %d\n", ret); - return ret; } static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param) @@ -2034,9 +2408,11 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, if (dma_spec->args_count != 3) return NULL; + memset(&data, 0, sizeof(data)); + data.dma_request = dma_spec->args[0]; data.peripheral_type = dma_spec->args[1]; - data.priority = dma_spec->args[2]; + data.priority = dma_spec->args[2] & 0xff; /* * init dma_request2 to zero, which is not used by the dts. * For P2P, dma_request2 is init from dma_request_channel(), @@ -2144,7 +2520,13 @@ static int sdma_probe(struct platform_device *pdev) vchan_init(&sdmac->vc, &sdma->dma_device); } - ret = sdma_init(sdma); + if (np) { + sdma->iram_pool = of_gen_pool_get(np, "iram", 0); + if (sdma->iram_pool) + dev_info(&pdev->dev, "alloc bd from iram.\n"); + } + + ret = sdma_init_sw(sdma); if (ret) goto err_init; @@ -2205,14 +2587,25 @@ static int sdma_probe(struct platform_device *pdev) */ ret = of_property_read_string(np, "fsl,sdma-ram-script-name", &fw_name); - if (ret) { + if (ret) dev_warn(&pdev->dev, "failed to get firmware name\n"); - } else { - ret = sdma_get_firmware(sdma, fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); + else + sdma->fw_name = fw_name; + + ret = sdma_get_firmware(sdma, sdma->fw_name); + if (ret) + dev_warn(sdma->dev, "failed to get firmware.\n"); + + /* enable autosuspend for pm_runtime */ + if (sdma->drvdata->pm_runtime) { + pm_runtime_set_autosuspend_delay(&pdev->dev, 8000); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_mark_last_busy(&pdev->dev); + pm_runtime_set_active(&pdev->dev); } + pm_runtime_enable(&pdev->dev); + return 0; err_register: @@ -2234,6 +2627,7 @@ static int sdma_remove(struct platform_device *pdev) devm_free_irq(&pdev->dev, sdma->irq, sdma); dma_async_device_unregister(&sdma->dma_device); kfree(sdma->script_addrs); + kfree(sdma->fw_data); clk_unprepare(sdma->clk_ahb); clk_unprepare(sdma->clk_ipg); /* Kill the tasklet */ @@ -2244,14 +2638,127 @@ static int sdma_remove(struct platform_device *pdev) sdma_free_chan_resources(&sdmac->vc.chan); } + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); + platform_set_drvdata(pdev, NULL); return 0; } +#ifdef CONFIG_PM_SLEEP +static int sdma_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sdma_engine *sdma = platform_get_drvdata(pdev); + int i, ret = 0; + + /* Do nothing if not i.MX6SX/6UL or i.MX7D, i.MX8MP */ + if (sdma->drvdata != &sdma_imx6sx && sdma->drvdata != &sdma_imx7d + && sdma->drvdata != &sdma_imx6ul && sdma->drvdata != &sdma_imx8mp) + return 0; + + if (!sdma->is_on) + return 0; + + ret = sdma_save_restore_context(sdma, true); + if (ret) { + dev_err(sdma->dev, "save context error!\n"); + return ret; + } + + /* save regs */ + for (i = 0; i < MXC_SDMA_SAVED_REG_NUM; i++) { + /* + * 0x78(SDMA_XTRIG_CONF2+4)~0x100(SDMA_CHNPRI_O) registers are + * reserved and can't be touched. Skip these regs. + */ + if (i > SDMA_XTRIG_CONF2 / 4) + sdma->save_regs[i] = readl_relaxed(sdma->regs + + MXC_SDMA_RESERVED_REG + + 4 * i); + else + sdma->save_regs[i] = readl_relaxed(sdma->regs + 4 * i); + } + + if (sdma->drvdata->has_done0) { + for (i = 0; i < 2; i++) + sdma->save_done0_regs[i] = + readl_relaxed(sdma->regs + SDMA_DONE0_CONFIG + 4 * i); + } + + return 0; +} + +static int sdma_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct sdma_engine *sdma = platform_get_drvdata(pdev); + int i, ret; + + /* Do nothing if not i.MX6SX/6UL or i.MX7D, i.MX8MP */ + if (sdma->drvdata != &sdma_imx6sx && sdma->drvdata != &sdma_imx7d + && sdma->drvdata != &sdma_imx6ul && sdma->drvdata != &sdma_imx8mp) + return 0; + + if (!sdma->is_on) + return 0; + + /* Do nothing if mega/fast mix not turned off */ + if (readl_relaxed(sdma->regs + SDMA_H_C0PTR)) + return 0; + + /* Firmware was lost, mark as "not ready" */ + sdma->fw_loaded = false; + + /* restore regs and load firmware */ + for (i = 0; i < MXC_SDMA_SAVED_REG_NUM; i++) { + /* + * 0x78(SDMA_XTRIG_CONF2+4)~0x100(SDMA_CHNPRI_O) registers are + * reserved and can't be touched. Skip these regs. + */ + if (i > SDMA_XTRIG_CONF2 / 4) + writel_relaxed(sdma->save_regs[i], sdma->regs + + MXC_SDMA_RESERVED_REG + 4 * i); + /* set static context switch mode before channel0 running */ + else if (i == SDMA_H_CONFIG / 4) + writel_relaxed(sdma->save_regs[i] & ~SDMA_H_CONFIG_CSM, + sdma->regs + SDMA_H_CONFIG); + else + writel_relaxed(sdma->save_regs[i], sdma->regs + 4 * i); + } + + /* restore SDMA_DONEx_CONFIG */ + if (sdma->drvdata->has_done0) { + for (i = 0; i < 2; i++) + writel_relaxed(sdma->save_done0_regs[i], + sdma->regs + SDMA_DONE0_CONFIG + 4 * i); + } + + /* prepare priority for channel0 to start */ + sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_MAX_PRIORITY); + + if (sdma_load_script(sdma)) + dev_warn(sdma->dev, "failed to load firmware.\n"); + + ret = sdma_save_restore_context(sdma, false); + if (ret) + dev_err(sdma->dev, "restore context error!\n"); + + return ret; +} +#endif + +static const struct dev_pm_ops sdma_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(sdma_suspend, sdma_resume) + SET_RUNTIME_PM_OPS(sdma_runtime_suspend, sdma_runtime_resume, NULL) +}; + static struct platform_driver sdma_driver = { .driver = { .name = "imx-sdma", .of_match_table = sdma_dt_ids, + .pm = &sdma_pm_ops, }, .remove = sdma_remove, .probe = sdma_probe, diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index e5c2c9e71bf1..daa6e5f409b6 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -415,6 +415,8 @@ enum dma_slave_buswidth { * loops in this area in order to transfer the data. * @dst_port_window_size: same as src_port_window_size but for the destination * port. + * @src_fifo_num: bit 0-7 is the fifo number, bit:8-11 is the fifo offset; + * @dst_fifo_num: same as src_fifo_num * @device_fc: Flow Controller Settings. Only valid for slave channels. Fill * with 'true' if peripheral should be flow controller. Direction will be * selected at Runtime. diff --git a/include/linux/platform_data/dma-imx.h b/include/linux/platform_data/dma-imx.h index 281adbb26e6b..89eb61c5ba1d 100644 --- a/include/linux/platform_data/dma-imx.h +++ b/include/linux/platform_data/dma-imx.h @@ -39,6 +39,9 @@ enum sdma_peripheral_type { IMX_DMATYPE_SSI_DUAL, /* SSI Dual FIFO */ IMX_DMATYPE_ASRC_SP, /* Shared ASRC */ IMX_DMATYPE_SAI, /* SAI */ + IMX_DMATYPE_MULTI_SAI, /* MULTI FIFOs For Audio */ + IMX_DMATYPE_HDMI, /* HDMI Audio */ + IMX_DMATYPE_I2C, /* I2C */ }; enum imx_dma_prio { @@ -47,6 +50,39 @@ enum imx_dma_prio { DMA_PRIO_LOW = 2 }; +/** + * struct sdma_audio_config - special sdma config for audio case + * @src_fifo_num: source fifo number for mcu_2_sai/sai_2_mcu script + * For example, if there are 4 fifos, sdma will fetch + * fifos one by one and roll back to the first fifo after + * the 4th fifo fetch. + * @dst_fifo_num: similar as src_fifo_num, but dest fifo instead. + * @src_fifo_off: source fifo offset, 0 means all fifos are continuous, 1 + * means 1 word offset between fifos. All offset between + * fifos should be same. + * @dst_fifo_off: dst fifo offset, similar as @src_fifo_off. + * @words_per_fifo: numbers of words per fifo fetch/fill, 0 means + * one channel per fifo, 1 means 2 channels per fifo.. + * If 'src_fifo_num = 4' and 'chans_per_fifo = 1', it + * means the first two words(channels) fetch from fifo1 + * and then jump to fifo2 for next two words, and so on + * after the last fifo4 fetched, roll back to fifo1. + * @sw_done_sel: software done selector, PDM need enable software done feature + * in mcu_2_sai/sai_2_mcu script. + * Bit31: sw_done eanbled or not + * Bit16~Bit0: selector + * For example: 0x80000000 means sw_done enabled for done0 + * sector which is for PDM on i.mx8mm. + */ +struct sdma_audio_config { + u8 src_fifo_num; + u8 dst_fifo_num; + u8 src_fifo_off; + u8 dst_fifo_off; + u8 words_per_fifo; + u32 sw_done_sel; +}; + struct imx_dma_data { int dma_request; /* DMA request line */ int dma_request2; /* secondary DMA request line */ |