| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20260130110159.359501-5-s-adivi@ti.com>
Date: Fri, 30 Jan 2026 16:31:44 +0530
From: Sai Sree Kartheek Adivi <s-adivi@...com>
To: <peter.ujfalusi@...il.com>, <vkoul@...nel.org>, <robh@...nel.org>,
<krzk+dt@...nel.org>, <conor+dt@...nel.org>, <nm@...com>,
<ssantosh@...nel.org>, <dmaengine@...r.kernel.org>,
<devicetree@...r.kernel.org>, <linux-kernel@...r.kernel.org>,
<linux-arm-kernel@...ts.infradead.org>, <vigneshr@...com>, <s-adivi@...com>
CC: <r-sharma3@...com>, <gehariprasath@...com>
Subject: [PATCH v4 04/19] dmaengine: ti: k3-udma: move descriptor management to k3-udma-common.c
Refactor the K3 UDMA driver by moving all DMA descriptor handling
functions from k3-udma.c to a new common library, k3-udma-common.c.
This prepares the driver for supporting new K3 UDMA v2 variant
(used in AM62L) that can reuse the same descriptor handling logic.
To ensure this common code works correctly across all build
configurations (built-in vs modules), introduce a hidden Kconfig symbol:
CONFIG_TI_K3_UDMA_COMMON.
No functional changes intended.
Signed-off-by: Sai Sree Kartheek Adivi <s-adivi@...com>
---
drivers/dma/ti/Kconfig | 5 +
drivers/dma/ti/Makefile | 1 +
drivers/dma/ti/k3-udma-common.c | 1243 +++++++++++++++++++++++++++++++
drivers/dma/ti/k3-udma.c | 1218 ------------------------------
drivers/dma/ti/k3-udma.h | 54 ++
5 files changed, 1303 insertions(+), 1218 deletions(-)
create mode 100644 drivers/dma/ti/k3-udma-common.c
diff --git a/drivers/dma/ti/Kconfig b/drivers/dma/ti/Kconfig
index cbc30ab627832..712e456015459 100644
--- a/drivers/dma/ti/Kconfig
+++ b/drivers/dma/ti/Kconfig
@@ -42,12 +42,17 @@ config TI_K3_UDMA
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
select SOC_TI
+ select TI_K3_UDMA_COMMON
select TI_K3_RINGACC
select TI_K3_PSIL
help
Enable support for the TI UDMA (Unified DMA) controller. This
DMA engine is used in AM65x and j721e.
+config TI_K3_UDMA_COMMON
+ tristate
+ default n
+
config TI_K3_UDMA_GLUE_LAYER
tristate "Texas Instruments UDMA Glue layer for non DMAengine users"
depends on ARCH_K3 || COMPILE_TEST
diff --git a/drivers/dma/ti/Makefile b/drivers/dma/ti/Makefile
index d376c117cecf6..3b91c02e55eaf 100644
--- a/drivers/dma/ti/Makefile
+++ b/drivers/dma/ti/Makefile
@@ -3,6 +3,7 @@ obj-$(CONFIG_TI_CPPI41) += cppi41.o
obj-$(CONFIG_TI_EDMA) += edma.o
obj-$(CONFIG_DMA_OMAP) += omap-dma.o
obj-$(CONFIG_TI_K3_UDMA) += k3-udma.o
+obj-$(CONFIG_TI_K3_UDMA_COMMON) += k3-udma-common.o
obj-$(CONFIG_TI_K3_UDMA_GLUE_LAYER) += k3-udma-glue.o
k3-psil-lib-objs := k3-psil.o \
k3-psil-am654.o \
diff --git a/drivers/dma/ti/k3-udma-common.c b/drivers/dma/ti/k3-udma-common.c
new file mode 100644
index 0000000000000..9cb35759c70bb
--- /dev/null
+++ b/drivers/dma/ti/k3-udma-common.c
@@ -0,0 +1,1243 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025 Texas Instruments Incorporated - http://www.ti.com
+ * Author: Peter Ujfalusi <peter.ujfalusi@...com>
+ */
+
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/platform_device.h>
+#include <linux/soc/ti/k3-ringacc.h>
+
+#include "k3-udma.h"
+
+struct dma_descriptor_metadata_ops metadata_ops = {
+ .attach = udma_attach_metadata,
+ .get_ptr = udma_get_metadata_ptr,
+ .set_len = udma_set_metadata_len,
+};
+
+struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,
+ dma_addr_t paddr)
+{
+ struct udma_desc *d = uc->terminated_desc;
+
+ if (d) {
+ dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
+ d->desc_idx);
+
+ if (desc_paddr != paddr)
+ d = NULL;
+ }
+
+ if (!d) {
+ d = uc->desc;
+ if (d) {
+ dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
+ d->desc_idx);
+
+ if (desc_paddr != paddr)
+ d = NULL;
+ }
+ }
+
+ return d;
+}
+EXPORT_SYMBOL_GPL(udma_udma_desc_from_paddr);
+
+void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d)
+{
+ if (uc->use_dma_pool) {
+ int i;
+
+ for (i = 0; i < d->hwdesc_count; i++) {
+ if (!d->hwdesc[i].cppi5_desc_vaddr)
+ continue;
+
+ dma_pool_free(uc->hdesc_pool,
+ d->hwdesc[i].cppi5_desc_vaddr,
+ d->hwdesc[i].cppi5_desc_paddr);
+
+ d->hwdesc[i].cppi5_desc_vaddr = NULL;
+ }
+ } else if (d->hwdesc[0].cppi5_desc_vaddr) {
+ dma_free_coherent(uc->dma_dev, d->hwdesc[0].cppi5_desc_size,
+ d->hwdesc[0].cppi5_desc_vaddr,
+ d->hwdesc[0].cppi5_desc_paddr);
+
+ d->hwdesc[0].cppi5_desc_vaddr = NULL;
+ }
+}
+
+void udma_purge_desc_work(struct work_struct *work)
+{
+ struct udma_dev *ud = container_of(work, typeof(*ud), purge_work);
+ struct virt_dma_desc *vd, *_vd;
+ unsigned long flags;
+ LIST_HEAD(head);
+
+ spin_lock_irqsave(&ud->lock, flags);
+ list_splice_tail_init(&ud->desc_to_purge, &head);
+ spin_unlock_irqrestore(&ud->lock, flags);
+
+ list_for_each_entry_safe(vd, _vd, &head, node) {
+ struct udma_chan *uc = to_udma_chan(vd->tx.chan);
+ struct udma_desc *d = to_udma_desc(&vd->tx);
+
+ udma_free_hwdesc(uc, d);
+ list_del(&vd->node);
+ kfree(d);
+ }
+
+ /* If more to purge, schedule the work again */
+ if (!list_empty(&ud->desc_to_purge))
+ schedule_work(&ud->purge_work);
+}
+EXPORT_SYMBOL_GPL(udma_purge_desc_work);
+
+void udma_desc_free(struct virt_dma_desc *vd)
+{
+ struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);
+ struct udma_chan *uc = to_udma_chan(vd->tx.chan);
+ struct udma_desc *d = to_udma_desc(&vd->tx);
+ unsigned long flags;
+
+ if (uc->terminated_desc == d)
+ uc->terminated_desc = NULL;
+
+ if (uc->use_dma_pool) {
+ udma_free_hwdesc(uc, d);
+ kfree(d);
+ return;
+ }
+
+ spin_lock_irqsave(&ud->lock, flags);
+ list_add_tail(&vd->node, &ud->desc_to_purge);
+ spin_unlock_irqrestore(&ud->lock, flags);
+
+ schedule_work(&ud->purge_work);
+}
+EXPORT_SYMBOL_GPL(udma_desc_free);
+
+bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr)
+{
+ if (uc->config.dir != DMA_DEV_TO_MEM)
+ return false;
+
+ if (addr == udma_get_rx_flush_hwdesc_paddr(uc))
+ return true;
+
+ return false;
+}
+EXPORT_SYMBOL_GPL(udma_desc_is_rx_flush);
+
+bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d)
+{
+ u32 peer_bcnt, bcnt;
+
+ /*
+ * Only TX towards PDMA is affected.
+ * If DMA_PREP_INTERRUPT is not set by consumer then skip the transfer
+ * completion calculation, consumer must ensure that there is no stale
+ * data in DMA fabric in this case.
+ */
+ if (uc->config.ep_type == PSIL_EP_NATIVE ||
+ uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT))
+ return true;
+
+ peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
+ bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
+
+ /* Transfer is incomplete, store current residue and time stamp */
+ if (peer_bcnt < bcnt) {
+ uc->tx_drain.residue = bcnt - peer_bcnt;
+ uc->tx_drain.tstamp = ktime_get();
+ return false;
+ }
+
+ return true;
+}
+EXPORT_SYMBOL_GPL(udma_is_desc_really_done);
+
+struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc,
+ size_t tr_size, int tr_count,
+ enum dma_transfer_direction dir)
+{
+ struct udma_hwdesc *hwdesc;
+ struct cppi5_desc_hdr_t *tr_desc;
+ struct udma_desc *d;
+ u32 reload_count = 0;
+ u32 ring_id;
+
+ switch (tr_size) {
+ case 16:
+ case 32:
+ case 64:
+ case 128:
+ break;
+ default:
+ dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size);
+ return NULL;
+ }
+
+ /* We have only one descriptor containing multiple TRs */
+ d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT);
+ if (!d)
+ return NULL;
+
+ d->sglen = tr_count;
+
+ d->hwdesc_count = 1;
+ hwdesc = &d->hwdesc[0];
+
+ /* Allocate memory for DMA ring descriptor */
+ if (uc->use_dma_pool) {
+ hwdesc->cppi5_desc_size = uc->config.hdesc_size;
+ hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
+ GFP_NOWAIT,
+ &hwdesc->cppi5_desc_paddr);
+ } else {
+ hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size,
+ tr_count);
+ hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
+ uc->ud->desc_align);
+ hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev,
+ hwdesc->cppi5_desc_size,
+ &hwdesc->cppi5_desc_paddr,
+ GFP_NOWAIT);
+ }
+
+ if (!hwdesc->cppi5_desc_vaddr) {
+ kfree(d);
+ return NULL;
+ }
+
+ /* Start of the TR req records */
+ hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
+ /* Start address of the TR response array */
+ hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count;
+
+ tr_desc = hwdesc->cppi5_desc_vaddr;
+
+ if (uc->cyclic)
+ reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE;
+
+ if (dir == DMA_DEV_TO_MEM)
+ ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
+ else
+ ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
+
+ cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count);
+ cppi5_desc_set_pktids(tr_desc, uc->id,
+ CPPI5_INFO1_DESC_FLOWID_DEFAULT);
+ cppi5_desc_set_retpolicy(tr_desc, 0, ring_id);
+
+ return d;
+}
+
+/**
+ * udma_get_tr_counters - calculate TR counters for a given length
+ * @len: Length of the transfer
+ * @align_to: Preferred alignment
+ * @tr0_cnt0: First TR icnt0
+ * @tr0_cnt1: First TR icnt1
+ * @tr1_cnt0: Second (if used) TR icnt0
+ *
+ * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated
+ * For len >= SZ_64K two TRs are used in a simple way:
+ * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1)
+ * Second TR: the remaining length (tr1_cnt0)
+ *
+ * Returns the number of TRs the length needs (1 or 2)
+ * -EINVAL if the length can not be supported
+ */
+int udma_get_tr_counters(size_t len, unsigned long align_to,
+ u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0)
+{
+ if (len < SZ_64K) {
+ *tr0_cnt0 = len;
+ *tr0_cnt1 = 1;
+
+ return 1;
+ }
+
+ if (align_to > 3)
+ align_to = 3;
+
+realign:
+ *tr0_cnt0 = SZ_64K - BIT(align_to);
+ if (len / *tr0_cnt0 >= SZ_64K) {
+ if (align_to) {
+ align_to--;
+ goto realign;
+ }
+ return -EINVAL;
+ }
+
+ *tr0_cnt1 = len / *tr0_cnt0;
+ *tr1_cnt0 = len % *tr0_cnt0;
+
+ return 2;
+}
+
+struct udma_desc *
+udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl,
+ unsigned int sglen, enum dma_transfer_direction dir,
+ unsigned long tx_flags, void *context)
+{
+ struct scatterlist *sgent;
+ struct udma_desc *d;
+ struct cppi5_tr_type1_t *tr_req = NULL;
+ u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
+ unsigned int i;
+ size_t tr_size;
+ int num_tr = 0;
+ int tr_idx = 0;
+ u64 asel;
+
+ /* estimate the number of TRs we will need */
+ for_each_sg(sgl, sgent, sglen, i) {
+ if (sg_dma_len(sgent) < SZ_64K)
+ num_tr++;
+ else
+ num_tr += 2;
+ }
+
+ /* Now allocate and setup the descriptor. */
+ tr_size = sizeof(struct cppi5_tr_type1_t);
+ d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
+ if (!d)
+ return NULL;
+
+ d->sglen = sglen;
+
+ if (uc->ud->match_data->type == DMA_TYPE_UDMA)
+ asel = 0;
+ else
+ asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
+
+ tr_req = d->hwdesc[0].tr_req_base;
+ for_each_sg(sgl, sgent, sglen, i) {
+ dma_addr_t sg_addr = sg_dma_address(sgent);
+
+ num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr),
+ &tr0_cnt0, &tr0_cnt1, &tr1_cnt0);
+ if (num_tr < 0) {
+ dev_err(uc->ud->dev, "size %u is not supported\n",
+ sg_dma_len(sgent));
+ udma_free_hwdesc(uc, d);
+ kfree(d);
+ return NULL;
+ }
+
+ cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
+ false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+ cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT);
+
+ sg_addr |= asel;
+ tr_req[tr_idx].addr = sg_addr;
+ tr_req[tr_idx].icnt0 = tr0_cnt0;
+ tr_req[tr_idx].icnt1 = tr0_cnt1;
+ tr_req[tr_idx].dim1 = tr0_cnt0;
+ tr_idx++;
+
+ if (num_tr == 2) {
+ cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
+ false, false,
+ CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+ cppi5_tr_csf_set(&tr_req[tr_idx].flags,
+ CPPI5_TR_CSF_SUPR_EVT);
+
+ tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0;
+ tr_req[tr_idx].icnt0 = tr1_cnt0;
+ tr_req[tr_idx].icnt1 = 1;
+ tr_req[tr_idx].dim1 = tr1_cnt0;
+ tr_idx++;
+ }
+
+ d->residue += sg_dma_len(sgent);
+ }
+
+ cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
+ CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
+
+ return d;
+}
+
+struct udma_desc *
+udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl,
+ unsigned int sglen,
+ enum dma_transfer_direction dir,
+ unsigned long tx_flags, void *context)
+{
+ struct scatterlist *sgent;
+ struct cppi5_tr_type15_t *tr_req = NULL;
+ enum dma_slave_buswidth dev_width;
+ u32 csf = CPPI5_TR_CSF_SUPR_EVT;
+ u16 tr_cnt0, tr_cnt1;
+ dma_addr_t dev_addr;
+ struct udma_desc *d;
+ unsigned int i;
+ size_t tr_size, sg_len;
+ int num_tr = 0;
+ int tr_idx = 0;
+ u32 burst, trigger_size, port_window;
+ u64 asel;
+
+ if (dir == DMA_DEV_TO_MEM) {
+ dev_addr = uc->cfg.src_addr;
+ dev_width = uc->cfg.src_addr_width;
+ burst = uc->cfg.src_maxburst;
+ port_window = uc->cfg.src_port_window_size;
+ } else if (dir == DMA_MEM_TO_DEV) {
+ dev_addr = uc->cfg.dst_addr;
+ dev_width = uc->cfg.dst_addr_width;
+ burst = uc->cfg.dst_maxburst;
+ port_window = uc->cfg.dst_port_window_size;
+ } else {
+ dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
+ return NULL;
+ }
+
+ if (!burst)
+ burst = 1;
+
+ if (port_window) {
+ if (port_window != burst) {
+ dev_err(uc->ud->dev,
+ "The burst must be equal to port_window\n");
+ return NULL;
+ }
+
+ tr_cnt0 = dev_width * port_window;
+ tr_cnt1 = 1;
+ } else {
+ tr_cnt0 = dev_width;
+ tr_cnt1 = burst;
+ }
+ trigger_size = tr_cnt0 * tr_cnt1;
+
+ /* estimate the number of TRs we will need */
+ for_each_sg(sgl, sgent, sglen, i) {
+ sg_len = sg_dma_len(sgent);
+
+ if (sg_len % trigger_size) {
+ dev_err(uc->ud->dev,
+ "Not aligned SG entry (%zu for %u)\n", sg_len,
+ trigger_size);
+ return NULL;
+ }
+
+ if (sg_len / trigger_size < SZ_64K)
+ num_tr++;
+ else
+ num_tr += 2;
+ }
+
+ /* Now allocate and setup the descriptor. */
+ tr_size = sizeof(struct cppi5_tr_type15_t);
+ d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
+ if (!d)
+ return NULL;
+
+ d->sglen = sglen;
+
+ if (uc->ud->match_data->type == DMA_TYPE_UDMA) {
+ asel = 0;
+ csf |= CPPI5_TR_CSF_EOL_ICNT0;
+ } else {
+ asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
+ dev_addr |= asel;
+ }
+
+ tr_req = d->hwdesc[0].tr_req_base;
+ for_each_sg(sgl, sgent, sglen, i) {
+ u16 tr0_cnt2, tr0_cnt3, tr1_cnt2;
+ dma_addr_t sg_addr = sg_dma_address(sgent);
+
+ sg_len = sg_dma_len(sgent);
+ num_tr = udma_get_tr_counters(sg_len / trigger_size, 0,
+ &tr0_cnt2, &tr0_cnt3, &tr1_cnt2);
+ if (num_tr < 0) {
+ dev_err(uc->ud->dev, "size %zu is not supported\n",
+ sg_len);
+ udma_free_hwdesc(uc, d);
+ kfree(d);
+ return NULL;
+ }
+
+ cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false,
+ true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+ cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf);
+ cppi5_tr_set_trigger(&tr_req[tr_idx].flags,
+ uc->config.tr_trigger_type,
+ CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0);
+
+ sg_addr |= asel;
+ if (dir == DMA_DEV_TO_MEM) {
+ tr_req[tr_idx].addr = dev_addr;
+ tr_req[tr_idx].icnt0 = tr_cnt0;
+ tr_req[tr_idx].icnt1 = tr_cnt1;
+ tr_req[tr_idx].icnt2 = tr0_cnt2;
+ tr_req[tr_idx].icnt3 = tr0_cnt3;
+ tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
+
+ tr_req[tr_idx].daddr = sg_addr;
+ tr_req[tr_idx].dicnt0 = tr_cnt0;
+ tr_req[tr_idx].dicnt1 = tr_cnt1;
+ tr_req[tr_idx].dicnt2 = tr0_cnt2;
+ tr_req[tr_idx].dicnt3 = tr0_cnt3;
+ tr_req[tr_idx].ddim1 = tr_cnt0;
+ tr_req[tr_idx].ddim2 = trigger_size;
+ tr_req[tr_idx].ddim3 = trigger_size * tr0_cnt2;
+ } else {
+ tr_req[tr_idx].addr = sg_addr;
+ tr_req[tr_idx].icnt0 = tr_cnt0;
+ tr_req[tr_idx].icnt1 = tr_cnt1;
+ tr_req[tr_idx].icnt2 = tr0_cnt2;
+ tr_req[tr_idx].icnt3 = tr0_cnt3;
+ tr_req[tr_idx].dim1 = tr_cnt0;
+ tr_req[tr_idx].dim2 = trigger_size;
+ tr_req[tr_idx].dim3 = trigger_size * tr0_cnt2;
+
+ tr_req[tr_idx].daddr = dev_addr;
+ tr_req[tr_idx].dicnt0 = tr_cnt0;
+ tr_req[tr_idx].dicnt1 = tr_cnt1;
+ tr_req[tr_idx].dicnt2 = tr0_cnt2;
+ tr_req[tr_idx].dicnt3 = tr0_cnt3;
+ tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
+ }
+
+ tr_idx++;
+
+ if (num_tr == 2) {
+ cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15,
+ false, true,
+ CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+ cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf);
+ cppi5_tr_set_trigger(&tr_req[tr_idx].flags,
+ uc->config.tr_trigger_type,
+ CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC,
+ 0, 0);
+
+ sg_addr += trigger_size * tr0_cnt2 * tr0_cnt3;
+ if (dir == DMA_DEV_TO_MEM) {
+ tr_req[tr_idx].addr = dev_addr;
+ tr_req[tr_idx].icnt0 = tr_cnt0;
+ tr_req[tr_idx].icnt1 = tr_cnt1;
+ tr_req[tr_idx].icnt2 = tr1_cnt2;
+ tr_req[tr_idx].icnt3 = 1;
+ tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
+
+ tr_req[tr_idx].daddr = sg_addr;
+ tr_req[tr_idx].dicnt0 = tr_cnt0;
+ tr_req[tr_idx].dicnt1 = tr_cnt1;
+ tr_req[tr_idx].dicnt2 = tr1_cnt2;
+ tr_req[tr_idx].dicnt3 = 1;
+ tr_req[tr_idx].ddim1 = tr_cnt0;
+ tr_req[tr_idx].ddim2 = trigger_size;
+ } else {
+ tr_req[tr_idx].addr = sg_addr;
+ tr_req[tr_idx].icnt0 = tr_cnt0;
+ tr_req[tr_idx].icnt1 = tr_cnt1;
+ tr_req[tr_idx].icnt2 = tr1_cnt2;
+ tr_req[tr_idx].icnt3 = 1;
+ tr_req[tr_idx].dim1 = tr_cnt0;
+ tr_req[tr_idx].dim2 = trigger_size;
+
+ tr_req[tr_idx].daddr = dev_addr;
+ tr_req[tr_idx].dicnt0 = tr_cnt0;
+ tr_req[tr_idx].dicnt1 = tr_cnt1;
+ tr_req[tr_idx].dicnt2 = tr1_cnt2;
+ tr_req[tr_idx].dicnt3 = 1;
+ tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
+ }
+ tr_idx++;
+ }
+
+ d->residue += sg_len;
+ }
+
+ cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, csf | CPPI5_TR_CSF_EOP);
+
+ return d;
+}
+
+int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d,
+ enum dma_slave_buswidth dev_width,
+ u16 elcnt)
+{
+ if (uc->config.ep_type != PSIL_EP_PDMA_XY)
+ return 0;
+
+ /* Bus width translates to the element size (ES) */
+ switch (dev_width) {
+ case DMA_SLAVE_BUSWIDTH_1_BYTE:
+ d->static_tr.elsize = 0;
+ break;
+ case DMA_SLAVE_BUSWIDTH_2_BYTES:
+ d->static_tr.elsize = 1;
+ break;
+ case DMA_SLAVE_BUSWIDTH_3_BYTES:
+ d->static_tr.elsize = 2;
+ break;
+ case DMA_SLAVE_BUSWIDTH_4_BYTES:
+ d->static_tr.elsize = 3;
+ break;
+ case DMA_SLAVE_BUSWIDTH_8_BYTES:
+ d->static_tr.elsize = 4;
+ break;
+ default: /* not reached */
+ return -EINVAL;
+ }
+
+ d->static_tr.elcnt = elcnt;
+
+ if (uc->config.pkt_mode || !uc->cyclic) {
+ /*
+ * PDMA must close the packet when the channel is in packet mode.
+ * For TR mode when the channel is not cyclic we also need PDMA
+ * to close the packet otherwise the transfer will stall because
+ * PDMA holds on the data it has received from the peripheral.
+ */
+ unsigned int div = dev_width * elcnt;
+
+ if (uc->cyclic)
+ d->static_tr.bstcnt = d->residue / d->sglen / div;
+ else
+ d->static_tr.bstcnt = d->residue / div;
+ } else if (uc->ud->match_data->type == DMA_TYPE_BCDMA &&
+ uc->config.dir == DMA_DEV_TO_MEM &&
+ uc->cyclic) {
+ /*
+ * For cyclic mode with BCDMA we have to set EOP in each TR to
+ * prevent short packet errors seen on channel teardown. So the
+ * PDMA must close the packet after every TR transfer by setting
+ * burst count equal to the number of bytes transferred.
+ */
+ struct cppi5_tr_type1_t *tr_req = d->hwdesc[0].tr_req_base;
+
+ d->static_tr.bstcnt =
+ (tr_req->icnt0 * tr_req->icnt1) / dev_width;
+ } else {
+ d->static_tr.bstcnt = 0;
+ }
+
+ if (uc->config.dir == DMA_DEV_TO_MEM &&
+ d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask)
+ return -EINVAL;
+
+ return 0;
+}
+
+struct udma_desc *
+udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl,
+ unsigned int sglen, enum dma_transfer_direction dir,
+ unsigned long tx_flags, void *context)
+{
+ struct scatterlist *sgent;
+ struct cppi5_host_desc_t *h_desc = NULL;
+ struct udma_desc *d;
+ u32 ring_id;
+ unsigned int i;
+ u64 asel;
+
+ d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT);
+ if (!d)
+ return NULL;
+
+ d->sglen = sglen;
+ d->hwdesc_count = sglen;
+
+ if (dir == DMA_DEV_TO_MEM)
+ ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
+ else
+ ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
+
+ if (uc->ud->match_data->type == DMA_TYPE_UDMA)
+ asel = 0;
+ else
+ asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
+
+ for_each_sg(sgl, sgent, sglen, i) {
+ struct udma_hwdesc *hwdesc = &d->hwdesc[i];
+ dma_addr_t sg_addr = sg_dma_address(sgent);
+ struct cppi5_host_desc_t *desc;
+ size_t sg_len = sg_dma_len(sgent);
+
+ hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
+ GFP_NOWAIT,
+ &hwdesc->cppi5_desc_paddr);
+ if (!hwdesc->cppi5_desc_vaddr) {
+ dev_err(uc->ud->dev,
+ "descriptor%d allocation failed\n", i);
+
+ udma_free_hwdesc(uc, d);
+ kfree(d);
+ return NULL;
+ }
+
+ d->residue += sg_len;
+ hwdesc->cppi5_desc_size = uc->config.hdesc_size;
+ desc = hwdesc->cppi5_desc_vaddr;
+
+ if (i == 0) {
+ cppi5_hdesc_init(desc, 0, 0);
+ /* Flow and Packed ID */
+ cppi5_desc_set_pktids(&desc->hdr, uc->id,
+ CPPI5_INFO1_DESC_FLOWID_DEFAULT);
+ cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id);
+ } else {
+ cppi5_hdesc_reset_hbdesc(desc);
+ cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff);
+ }
+
+ /* attach the sg buffer to the descriptor */
+ sg_addr |= asel;
+ cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len);
+
+ /* Attach link as host buffer descriptor */
+ if (h_desc)
+ cppi5_hdesc_link_hbdesc(h_desc,
+ hwdesc->cppi5_desc_paddr | asel);
+
+ if (uc->ud->match_data->type == DMA_TYPE_PKTDMA ||
+ dir == DMA_MEM_TO_DEV)
+ h_desc = desc;
+ }
+
+ if (d->residue >= SZ_4M) {
+ dev_err(uc->ud->dev,
+ "%s: Transfer size %u is over the supported 4M range\n",
+ __func__, d->residue);
+ udma_free_hwdesc(uc, d);
+ kfree(d);
+ return NULL;
+ }
+
+ h_desc = d->hwdesc[0].cppi5_desc_vaddr;
+ cppi5_hdesc_set_pktlen(h_desc, d->residue);
+
+ return d;
+}
+
+int udma_attach_metadata(struct dma_async_tx_descriptor *desc,
+ void *data, size_t len)
+{
+ struct udma_desc *d = to_udma_desc(desc);
+ struct udma_chan *uc = to_udma_chan(desc->chan);
+ struct cppi5_host_desc_t *h_desc;
+ u32 psd_size = len;
+ u32 flags = 0;
+
+ if (!uc->config.pkt_mode || !uc->config.metadata_size)
+ return -EOPNOTSUPP;
+
+ if (!data || len > uc->config.metadata_size)
+ return -EINVAL;
+
+ if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE)
+ return -EINVAL;
+
+ h_desc = d->hwdesc[0].cppi5_desc_vaddr;
+ if (d->dir == DMA_MEM_TO_DEV)
+ memcpy(h_desc->epib, data, len);
+
+ if (uc->config.needs_epib)
+ psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
+
+ d->metadata = data;
+ d->metadata_size = len;
+ if (uc->config.needs_epib)
+ flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
+
+ cppi5_hdesc_update_flags(h_desc, flags);
+ cppi5_hdesc_update_psdata_size(h_desc, psd_size);
+
+ return 0;
+}
+
+void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
+ size_t *payload_len, size_t *max_len)
+{
+ struct udma_desc *d = to_udma_desc(desc);
+ struct udma_chan *uc = to_udma_chan(desc->chan);
+ struct cppi5_host_desc_t *h_desc;
+
+ if (!uc->config.pkt_mode || !uc->config.metadata_size)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ h_desc = d->hwdesc[0].cppi5_desc_vaddr;
+
+ *max_len = uc->config.metadata_size;
+
+ *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ?
+ CPPI5_INFO0_HDESC_EPIB_SIZE : 0;
+ *payload_len += cppi5_hdesc_get_psdata_size(h_desc);
+
+ return h_desc->epib;
+}
+
+int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
+ size_t payload_len)
+{
+ struct udma_desc *d = to_udma_desc(desc);
+ struct udma_chan *uc = to_udma_chan(desc->chan);
+ struct cppi5_host_desc_t *h_desc;
+ u32 psd_size = payload_len;
+ u32 flags = 0;
+
+ if (!uc->config.pkt_mode || !uc->config.metadata_size)
+ return -EOPNOTSUPP;
+
+ if (payload_len > uc->config.metadata_size)
+ return -EINVAL;
+
+ if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE)
+ return -EINVAL;
+
+ h_desc = d->hwdesc[0].cppi5_desc_vaddr;
+
+ if (uc->config.needs_epib) {
+ psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
+ flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
+ }
+
+ cppi5_hdesc_update_flags(h_desc, flags);
+ cppi5_hdesc_update_psdata_size(h_desc, psd_size);
+
+ return 0;
+}
+
+struct dma_async_tx_descriptor *
+udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
+ unsigned int sglen, enum dma_transfer_direction dir,
+ unsigned long tx_flags, void *context)
+{
+ struct udma_chan *uc = to_udma_chan(chan);
+ enum dma_slave_buswidth dev_width;
+ struct udma_desc *d;
+ u32 burst;
+
+ if (dir != uc->config.dir &&
+ (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type)) {
+ dev_err(chan->device->dev,
+ "%s: chan%d is for %s, not supporting %s\n",
+ __func__, uc->id,
+ dmaengine_get_direction_text(uc->config.dir),
+ dmaengine_get_direction_text(dir));
+ return NULL;
+ }
+
+ if (dir == DMA_DEV_TO_MEM) {
+ dev_width = uc->cfg.src_addr_width;
+ burst = uc->cfg.src_maxburst;
+ } else if (dir == DMA_MEM_TO_DEV) {
+ dev_width = uc->cfg.dst_addr_width;
+ burst = uc->cfg.dst_maxburst;
+ } else {
+ dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
+ return NULL;
+ }
+
+ if (!burst)
+ burst = 1;
+
+ uc->config.tx_flags = tx_flags;
+
+ if (uc->config.pkt_mode)
+ d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags,
+ context);
+ else if (is_slave_direction(uc->config.dir))
+ d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags,
+ context);
+ else
+ d = udma_prep_slave_sg_triggered_tr(uc, sgl, sglen, dir,
+ tx_flags, context);
+
+ if (!d)
+ return NULL;
+
+ d->dir = dir;
+ d->desc_idx = 0;
+ d->tr_idx = 0;
+
+ /* static TR for remote PDMA */
+ if (udma_configure_statictr(uc, d, dev_width, burst)) {
+ dev_err(uc->ud->dev,
+ "%s: StaticTR Z is limited to maximum %u (%u)\n",
+ __func__, uc->ud->match_data->statictr_z_mask,
+ d->static_tr.bstcnt);
+
+ udma_free_hwdesc(uc, d);
+ kfree(d);
+ return NULL;
+ }
+
+ if (uc->config.metadata_size)
+ d->vd.tx.metadata_ops = &metadata_ops;
+
+ return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
+}
+EXPORT_SYMBOL_GPL(udma_prep_slave_sg);
+
+struct udma_desc *
+udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr,
+ size_t buf_len, size_t period_len,
+ enum dma_transfer_direction dir, unsigned long flags)
+{
+ struct udma_desc *d;
+ size_t tr_size, period_addr;
+ struct cppi5_tr_type1_t *tr_req;
+ unsigned int periods = buf_len / period_len;
+ u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
+ unsigned int i;
+ int num_tr;
+ u32 period_csf = 0;
+
+ num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0,
+ &tr0_cnt1, &tr1_cnt0);
+ if (num_tr < 0) {
+ dev_err(uc->ud->dev, "size %zu is not supported\n",
+ period_len);
+ return NULL;
+ }
+
+ /* Now allocate and setup the descriptor. */
+ tr_size = sizeof(struct cppi5_tr_type1_t);
+ d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir);
+ if (!d)
+ return NULL;
+
+ tr_req = d->hwdesc[0].tr_req_base;
+ if (uc->ud->match_data->type == DMA_TYPE_UDMA)
+ period_addr = buf_addr;
+ else
+ period_addr = buf_addr |
+ ((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT);
+
+ /*
+ * For BCDMA <-> PDMA transfers, the EOP flag needs to be set on the
+ * last TR of a descriptor, to mark the packet as complete.
+ * This is required for getting the teardown completion message in case
+ * of TX, and to avoid short-packet error in case of RX.
+ *
+ * As we are in cyclic mode, we do not know which period might be the
+ * last one, so set the flag for each period.
+ */
+ if (uc->config.ep_type == PSIL_EP_PDMA_XY &&
+ uc->ud->match_data->type == DMA_TYPE_BCDMA) {
+ period_csf = CPPI5_TR_CSF_EOP;
+ }
+
+ for (i = 0; i < periods; i++) {
+ int tr_idx = i * num_tr;
+
+ cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
+ false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+
+ tr_req[tr_idx].addr = period_addr;
+ tr_req[tr_idx].icnt0 = tr0_cnt0;
+ tr_req[tr_idx].icnt1 = tr0_cnt1;
+ tr_req[tr_idx].dim1 = tr0_cnt0;
+
+ if (num_tr == 2) {
+ cppi5_tr_csf_set(&tr_req[tr_idx].flags,
+ CPPI5_TR_CSF_SUPR_EVT);
+ tr_idx++;
+
+ cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
+ false, false,
+ CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+
+ tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0;
+ tr_req[tr_idx].icnt0 = tr1_cnt0;
+ tr_req[tr_idx].icnt1 = 1;
+ tr_req[tr_idx].dim1 = tr1_cnt0;
+ }
+
+ if (!(flags & DMA_PREP_INTERRUPT))
+ period_csf |= CPPI5_TR_CSF_SUPR_EVT;
+
+ if (period_csf)
+ cppi5_tr_csf_set(&tr_req[tr_idx].flags, period_csf);
+
+ period_addr += period_len;
+ }
+
+ return d;
+}
+
+struct udma_desc *
+udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr,
+ size_t buf_len, size_t period_len,
+ enum dma_transfer_direction dir, unsigned long flags)
+{
+ struct udma_desc *d;
+ u32 ring_id;
+ int i;
+ int periods = buf_len / period_len;
+
+ if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1))
+ return NULL;
+
+ if (period_len >= SZ_4M)
+ return NULL;
+
+ d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT);
+ if (!d)
+ return NULL;
+
+ d->hwdesc_count = periods;
+
+ /* TODO: re-check this... */
+ if (dir == DMA_DEV_TO_MEM)
+ ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
+ else
+ ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
+
+ if (uc->ud->match_data->type != DMA_TYPE_UDMA)
+ buf_addr |= (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
+
+ for (i = 0; i < periods; i++) {
+ struct udma_hwdesc *hwdesc = &d->hwdesc[i];
+ dma_addr_t period_addr = buf_addr + (period_len * i);
+ struct cppi5_host_desc_t *h_desc;
+
+ hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
+ GFP_NOWAIT,
+ &hwdesc->cppi5_desc_paddr);
+ if (!hwdesc->cppi5_desc_vaddr) {
+ dev_err(uc->ud->dev,
+ "descriptor%d allocation failed\n", i);
+
+ udma_free_hwdesc(uc, d);
+ kfree(d);
+ return NULL;
+ }
+
+ hwdesc->cppi5_desc_size = uc->config.hdesc_size;
+ h_desc = hwdesc->cppi5_desc_vaddr;
+
+ cppi5_hdesc_init(h_desc, 0, 0);
+ cppi5_hdesc_set_pktlen(h_desc, period_len);
+
+ /* Flow and Packed ID */
+ cppi5_desc_set_pktids(&h_desc->hdr, uc->id,
+ CPPI5_INFO1_DESC_FLOWID_DEFAULT);
+ cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id);
+
+ /* attach each period to a new descriptor */
+ cppi5_hdesc_attach_buf(h_desc,
+ period_addr, period_len,
+ period_addr, period_len);
+ }
+
+ return d;
+}
+
+struct dma_async_tx_descriptor *
+udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
+ size_t period_len, enum dma_transfer_direction dir,
+ unsigned long flags)
+{
+ struct udma_chan *uc = to_udma_chan(chan);
+ enum dma_slave_buswidth dev_width;
+ struct udma_desc *d;
+ u32 burst;
+
+ if (dir != uc->config.dir) {
+ dev_err(chan->device->dev,
+ "%s: chan%d is for %s, not supporting %s\n",
+ __func__, uc->id,
+ dmaengine_get_direction_text(uc->config.dir),
+ dmaengine_get_direction_text(dir));
+ return NULL;
+ }
+
+ uc->cyclic = true;
+
+ if (dir == DMA_DEV_TO_MEM) {
+ dev_width = uc->cfg.src_addr_width;
+ burst = uc->cfg.src_maxburst;
+ } else if (dir == DMA_MEM_TO_DEV) {
+ dev_width = uc->cfg.dst_addr_width;
+ burst = uc->cfg.dst_maxburst;
+ } else {
+ dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
+ return NULL;
+ }
+
+ if (!burst)
+ burst = 1;
+
+ if (uc->config.pkt_mode)
+ d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len,
+ dir, flags);
+ else
+ d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len,
+ dir, flags);
+
+ if (!d)
+ return NULL;
+
+ d->sglen = buf_len / period_len;
+
+ d->dir = dir;
+ d->residue = buf_len;
+
+ /* static TR for remote PDMA */
+ if (udma_configure_statictr(uc, d, dev_width, burst)) {
+ dev_err(uc->ud->dev,
+ "%s: StaticTR Z is limited to maximum %u (%u)\n",
+ __func__, uc->ud->match_data->statictr_z_mask,
+ d->static_tr.bstcnt);
+
+ udma_free_hwdesc(uc, d);
+ kfree(d);
+ return NULL;
+ }
+
+ if (uc->config.metadata_size)
+ d->vd.tx.metadata_ops = &metadata_ops;
+
+ return vchan_tx_prep(&uc->vc, &d->vd, flags);
+}
+EXPORT_SYMBOL_GPL(udma_prep_dma_cyclic);
+
+struct dma_async_tx_descriptor *
+udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
+ size_t len, unsigned long tx_flags)
+{
+ struct udma_chan *uc = to_udma_chan(chan);
+ struct udma_desc *d;
+ struct cppi5_tr_type15_t *tr_req;
+ int num_tr;
+ size_t tr_size = sizeof(struct cppi5_tr_type15_t);
+ u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
+ u32 csf = CPPI5_TR_CSF_SUPR_EVT;
+
+ if (uc->config.dir != DMA_MEM_TO_MEM) {
+ dev_err(chan->device->dev,
+ "%s: chan%d is for %s, not supporting %s\n",
+ __func__, uc->id,
+ dmaengine_get_direction_text(uc->config.dir),
+ dmaengine_get_direction_text(DMA_MEM_TO_MEM));
+ return NULL;
+ }
+
+ num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0,
+ &tr0_cnt1, &tr1_cnt0);
+ if (num_tr < 0) {
+ dev_err(uc->ud->dev, "size %zu is not supported\n",
+ len);
+ return NULL;
+ }
+
+ d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM);
+ if (!d)
+ return NULL;
+
+ d->dir = DMA_MEM_TO_MEM;
+ d->desc_idx = 0;
+ d->tr_idx = 0;
+ d->residue = len;
+
+ if (uc->ud->match_data->type != DMA_TYPE_UDMA) {
+ src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
+ dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
+ } else {
+ csf |= CPPI5_TR_CSF_EOL_ICNT0;
+ }
+
+ tr_req = d->hwdesc[0].tr_req_base;
+
+ cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
+ CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+ cppi5_tr_csf_set(&tr_req[0].flags, csf);
+
+ tr_req[0].addr = src;
+ tr_req[0].icnt0 = tr0_cnt0;
+ tr_req[0].icnt1 = tr0_cnt1;
+ tr_req[0].icnt2 = 1;
+ tr_req[0].icnt3 = 1;
+ tr_req[0].dim1 = tr0_cnt0;
+
+ tr_req[0].daddr = dest;
+ tr_req[0].dicnt0 = tr0_cnt0;
+ tr_req[0].dicnt1 = tr0_cnt1;
+ tr_req[0].dicnt2 = 1;
+ tr_req[0].dicnt3 = 1;
+ tr_req[0].ddim1 = tr0_cnt0;
+
+ if (num_tr == 2) {
+ cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
+ CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
+ cppi5_tr_csf_set(&tr_req[1].flags, csf);
+
+ tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
+ tr_req[1].icnt0 = tr1_cnt0;
+ tr_req[1].icnt1 = 1;
+ tr_req[1].icnt2 = 1;
+ tr_req[1].icnt3 = 1;
+
+ tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
+ tr_req[1].dicnt0 = tr1_cnt0;
+ tr_req[1].dicnt1 = 1;
+ tr_req[1].dicnt2 = 1;
+ tr_req[1].dicnt3 = 1;
+ }
+
+ cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, csf | CPPI5_TR_CSF_EOP);
+
+ if (uc->config.metadata_size)
+ d->vd.tx.metadata_ops = &metadata_ops;
+
+ return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
+}
+EXPORT_SYMBOL_GPL(udma_prep_dma_memcpy);
+
+void udma_desc_pre_callback(struct virt_dma_chan *vc,
+ struct virt_dma_desc *vd,
+ struct dmaengine_result *result)
+{
+ struct udma_chan *uc = to_udma_chan(&vc->chan);
+ struct udma_desc *d;
+ u8 status;
+
+ if (!vd)
+ return;
+
+ d = to_udma_desc(&vd->tx);
+
+ if (d->metadata_size)
+ udma_fetch_epib(uc, d);
+
+ if (result) {
+ void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx);
+
+ if (cppi5_desc_get_type(desc_vaddr) ==
+ CPPI5_INFO0_DESC_TYPE_VAL_HOST) {
+ /* Provide residue information for the client */
+ result->residue = d->residue -
+ cppi5_hdesc_get_pktlen(desc_vaddr);
+ if (result->residue)
+ result->result = DMA_TRANS_ABORTED;
+ else
+ result->result = DMA_TRANS_NOERROR;
+ } else {
+ result->residue = 0;
+ /* Propagate TR Response errors to the client */
+ status = d->hwdesc[0].tr_resp_base->status;
+ if (status)
+ result->result = DMA_TRANS_ABORTED;
+ else
+ result->result = DMA_TRANS_NOERROR;
+ }
+ }
+}
+EXPORT_SYMBOL_GPL(udma_desc_pre_callback);
+
+MODULE_DESCRIPTION("Texas Instruments K3 UDMA Common Library");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c
index 4adcd679c6997..0a1291829611f 100644
--- a/drivers/dma/ti/k3-udma.c
+++ b/drivers/dma/ti/k3-udma.c
@@ -131,105 +131,6 @@ static void udma_dump_chan_stdata(struct udma_chan *uc)
}
}
-static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,
- dma_addr_t paddr)
-{
- struct udma_desc *d = uc->terminated_desc;
-
- if (d) {
- dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
- d->desc_idx);
-
- if (desc_paddr != paddr)
- d = NULL;
- }
-
- if (!d) {
- d = uc->desc;
- if (d) {
- dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
- d->desc_idx);
-
- if (desc_paddr != paddr)
- d = NULL;
- }
- }
-
- return d;
-}
-
-static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d)
-{
- if (uc->use_dma_pool) {
- int i;
-
- for (i = 0; i < d->hwdesc_count; i++) {
- if (!d->hwdesc[i].cppi5_desc_vaddr)
- continue;
-
- dma_pool_free(uc->hdesc_pool,
- d->hwdesc[i].cppi5_desc_vaddr,
- d->hwdesc[i].cppi5_desc_paddr);
-
- d->hwdesc[i].cppi5_desc_vaddr = NULL;
- }
- } else if (d->hwdesc[0].cppi5_desc_vaddr) {
- dma_free_coherent(uc->dma_dev, d->hwdesc[0].cppi5_desc_size,
- d->hwdesc[0].cppi5_desc_vaddr,
- d->hwdesc[0].cppi5_desc_paddr);
-
- d->hwdesc[0].cppi5_desc_vaddr = NULL;
- }
-}
-
-static void udma_purge_desc_work(struct work_struct *work)
-{
- struct udma_dev *ud = container_of(work, typeof(*ud), purge_work);
- struct virt_dma_desc *vd, *_vd;
- unsigned long flags;
- LIST_HEAD(head);
-
- spin_lock_irqsave(&ud->lock, flags);
- list_splice_tail_init(&ud->desc_to_purge, &head);
- spin_unlock_irqrestore(&ud->lock, flags);
-
- list_for_each_entry_safe(vd, _vd, &head, node) {
- struct udma_chan *uc = to_udma_chan(vd->tx.chan);
- struct udma_desc *d = to_udma_desc(&vd->tx);
-
- udma_free_hwdesc(uc, d);
- list_del(&vd->node);
- kfree(d);
- }
-
- /* If more to purge, schedule the work again */
- if (!list_empty(&ud->desc_to_purge))
- schedule_work(&ud->purge_work);
-}
-
-static void udma_desc_free(struct virt_dma_desc *vd)
-{
- struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);
- struct udma_chan *uc = to_udma_chan(vd->tx.chan);
- struct udma_desc *d = to_udma_desc(&vd->tx);
- unsigned long flags;
-
- if (uc->terminated_desc == d)
- uc->terminated_desc = NULL;
-
- if (uc->use_dma_pool) {
- udma_free_hwdesc(uc, d);
- kfree(d);
- return;
- }
-
- spin_lock_irqsave(&ud->lock, flags);
- list_add_tail(&vd->node, &ud->desc_to_purge);
- spin_unlock_irqrestore(&ud->lock, flags);
-
- schedule_work(&ud->purge_work);
-}
-
static bool udma_is_chan_running(struct udma_chan *uc)
{
u32 trt_ctl = 0;
@@ -303,17 +204,6 @@ static int udma_push_to_ring(struct udma_chan *uc, int idx)
return k3_ringacc_ring_push(ring, &paddr);
}
-static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr)
-{
- if (uc->config.dir != DMA_DEV_TO_MEM)
- return false;
-
- if (addr == udma_get_rx_flush_hwdesc_paddr(uc))
- return true;
-
- return false;
-}
-
static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
{
struct k3_ring *ring = NULL;
@@ -674,33 +564,6 @@ static void udma_cyclic_packet_elapsed(struct udma_chan *uc)
d->desc_idx = (d->desc_idx + 1) % d->sglen;
}
-static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d)
-{
- u32 peer_bcnt, bcnt;
-
- /*
- * Only TX towards PDMA is affected.
- * If DMA_PREP_INTERRUPT is not set by consumer then skip the transfer
- * completion calculation, consumer must ensure that there is no stale
- * data in DMA fabric in this case.
- */
- if (uc->config.ep_type == PSIL_EP_NATIVE ||
- uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT))
- return true;
-
- peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
- bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
-
- /* Transfer is incomplete, store current residue and time stamp */
- if (peer_bcnt < bcnt) {
- uc->tx_drain.residue = bcnt - peer_bcnt;
- uc->tx_drain.tstamp = ktime_get();
- return false;
- }
-
- return true;
-}
-
static void udma_check_tx_completion(struct work_struct *work)
{
struct udma_chan *uc = container_of(work, typeof(*uc),
@@ -2344,1047 +2207,6 @@ static int udma_slave_config(struct dma_chan *chan,
return 0;
}
-static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc,
- size_t tr_size, int tr_count,
- enum dma_transfer_direction dir)
-{
- struct udma_hwdesc *hwdesc;
- struct cppi5_desc_hdr_t *tr_desc;
- struct udma_desc *d;
- u32 reload_count = 0;
- u32 ring_id;
-
- switch (tr_size) {
- case 16:
- case 32:
- case 64:
- case 128:
- break;
- default:
- dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size);
- return NULL;
- }
-
- /* We have only one descriptor containing multiple TRs */
- d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT);
- if (!d)
- return NULL;
-
- d->sglen = tr_count;
-
- d->hwdesc_count = 1;
- hwdesc = &d->hwdesc[0];
-
- /* Allocate memory for DMA ring descriptor */
- if (uc->use_dma_pool) {
- hwdesc->cppi5_desc_size = uc->config.hdesc_size;
- hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
- GFP_NOWAIT,
- &hwdesc->cppi5_desc_paddr);
- } else {
- hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size,
- tr_count);
- hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
- uc->ud->desc_align);
- hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev,
- hwdesc->cppi5_desc_size,
- &hwdesc->cppi5_desc_paddr,
- GFP_NOWAIT);
- }
-
- if (!hwdesc->cppi5_desc_vaddr) {
- kfree(d);
- return NULL;
- }
-
- /* Start of the TR req records */
- hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
- /* Start address of the TR response array */
- hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count;
-
- tr_desc = hwdesc->cppi5_desc_vaddr;
-
- if (uc->cyclic)
- reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE;
-
- if (dir == DMA_DEV_TO_MEM)
- ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
- else
- ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
-
- cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count);
- cppi5_desc_set_pktids(tr_desc, uc->id,
- CPPI5_INFO1_DESC_FLOWID_DEFAULT);
- cppi5_desc_set_retpolicy(tr_desc, 0, ring_id);
-
- return d;
-}
-
-/**
- * udma_get_tr_counters - calculate TR counters for a given length
- * @len: Length of the trasnfer
- * @align_to: Preferred alignment
- * @tr0_cnt0: First TR icnt0
- * @tr0_cnt1: First TR icnt1
- * @tr1_cnt0: Second (if used) TR icnt0
- *
- * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated
- * For len >= SZ_64K two TRs are used in a simple way:
- * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1)
- * Second TR: the remaining length (tr1_cnt0)
- *
- * Returns the number of TRs the length needs (1 or 2)
- * -EINVAL if the length can not be supported
- */
-static int udma_get_tr_counters(size_t len, unsigned long align_to,
- u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0)
-{
- if (len < SZ_64K) {
- *tr0_cnt0 = len;
- *tr0_cnt1 = 1;
-
- return 1;
- }
-
- if (align_to > 3)
- align_to = 3;
-
-realign:
- *tr0_cnt0 = SZ_64K - BIT(align_to);
- if (len / *tr0_cnt0 >= SZ_64K) {
- if (align_to) {
- align_to--;
- goto realign;
- }
- return -EINVAL;
- }
-
- *tr0_cnt1 = len / *tr0_cnt0;
- *tr1_cnt0 = len % *tr0_cnt0;
-
- return 2;
-}
-
-static struct udma_desc *
-udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl,
- unsigned int sglen, enum dma_transfer_direction dir,
- unsigned long tx_flags, void *context)
-{
- struct scatterlist *sgent;
- struct udma_desc *d;
- struct cppi5_tr_type1_t *tr_req = NULL;
- u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
- unsigned int i;
- size_t tr_size;
- int num_tr = 0;
- int tr_idx = 0;
- u64 asel;
-
- /* estimate the number of TRs we will need */
- for_each_sg(sgl, sgent, sglen, i) {
- if (sg_dma_len(sgent) < SZ_64K)
- num_tr++;
- else
- num_tr += 2;
- }
-
- /* Now allocate and setup the descriptor. */
- tr_size = sizeof(struct cppi5_tr_type1_t);
- d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
- if (!d)
- return NULL;
-
- d->sglen = sglen;
-
- if (uc->ud->match_data->type == DMA_TYPE_UDMA)
- asel = 0;
- else
- asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
-
- tr_req = d->hwdesc[0].tr_req_base;
- for_each_sg(sgl, sgent, sglen, i) {
- dma_addr_t sg_addr = sg_dma_address(sgent);
-
- num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr),
- &tr0_cnt0, &tr0_cnt1, &tr1_cnt0);
- if (num_tr < 0) {
- dev_err(uc->ud->dev, "size %u is not supported\n",
- sg_dma_len(sgent));
- udma_free_hwdesc(uc, d);
- kfree(d);
- return NULL;
- }
-
- cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
- false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
- cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT);
-
- sg_addr |= asel;
- tr_req[tr_idx].addr = sg_addr;
- tr_req[tr_idx].icnt0 = tr0_cnt0;
- tr_req[tr_idx].icnt1 = tr0_cnt1;
- tr_req[tr_idx].dim1 = tr0_cnt0;
- tr_idx++;
-
- if (num_tr == 2) {
- cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
- false, false,
- CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
- cppi5_tr_csf_set(&tr_req[tr_idx].flags,
- CPPI5_TR_CSF_SUPR_EVT);
-
- tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0;
- tr_req[tr_idx].icnt0 = tr1_cnt0;
- tr_req[tr_idx].icnt1 = 1;
- tr_req[tr_idx].dim1 = tr1_cnt0;
- tr_idx++;
- }
-
- d->residue += sg_dma_len(sgent);
- }
-
- cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
- CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
-
- return d;
-}
-
-static struct udma_desc *
-udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl,
- unsigned int sglen,
- enum dma_transfer_direction dir,
- unsigned long tx_flags, void *context)
-{
- struct scatterlist *sgent;
- struct cppi5_tr_type15_t *tr_req = NULL;
- enum dma_slave_buswidth dev_width;
- u32 csf = CPPI5_TR_CSF_SUPR_EVT;
- u16 tr_cnt0, tr_cnt1;
- dma_addr_t dev_addr;
- struct udma_desc *d;
- unsigned int i;
- size_t tr_size, sg_len;
- int num_tr = 0;
- int tr_idx = 0;
- u32 burst, trigger_size, port_window;
- u64 asel;
-
- if (dir == DMA_DEV_TO_MEM) {
- dev_addr = uc->cfg.src_addr;
- dev_width = uc->cfg.src_addr_width;
- burst = uc->cfg.src_maxburst;
- port_window = uc->cfg.src_port_window_size;
- } else if (dir == DMA_MEM_TO_DEV) {
- dev_addr = uc->cfg.dst_addr;
- dev_width = uc->cfg.dst_addr_width;
- burst = uc->cfg.dst_maxburst;
- port_window = uc->cfg.dst_port_window_size;
- } else {
- dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
- return NULL;
- }
-
- if (!burst)
- burst = 1;
-
- if (port_window) {
- if (port_window != burst) {
- dev_err(uc->ud->dev,
- "The burst must be equal to port_window\n");
- return NULL;
- }
-
- tr_cnt0 = dev_width * port_window;
- tr_cnt1 = 1;
- } else {
- tr_cnt0 = dev_width;
- tr_cnt1 = burst;
- }
- trigger_size = tr_cnt0 * tr_cnt1;
-
- /* estimate the number of TRs we will need */
- for_each_sg(sgl, sgent, sglen, i) {
- sg_len = sg_dma_len(sgent);
-
- if (sg_len % trigger_size) {
- dev_err(uc->ud->dev,
- "Not aligned SG entry (%zu for %u)\n", sg_len,
- trigger_size);
- return NULL;
- }
-
- if (sg_len / trigger_size < SZ_64K)
- num_tr++;
- else
- num_tr += 2;
- }
-
- /* Now allocate and setup the descriptor. */
- tr_size = sizeof(struct cppi5_tr_type15_t);
- d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
- if (!d)
- return NULL;
-
- d->sglen = sglen;
-
- if (uc->ud->match_data->type == DMA_TYPE_UDMA) {
- asel = 0;
- csf |= CPPI5_TR_CSF_EOL_ICNT0;
- } else {
- asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
- dev_addr |= asel;
- }
-
- tr_req = d->hwdesc[0].tr_req_base;
- for_each_sg(sgl, sgent, sglen, i) {
- u16 tr0_cnt2, tr0_cnt3, tr1_cnt2;
- dma_addr_t sg_addr = sg_dma_address(sgent);
-
- sg_len = sg_dma_len(sgent);
- num_tr = udma_get_tr_counters(sg_len / trigger_size, 0,
- &tr0_cnt2, &tr0_cnt3, &tr1_cnt2);
- if (num_tr < 0) {
- dev_err(uc->ud->dev, "size %zu is not supported\n",
- sg_len);
- udma_free_hwdesc(uc, d);
- kfree(d);
- return NULL;
- }
-
- cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false,
- true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
- cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf);
- cppi5_tr_set_trigger(&tr_req[tr_idx].flags,
- uc->config.tr_trigger_type,
- CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0);
-
- sg_addr |= asel;
- if (dir == DMA_DEV_TO_MEM) {
- tr_req[tr_idx].addr = dev_addr;
- tr_req[tr_idx].icnt0 = tr_cnt0;
- tr_req[tr_idx].icnt1 = tr_cnt1;
- tr_req[tr_idx].icnt2 = tr0_cnt2;
- tr_req[tr_idx].icnt3 = tr0_cnt3;
- tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
-
- tr_req[tr_idx].daddr = sg_addr;
- tr_req[tr_idx].dicnt0 = tr_cnt0;
- tr_req[tr_idx].dicnt1 = tr_cnt1;
- tr_req[tr_idx].dicnt2 = tr0_cnt2;
- tr_req[tr_idx].dicnt3 = tr0_cnt3;
- tr_req[tr_idx].ddim1 = tr_cnt0;
- tr_req[tr_idx].ddim2 = trigger_size;
- tr_req[tr_idx].ddim3 = trigger_size * tr0_cnt2;
- } else {
- tr_req[tr_idx].addr = sg_addr;
- tr_req[tr_idx].icnt0 = tr_cnt0;
- tr_req[tr_idx].icnt1 = tr_cnt1;
- tr_req[tr_idx].icnt2 = tr0_cnt2;
- tr_req[tr_idx].icnt3 = tr0_cnt3;
- tr_req[tr_idx].dim1 = tr_cnt0;
- tr_req[tr_idx].dim2 = trigger_size;
- tr_req[tr_idx].dim3 = trigger_size * tr0_cnt2;
-
- tr_req[tr_idx].daddr = dev_addr;
- tr_req[tr_idx].dicnt0 = tr_cnt0;
- tr_req[tr_idx].dicnt1 = tr_cnt1;
- tr_req[tr_idx].dicnt2 = tr0_cnt2;
- tr_req[tr_idx].dicnt3 = tr0_cnt3;
- tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
- }
-
- tr_idx++;
-
- if (num_tr == 2) {
- cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15,
- false, true,
- CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
- cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf);
- cppi5_tr_set_trigger(&tr_req[tr_idx].flags,
- uc->config.tr_trigger_type,
- CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC,
- 0, 0);
-
- sg_addr += trigger_size * tr0_cnt2 * tr0_cnt3;
- if (dir == DMA_DEV_TO_MEM) {
- tr_req[tr_idx].addr = dev_addr;
- tr_req[tr_idx].icnt0 = tr_cnt0;
- tr_req[tr_idx].icnt1 = tr_cnt1;
- tr_req[tr_idx].icnt2 = tr1_cnt2;
- tr_req[tr_idx].icnt3 = 1;
- tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
-
- tr_req[tr_idx].daddr = sg_addr;
- tr_req[tr_idx].dicnt0 = tr_cnt0;
- tr_req[tr_idx].dicnt1 = tr_cnt1;
- tr_req[tr_idx].dicnt2 = tr1_cnt2;
- tr_req[tr_idx].dicnt3 = 1;
- tr_req[tr_idx].ddim1 = tr_cnt0;
- tr_req[tr_idx].ddim2 = trigger_size;
- } else {
- tr_req[tr_idx].addr = sg_addr;
- tr_req[tr_idx].icnt0 = tr_cnt0;
- tr_req[tr_idx].icnt1 = tr_cnt1;
- tr_req[tr_idx].icnt2 = tr1_cnt2;
- tr_req[tr_idx].icnt3 = 1;
- tr_req[tr_idx].dim1 = tr_cnt0;
- tr_req[tr_idx].dim2 = trigger_size;
-
- tr_req[tr_idx].daddr = dev_addr;
- tr_req[tr_idx].dicnt0 = tr_cnt0;
- tr_req[tr_idx].dicnt1 = tr_cnt1;
- tr_req[tr_idx].dicnt2 = tr1_cnt2;
- tr_req[tr_idx].dicnt3 = 1;
- tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
- }
- tr_idx++;
- }
-
- d->residue += sg_len;
- }
-
- cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, csf | CPPI5_TR_CSF_EOP);
-
- return d;
-}
-
-static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d,
- enum dma_slave_buswidth dev_width,
- u16 elcnt)
-{
- if (uc->config.ep_type != PSIL_EP_PDMA_XY)
- return 0;
-
- /* Bus width translates to the element size (ES) */
- switch (dev_width) {
- case DMA_SLAVE_BUSWIDTH_1_BYTE:
- d->static_tr.elsize = 0;
- break;
- case DMA_SLAVE_BUSWIDTH_2_BYTES:
- d->static_tr.elsize = 1;
- break;
- case DMA_SLAVE_BUSWIDTH_3_BYTES:
- d->static_tr.elsize = 2;
- break;
- case DMA_SLAVE_BUSWIDTH_4_BYTES:
- d->static_tr.elsize = 3;
- break;
- case DMA_SLAVE_BUSWIDTH_8_BYTES:
- d->static_tr.elsize = 4;
- break;
- default: /* not reached */
- return -EINVAL;
- }
-
- d->static_tr.elcnt = elcnt;
-
- if (uc->config.pkt_mode || !uc->cyclic) {
- /*
- * PDMA must close the packet when the channel is in packet mode.
- * For TR mode when the channel is not cyclic we also need PDMA
- * to close the packet otherwise the transfer will stall because
- * PDMA holds on the data it has received from the peripheral.
- */
- unsigned int div = dev_width * elcnt;
-
- if (uc->cyclic)
- d->static_tr.bstcnt = d->residue / d->sglen / div;
- else
- d->static_tr.bstcnt = d->residue / div;
- } else if (uc->ud->match_data->type == DMA_TYPE_BCDMA &&
- uc->config.dir == DMA_DEV_TO_MEM &&
- uc->cyclic) {
- /*
- * For cyclic mode with BCDMA we have to set EOP in each TR to
- * prevent short packet errors seen on channel teardown. So the
- * PDMA must close the packet after every TR transfer by setting
- * burst count equal to the number of bytes transferred.
- */
- struct cppi5_tr_type1_t *tr_req = d->hwdesc[0].tr_req_base;
-
- d->static_tr.bstcnt =
- (tr_req->icnt0 * tr_req->icnt1) / dev_width;
- } else {
- d->static_tr.bstcnt = 0;
- }
-
- if (uc->config.dir == DMA_DEV_TO_MEM &&
- d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask)
- return -EINVAL;
-
- return 0;
-}
-
-static struct udma_desc *
-udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl,
- unsigned int sglen, enum dma_transfer_direction dir,
- unsigned long tx_flags, void *context)
-{
- struct scatterlist *sgent;
- struct cppi5_host_desc_t *h_desc = NULL;
- struct udma_desc *d;
- u32 ring_id;
- unsigned int i;
- u64 asel;
-
- d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT);
- if (!d)
- return NULL;
-
- d->sglen = sglen;
- d->hwdesc_count = sglen;
-
- if (dir == DMA_DEV_TO_MEM)
- ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
- else
- ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
-
- if (uc->ud->match_data->type == DMA_TYPE_UDMA)
- asel = 0;
- else
- asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
-
- for_each_sg(sgl, sgent, sglen, i) {
- struct udma_hwdesc *hwdesc = &d->hwdesc[i];
- dma_addr_t sg_addr = sg_dma_address(sgent);
- struct cppi5_host_desc_t *desc;
- size_t sg_len = sg_dma_len(sgent);
-
- hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
- GFP_NOWAIT,
- &hwdesc->cppi5_desc_paddr);
- if (!hwdesc->cppi5_desc_vaddr) {
- dev_err(uc->ud->dev,
- "descriptor%d allocation failed\n", i);
-
- udma_free_hwdesc(uc, d);
- kfree(d);
- return NULL;
- }
-
- d->residue += sg_len;
- hwdesc->cppi5_desc_size = uc->config.hdesc_size;
- desc = hwdesc->cppi5_desc_vaddr;
-
- if (i == 0) {
- cppi5_hdesc_init(desc, 0, 0);
- /* Flow and Packed ID */
- cppi5_desc_set_pktids(&desc->hdr, uc->id,
- CPPI5_INFO1_DESC_FLOWID_DEFAULT);
- cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id);
- } else {
- cppi5_hdesc_reset_hbdesc(desc);
- cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff);
- }
-
- /* attach the sg buffer to the descriptor */
- sg_addr |= asel;
- cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len);
-
- /* Attach link as host buffer descriptor */
- if (h_desc)
- cppi5_hdesc_link_hbdesc(h_desc,
- hwdesc->cppi5_desc_paddr | asel);
-
- if (uc->ud->match_data->type == DMA_TYPE_PKTDMA ||
- dir == DMA_MEM_TO_DEV)
- h_desc = desc;
- }
-
- if (d->residue >= SZ_4M) {
- dev_err(uc->ud->dev,
- "%s: Transfer size %u is over the supported 4M range\n",
- __func__, d->residue);
- udma_free_hwdesc(uc, d);
- kfree(d);
- return NULL;
- }
-
- h_desc = d->hwdesc[0].cppi5_desc_vaddr;
- cppi5_hdesc_set_pktlen(h_desc, d->residue);
-
- return d;
-}
-
-static int udma_attach_metadata(struct dma_async_tx_descriptor *desc,
- void *data, size_t len)
-{
- struct udma_desc *d = to_udma_desc(desc);
- struct udma_chan *uc = to_udma_chan(desc->chan);
- struct cppi5_host_desc_t *h_desc;
- u32 psd_size = len;
- u32 flags = 0;
-
- if (!uc->config.pkt_mode || !uc->config.metadata_size)
- return -ENOTSUPP;
-
- if (!data || len > uc->config.metadata_size)
- return -EINVAL;
-
- if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE)
- return -EINVAL;
-
- h_desc = d->hwdesc[0].cppi5_desc_vaddr;
- if (d->dir == DMA_MEM_TO_DEV)
- memcpy(h_desc->epib, data, len);
-
- if (uc->config.needs_epib)
- psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
-
- d->metadata = data;
- d->metadata_size = len;
- if (uc->config.needs_epib)
- flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
-
- cppi5_hdesc_update_flags(h_desc, flags);
- cppi5_hdesc_update_psdata_size(h_desc, psd_size);
-
- return 0;
-}
-
-static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
- size_t *payload_len, size_t *max_len)
-{
- struct udma_desc *d = to_udma_desc(desc);
- struct udma_chan *uc = to_udma_chan(desc->chan);
- struct cppi5_host_desc_t *h_desc;
-
- if (!uc->config.pkt_mode || !uc->config.metadata_size)
- return ERR_PTR(-ENOTSUPP);
-
- h_desc = d->hwdesc[0].cppi5_desc_vaddr;
-
- *max_len = uc->config.metadata_size;
-
- *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ?
- CPPI5_INFO0_HDESC_EPIB_SIZE : 0;
- *payload_len += cppi5_hdesc_get_psdata_size(h_desc);
-
- return h_desc->epib;
-}
-
-static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
- size_t payload_len)
-{
- struct udma_desc *d = to_udma_desc(desc);
- struct udma_chan *uc = to_udma_chan(desc->chan);
- struct cppi5_host_desc_t *h_desc;
- u32 psd_size = payload_len;
- u32 flags = 0;
-
- if (!uc->config.pkt_mode || !uc->config.metadata_size)
- return -ENOTSUPP;
-
- if (payload_len > uc->config.metadata_size)
- return -EINVAL;
-
- if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE)
- return -EINVAL;
-
- h_desc = d->hwdesc[0].cppi5_desc_vaddr;
-
- if (uc->config.needs_epib) {
- psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
- flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
- }
-
- cppi5_hdesc_update_flags(h_desc, flags);
- cppi5_hdesc_update_psdata_size(h_desc, psd_size);
-
- return 0;
-}
-
-static struct dma_descriptor_metadata_ops metadata_ops = {
- .attach = udma_attach_metadata,
- .get_ptr = udma_get_metadata_ptr,
- .set_len = udma_set_metadata_len,
-};
-
-static struct dma_async_tx_descriptor *
-udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
- unsigned int sglen, enum dma_transfer_direction dir,
- unsigned long tx_flags, void *context)
-{
- struct udma_chan *uc = to_udma_chan(chan);
- enum dma_slave_buswidth dev_width;
- struct udma_desc *d;
- u32 burst;
-
- if (dir != uc->config.dir &&
- (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type)) {
- dev_err(chan->device->dev,
- "%s: chan%d is for %s, not supporting %s\n",
- __func__, uc->id,
- dmaengine_get_direction_text(uc->config.dir),
- dmaengine_get_direction_text(dir));
- return NULL;
- }
-
- if (dir == DMA_DEV_TO_MEM) {
- dev_width = uc->cfg.src_addr_width;
- burst = uc->cfg.src_maxburst;
- } else if (dir == DMA_MEM_TO_DEV) {
- dev_width = uc->cfg.dst_addr_width;
- burst = uc->cfg.dst_maxburst;
- } else {
- dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
- return NULL;
- }
-
- if (!burst)
- burst = 1;
-
- uc->config.tx_flags = tx_flags;
-
- if (uc->config.pkt_mode)
- d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags,
- context);
- else if (is_slave_direction(uc->config.dir))
- d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags,
- context);
- else
- d = udma_prep_slave_sg_triggered_tr(uc, sgl, sglen, dir,
- tx_flags, context);
-
- if (!d)
- return NULL;
-
- d->dir = dir;
- d->desc_idx = 0;
- d->tr_idx = 0;
-
- /* static TR for remote PDMA */
- if (udma_configure_statictr(uc, d, dev_width, burst)) {
- dev_err(uc->ud->dev,
- "%s: StaticTR Z is limited to maximum %u (%u)\n",
- __func__, uc->ud->match_data->statictr_z_mask,
- d->static_tr.bstcnt);
-
- udma_free_hwdesc(uc, d);
- kfree(d);
- return NULL;
- }
-
- if (uc->config.metadata_size)
- d->vd.tx.metadata_ops = &metadata_ops;
-
- return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
-}
-
-static struct udma_desc *
-udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr,
- size_t buf_len, size_t period_len,
- enum dma_transfer_direction dir, unsigned long flags)
-{
- struct udma_desc *d;
- size_t tr_size, period_addr;
- struct cppi5_tr_type1_t *tr_req;
- unsigned int periods = buf_len / period_len;
- u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
- unsigned int i;
- int num_tr;
- u32 period_csf = 0;
-
- num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0,
- &tr0_cnt1, &tr1_cnt0);
- if (num_tr < 0) {
- dev_err(uc->ud->dev, "size %zu is not supported\n",
- period_len);
- return NULL;
- }
-
- /* Now allocate and setup the descriptor. */
- tr_size = sizeof(struct cppi5_tr_type1_t);
- d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir);
- if (!d)
- return NULL;
-
- tr_req = d->hwdesc[0].tr_req_base;
- if (uc->ud->match_data->type == DMA_TYPE_UDMA)
- period_addr = buf_addr;
- else
- period_addr = buf_addr |
- ((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT);
-
- /*
- * For BCDMA <-> PDMA transfers, the EOP flag needs to be set on the
- * last TR of a descriptor, to mark the packet as complete.
- * This is required for getting the teardown completion message in case
- * of TX, and to avoid short-packet error in case of RX.
- *
- * As we are in cyclic mode, we do not know which period might be the
- * last one, so set the flag for each period.
- */
- if (uc->config.ep_type == PSIL_EP_PDMA_XY &&
- uc->ud->match_data->type == DMA_TYPE_BCDMA) {
- period_csf = CPPI5_TR_CSF_EOP;
- }
-
- for (i = 0; i < periods; i++) {
- int tr_idx = i * num_tr;
-
- cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
- false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
-
- tr_req[tr_idx].addr = period_addr;
- tr_req[tr_idx].icnt0 = tr0_cnt0;
- tr_req[tr_idx].icnt1 = tr0_cnt1;
- tr_req[tr_idx].dim1 = tr0_cnt0;
-
- if (num_tr == 2) {
- cppi5_tr_csf_set(&tr_req[tr_idx].flags,
- CPPI5_TR_CSF_SUPR_EVT);
- tr_idx++;
-
- cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
- false, false,
- CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
-
- tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0;
- tr_req[tr_idx].icnt0 = tr1_cnt0;
- tr_req[tr_idx].icnt1 = 1;
- tr_req[tr_idx].dim1 = tr1_cnt0;
- }
-
- if (!(flags & DMA_PREP_INTERRUPT))
- period_csf |= CPPI5_TR_CSF_SUPR_EVT;
-
- if (period_csf)
- cppi5_tr_csf_set(&tr_req[tr_idx].flags, period_csf);
-
- period_addr += period_len;
- }
-
- return d;
-}
-
-static struct udma_desc *
-udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr,
- size_t buf_len, size_t period_len,
- enum dma_transfer_direction dir, unsigned long flags)
-{
- struct udma_desc *d;
- u32 ring_id;
- int i;
- int periods = buf_len / period_len;
-
- if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1))
- return NULL;
-
- if (period_len >= SZ_4M)
- return NULL;
-
- d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT);
- if (!d)
- return NULL;
-
- d->hwdesc_count = periods;
-
- /* TODO: re-check this... */
- if (dir == DMA_DEV_TO_MEM)
- ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
- else
- ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
-
- if (uc->ud->match_data->type != DMA_TYPE_UDMA)
- buf_addr |= (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
-
- for (i = 0; i < periods; i++) {
- struct udma_hwdesc *hwdesc = &d->hwdesc[i];
- dma_addr_t period_addr = buf_addr + (period_len * i);
- struct cppi5_host_desc_t *h_desc;
-
- hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
- GFP_NOWAIT,
- &hwdesc->cppi5_desc_paddr);
- if (!hwdesc->cppi5_desc_vaddr) {
- dev_err(uc->ud->dev,
- "descriptor%d allocation failed\n", i);
-
- udma_free_hwdesc(uc, d);
- kfree(d);
- return NULL;
- }
-
- hwdesc->cppi5_desc_size = uc->config.hdesc_size;
- h_desc = hwdesc->cppi5_desc_vaddr;
-
- cppi5_hdesc_init(h_desc, 0, 0);
- cppi5_hdesc_set_pktlen(h_desc, period_len);
-
- /* Flow and Packed ID */
- cppi5_desc_set_pktids(&h_desc->hdr, uc->id,
- CPPI5_INFO1_DESC_FLOWID_DEFAULT);
- cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id);
-
- /* attach each period to a new descriptor */
- cppi5_hdesc_attach_buf(h_desc,
- period_addr, period_len,
- period_addr, period_len);
- }
-
- return d;
-}
-
-static struct dma_async_tx_descriptor *
-udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
- size_t period_len, enum dma_transfer_direction dir,
- unsigned long flags)
-{
- struct udma_chan *uc = to_udma_chan(chan);
- enum dma_slave_buswidth dev_width;
- struct udma_desc *d;
- u32 burst;
-
- if (dir != uc->config.dir) {
- dev_err(chan->device->dev,
- "%s: chan%d is for %s, not supporting %s\n",
- __func__, uc->id,
- dmaengine_get_direction_text(uc->config.dir),
- dmaengine_get_direction_text(dir));
- return NULL;
- }
-
- uc->cyclic = true;
-
- if (dir == DMA_DEV_TO_MEM) {
- dev_width = uc->cfg.src_addr_width;
- burst = uc->cfg.src_maxburst;
- } else if (dir == DMA_MEM_TO_DEV) {
- dev_width = uc->cfg.dst_addr_width;
- burst = uc->cfg.dst_maxburst;
- } else {
- dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
- return NULL;
- }
-
- if (!burst)
- burst = 1;
-
- if (uc->config.pkt_mode)
- d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len,
- dir, flags);
- else
- d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len,
- dir, flags);
-
- if (!d)
- return NULL;
-
- d->sglen = buf_len / period_len;
-
- d->dir = dir;
- d->residue = buf_len;
-
- /* static TR for remote PDMA */
- if (udma_configure_statictr(uc, d, dev_width, burst)) {
- dev_err(uc->ud->dev,
- "%s: StaticTR Z is limited to maximum %u (%u)\n",
- __func__, uc->ud->match_data->statictr_z_mask,
- d->static_tr.bstcnt);
-
- udma_free_hwdesc(uc, d);
- kfree(d);
- return NULL;
- }
-
- if (uc->config.metadata_size)
- d->vd.tx.metadata_ops = &metadata_ops;
-
- return vchan_tx_prep(&uc->vc, &d->vd, flags);
-}
-
-static struct dma_async_tx_descriptor *
-udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
- size_t len, unsigned long tx_flags)
-{
- struct udma_chan *uc = to_udma_chan(chan);
- struct udma_desc *d;
- struct cppi5_tr_type15_t *tr_req;
- int num_tr;
- size_t tr_size = sizeof(struct cppi5_tr_type15_t);
- u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
- u32 csf = CPPI5_TR_CSF_SUPR_EVT;
-
- if (uc->config.dir != DMA_MEM_TO_MEM) {
- dev_err(chan->device->dev,
- "%s: chan%d is for %s, not supporting %s\n",
- __func__, uc->id,
- dmaengine_get_direction_text(uc->config.dir),
- dmaengine_get_direction_text(DMA_MEM_TO_MEM));
- return NULL;
- }
-
- num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0,
- &tr0_cnt1, &tr1_cnt0);
- if (num_tr < 0) {
- dev_err(uc->ud->dev, "size %zu is not supported\n",
- len);
- return NULL;
- }
-
- d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM);
- if (!d)
- return NULL;
-
- d->dir = DMA_MEM_TO_MEM;
- d->desc_idx = 0;
- d->tr_idx = 0;
- d->residue = len;
-
- if (uc->ud->match_data->type != DMA_TYPE_UDMA) {
- src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
- dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
- } else {
- csf |= CPPI5_TR_CSF_EOL_ICNT0;
- }
-
- tr_req = d->hwdesc[0].tr_req_base;
-
- cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
- CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
- cppi5_tr_csf_set(&tr_req[0].flags, csf);
-
- tr_req[0].addr = src;
- tr_req[0].icnt0 = tr0_cnt0;
- tr_req[0].icnt1 = tr0_cnt1;
- tr_req[0].icnt2 = 1;
- tr_req[0].icnt3 = 1;
- tr_req[0].dim1 = tr0_cnt0;
-
- tr_req[0].daddr = dest;
- tr_req[0].dicnt0 = tr0_cnt0;
- tr_req[0].dicnt1 = tr0_cnt1;
- tr_req[0].dicnt2 = 1;
- tr_req[0].dicnt3 = 1;
- tr_req[0].ddim1 = tr0_cnt0;
-
- if (num_tr == 2) {
- cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
- CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
- cppi5_tr_csf_set(&tr_req[1].flags, csf);
-
- tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
- tr_req[1].icnt0 = tr1_cnt0;
- tr_req[1].icnt1 = 1;
- tr_req[1].icnt2 = 1;
- tr_req[1].icnt3 = 1;
-
- tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
- tr_req[1].dicnt0 = tr1_cnt0;
- tr_req[1].dicnt1 = 1;
- tr_req[1].dicnt2 = 1;
- tr_req[1].dicnt3 = 1;
- }
-
- cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, csf | CPPI5_TR_CSF_EOP);
-
- if (uc->config.metadata_size)
- d->vd.tx.metadata_ops = &metadata_ops;
-
- return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
-}
-
static void udma_issue_pending(struct dma_chan *chan)
{
struct udma_chan *uc = to_udma_chan(chan);
@@ -3587,46 +2409,6 @@ static void udma_synchronize(struct dma_chan *chan)
udma_reset_rings(uc);
}
-static void udma_desc_pre_callback(struct virt_dma_chan *vc,
- struct virt_dma_desc *vd,
- struct dmaengine_result *result)
-{
- struct udma_chan *uc = to_udma_chan(&vc->chan);
- struct udma_desc *d;
- u8 status;
-
- if (!vd)
- return;
-
- d = to_udma_desc(&vd->tx);
-
- if (d->metadata_size)
- udma_fetch_epib(uc, d);
-
- if (result) {
- void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx);
-
- if (cppi5_desc_get_type(desc_vaddr) ==
- CPPI5_INFO0_DESC_TYPE_VAL_HOST) {
- /* Provide residue information for the client */
- result->residue = d->residue -
- cppi5_hdesc_get_pktlen(desc_vaddr);
- if (result->residue)
- result->result = DMA_TRANS_ABORTED;
- else
- result->result = DMA_TRANS_NOERROR;
- } else {
- result->residue = 0;
- /* Propagate TR Response errors to the client */
- status = d->hwdesc[0].tr_resp_base->status;
- if (status)
- result->result = DMA_TRANS_ABORTED;
- else
- result->result = DMA_TRANS_NOERROR;
- }
- }
-}
-
/*
* This tasklet handles the completion of a DMA descriptor by
* calling its callback and freeing it.
diff --git a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h
index 3a786b3eddc67..7c807bd9e178b 100644
--- a/drivers/dma/ti/k3-udma.h
+++ b/drivers/dma/ti/k3-udma.h
@@ -556,6 +556,60 @@ static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d)
memcpy(d->metadata, h_desc->epib, d->metadata_size);
}
+/* Common functions */
+struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,
+ dma_addr_t paddr);
+void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d);
+void udma_purge_desc_work(struct work_struct *work);
+void udma_desc_free(struct virt_dma_desc *vd);
+bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr);
+bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d);
+struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc,
+ size_t tr_size, int tr_count,
+ enum dma_transfer_direction dir);
+int udma_get_tr_counters(size_t len, unsigned long align_to,
+ u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0);
+struct udma_desc *udma_prep_slave_sg_tr(struct udma_chan *uc,
+ struct scatterlist *sgl, unsigned int sglen,
+ enum dma_transfer_direction dir,
+ unsigned long tx_flags, void *context);
+struct udma_desc *udma_prep_slave_sg_triggered_tr(struct udma_chan *uc,
+ struct scatterlist *sgl, unsigned int sglen,
+ enum dma_transfer_direction dir,
+ unsigned long tx_flags, void *context);
+int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d,
+ enum dma_slave_buswidth dev_width, u16 elcnt);
+struct udma_desc *udma_prep_slave_sg_pkt(struct udma_chan *uc,
+ struct scatterlist *sgl, unsigned int sglen,
+ enum dma_transfer_direction dir,
+ unsigned long tx_flags, void *context);
+int udma_attach_metadata(struct dma_async_tx_descriptor *desc,
+ void *data, size_t len);
+void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
+ size_t *payload_len, size_t *max_len);
+int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
+ size_t payload_len);
+struct dma_async_tx_descriptor *udma_prep_slave_sg(struct dma_chan *chan,
+ struct scatterlist *sgl, unsigned int sglen,
+ enum dma_transfer_direction dir,
+ unsigned long tx_flags, void *context);
+struct udma_desc *udma_prep_dma_cyclic_tr(struct udma_chan *uc,
+ dma_addr_t buf_addr, size_t buf_len, size_t period_len,
+ enum dma_transfer_direction dir, unsigned long flags);
+struct udma_desc *udma_prep_dma_cyclic_pkt(struct udma_chan *uc,
+ dma_addr_t buf_addr, size_t buf_len, size_t period_len,
+ enum dma_transfer_direction dir, unsigned long flags);
+struct dma_async_tx_descriptor *udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
+ size_t buf_len, size_t period_len,
+ enum dma_transfer_direction dir,
+ unsigned long flags);
+struct dma_async_tx_descriptor *udma_prep_dma_memcpy(struct dma_chan *chan,
+ dma_addr_t dest, dma_addr_t src,
+ size_t len, unsigned long tx_flags);
+void udma_desc_pre_callback(struct virt_dma_chan *vc,
+ struct virt_dma_desc *vd,
+ struct dmaengine_result *result);
+
/* Direct access to UDMA low lever resources for the glue layer */
int xudma_navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread);
int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
--
2.34.1
Powered by blists - more mailing lists