| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20251211-siva_mhi_dp2-v1-11-d2895c4ec73a@oss.qualcomm.com>
Date: Thu, 11 Dec 2025 13:37:43 +0530
From: Sivareddy Surasani <sivareddy.surasani@....qualcomm.com>
To: Manivannan Sadhasivam <mani@...nel.org>, Jonathan Corbet <corbet@....net>,
Arnd Bergmann <arnd@...db.de>,
Greg Kroah-Hartman <gregkh@...uxfoundation.org>
Cc: mhi@...ts.linux.dev, linux-arm-msm@...r.kernel.org,
linux-kernel@...r.kernel.org, linux-doc@...r.kernel.org,
Upal Kumar Saha <upal.saha@....qualcomm.com>,
Himanshu Shukla <quic_himashuk@...cinc.com>,
Sivareddy Surasani <sivareddy.surasani@....qualcomm.com>,
Vivek Pernamitta <vivek.pernamitta@....qualcomm.com>
Subject: [PATCH 11/11] char: qcom_csm_dp: Add data path driver for QDU100
device
Add a character device driver for the Qualcomm Cell Site Modem (CSM)
Data Path (DP) interface, required to support the QDU100 5G
distributed unit in cellular base station deployments.
Implement high-performance communication between the Layer 2
host (x86) and Qualcomm Distributed Unit (QDU100) by enabling
transmission and reception of FAPI packets over PCIe using the
Modem Host Interface (MHI).
Create an efficient zero-copy mechanism using shared rings and
memory pools to eliminate data copying between user and kernel space,
allowing high data rates with low latency.
Register as an MHI client and provide a character-based interface
to userspace via ioctls for memory pool management and packet
transmission. Support two DMA channels (control and data) with system
configuration to ensure proper channel assignment.
Implement Single Root I/O Virtualization (SR-IOV) support, allowing
the QDU100 to present itself as multiple virtual PCIe functions to
the host. Support up to 12 QDU100 devices with up to 4 virtual
functions per device.
FAPI:
https://www.techplayon.com/5g-fapi-femtocell-application-programming-interface/
dp-lib userspace:
https://github.com/qualcomm/dp-lib/tree/dp-driver-upstream-specific
Signed-off-by: Sivareddy Surasani <sivareddy.surasani@....qualcomm.com>
---
Documentation/misc-devices/qcom_csm_dp.rst | 138 +++
drivers/char/Kconfig | 2 +
drivers/char/Makefile | 1 +
drivers/char/qcom_csm_dp/Kconfig | 9 +
drivers/char/qcom_csm_dp/Makefile | 5 +
drivers/char/qcom_csm_dp/qcom_csm_dp.h | 173 ++++
drivers/char/qcom_csm_dp/qcom_csm_dp_cdev.c | 941 +++++++++++++++++++++
drivers/char/qcom_csm_dp/qcom_csm_dp_core.c | 571 +++++++++++++
drivers/char/qcom_csm_dp/qcom_csm_dp_debugfs.c | 993 ++++++++++++++++++++++
drivers/char/qcom_csm_dp/qcom_csm_dp_mem.c | 1078 ++++++++++++++++++++++++
drivers/char/qcom_csm_dp/qcom_csm_dp_mem.h | 292 +++++++
drivers/char/qcom_csm_dp/qcom_csm_dp_mhi.c | 651 ++++++++++++++
drivers/char/qcom_csm_dp/qcom_csm_dp_mhi.h | 81 ++
include/uapi/linux/qcom_csm_dp_ioctl.h | 306 +++++++
14 files changed, 5241 insertions(+)
diff --git a/Documentation/misc-devices/qcom_csm_dp.rst b/Documentation/misc-devices/qcom_csm_dp.rst
new file mode 100644
index 000000000000..88051dadabda
--- /dev/null
+++ b/Documentation/misc-devices/qcom_csm_dp.rst
@@ -0,0 +1,138 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=====================================
+Qualcomm QDU100 device CSM_DP driver
+=====================================
+
+CSM-DP stands for Cell Site Modem Data Path. Specifically designed to support
+high-performance data transmission between a host (typically an x86 server
+running Layer 2 software) and the distributed unit (QDU100).
+
+The CSM-DP driver enables the transmission and reception of FAPI (Functional
+Application Platform Interface) packets both control and data—between the L2
+host and the QDU.
+
+All data path traffic is transferred over the Modem Host Interface (MHI),
+with PCIe as the physical transport layer.
+
+Block Diagram
+=============
+
+ -------------------------------------------------------------------------------------------
+ | +-------------------+ +-------------------+ mmap +-------------------+ |
+ | | L2 Application |------->| DP-Lib |----------->| CharDev Interface | |
+ | | (dp_ping) | | |----------->| | |
+ | +-------------------+ +-------------------+ ioctl +-------------------+ |
+ | | |
+ | User Space | |
+ ---------------------------------------------------------------------+---------------------
+ |
+ -----------------------------------------------------------|-------------------------------------------------------------
+ | V |
+ | CSM-DP Driver +-------------------+ |
+ | | CSM-DP Core | |
+ | +-------------------+ |
+ | | |
+ | | |
+ | +----------------------------------+----------------------------------------+ |
+ | | | | |
+ | | | | |
+ | | | | |
+ | V V V |
+ | +-------------------+ +----------------------------------+ +-------------------+ |
+ | | CSM-DP Memory | | CSM-DP | | CSM-DP Sysfs | |
+ | | UL/DL Allocation | | MHI Client Interface | | Debug Interface | |
+ | +-------------------+ +----------------------------------+ +-------------------+ |
+ | | | |
+ | | | |
+ | | | |
+ | V V |
+ | +------------+ +------------+ |
+ | | IP_HW1 | | IP_HW2 | |
+ | | (Control) | | (Data) | |
+ | +------------+ +------------+ |
+ | | | |
+ ------------------------------------------------+---------------------+--------------------------------------------------
+ | |
+ | |
+ ------------------------------------------------+---------------------+--------------------------------------------------
+ | V V |
+ | +----------------------------------+ |
+ | | MHI Controllers (VF1…VF4) | |
+ | +----------------------------------+ |
+ | | |
+ | MHI Driver | (PCI-E Interface) |
+ ----------------------------------------------------------+--------------------------------------------------------------
+ V
+ +----------------------------------+
+ | QDU100 |
+ +----------------------------------+
+
+
+Supported chips
+---------------
+
+- QDU100
+
+Driver location
+===============
+
+drivers/char/qcom_csm_dp/qcom_csm_dp_core.c
+
+Driver type definitions
+=======================
+
+include/uapi/linux/qcom_csm_dp_ioctl.h
+
+Driver IOCTLs
+=============
+
+:c:macro::`CSM_DP_IOCTL_MEMPOOL_ALLOC`
+Mempool allocation for UL/DL.
+
+:c:macro::`CSM_DP_IOCTL_MEMPOOL_GET_CONFIG`
+Returns the allocated mempool config UL/DL.
+
+:c:macro::`CSM_DP_IOCTL_RX_GET_CONFIG`
+Returns the RX Queue configuration for UL Control channel.
+
+:c:macro::`CSM_DP_IOCTL_TX`
+To transmit UL data.
+
+:c:macro::`CSM_DP_IOCTL_SG_TX`
+To transmit UL data in Scatter Gather mode.
+
+:c:macro::`CSM_DP_IOCTL_RX_POLL`
+Poll operation for UL Data Channel.
+
+:c:macro::`CSM_DP_IOCTL_GET_STATS`
+Returns the UL/DL Packet stats information.
+
+CSM_DP Driver
+============
+
+The CSM_DP driver functions as a client driver for the
+MHI device. It utilizes MHI channels 104, 105, 106, and 107,
+where:
+ - Channels 104 and 105 are used for FAPI control packets.
+ - Channels 106 and 107 are used for FAPI data packets.
+The driver supports multiple Virtual Functions (VFs) to enable
+scalable and efficient communication.
+
+See available QDU100 devices(PF/VF) on PCIe bus::
+
+ # lspci | grep Qualcomm
+
+See available CSM_DP devices upon probe::
+
+ # ls /dev/csm*
+ /dev/csm1-dp1 /dev/csm2-dp2 /dev/csm3-dp3 /dev/csm4-dp4
+
+CSM DP_LIB
+============
+Clone, Build and Install dp-lib code from following repository.
+https://github.com/qualcomm/dp-lib/tree/dp-driver-upstream-specific
+
+Run the dp_ping application for data traffic::
+
+ # dp_ping -B <BUS NUM> -V <VF NUM>
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index d2cfc584e202..60db50049c7c 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -411,6 +411,8 @@ source "drivers/s390/char/Kconfig"
source "drivers/char/xillybus/Kconfig"
+source "drivers/char/qcom_csm_dp/Kconfig"
+
config ADI
tristate "SPARC Privileged ADI driver"
depends on SPARC64
diff --git a/drivers/char/Makefile b/drivers/char/Makefile
index 1291369b9126..d85a6fa16b03 100644
--- a/drivers/char/Makefile
+++ b/drivers/char/Makefile
@@ -44,3 +44,4 @@ obj-$(CONFIG_PS3_FLASH) += ps3flash.o
obj-$(CONFIG_XILLYBUS_CLASS) += xillybus/
obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o
obj-$(CONFIG_ADI) += adi.o
+obj-$(CONFIG_QCOM_CSM_DP) += qcom_csm_dp/
diff --git a/drivers/char/qcom_csm_dp/Kconfig b/drivers/char/qcom_csm_dp/Kconfig
new file mode 100644
index 000000000000..472a5defe585
--- /dev/null
+++ b/drivers/char/qcom_csm_dp/Kconfig
@@ -0,0 +1,9 @@
+menuconfig QCOM_CSM_DP
+ tristate "CSM DP Interface Core"
+ depends on MHI_BUS
+ help
+ CSM Data Path (DP) driver is used to support the
+ transmission and reception of functional application platform
+ interface (FAPI) packets (control and data) between the L2 host (x86)
+ and the Qualcomm Distributed Unit (QDU100).
+ The CSM DP character driver provides a datapath server to user space.
diff --git a/drivers/char/qcom_csm_dp/Makefile b/drivers/char/qcom_csm_dp/Makefile
new file mode 100644
index 000000000000..e345844d3483
--- /dev/null
+++ b/drivers/char/qcom_csm_dp/Makefile
@@ -0,0 +1,5 @@
+obj-$(CONFIG_QCOM_CSM_DP) += qcom_csm_dp.o
+
+qcom_csm_dp-objs := qcom_csm_dp_core.o qcom_csm_dp_cdev.o
+qcom_csm_dp-objs += qcom_csm_dp_mhi.o qcom_csm_dp_mem.o
+qcom_csm_dp-objs += qcom_csm_dp_debugfs.o
diff --git a/drivers/char/qcom_csm_dp/qcom_csm_dp.h b/drivers/char/qcom_csm_dp/qcom_csm_dp.h
new file mode 100644
index 000000000000..da9ce499da35
--- /dev/null
+++ b/drivers/char/qcom_csm_dp/qcom_csm_dp.h
@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __QCOM_CSM_DP__
+#define __QCOM_CSM_DP__
+
+#include <linux/atomic.h>
+#include <linux/cdev.h>
+#include <linux/device.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/netdevice.h>
+#include <linux/qcom_csm_dp_ioctl.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+
+#include "qcom_csm_dp_mem.h"
+#include "qcom_csm_dp_mhi.h"
+
+#define CSM_DP_MODULE_NAME "csm-dp"
+#define CSM_DP_DEV_CLASS_NAME CSM_DP_MODULE_NAME
+#define CSM_DP_CDEV_NAME CSM_DP_MODULE_NAME
+#define CSM_DP_NAPI_WEIGHT 64
+#define CSM_DP_DMA_MASK 40
+
+#define CSM_DP_MMAP_MEM_TYPE_SHIFT 24
+#define CSM_DP_MMAP_MEM_TYPE_MASK 0xFF
+#define CSM_DP_MMAP_TYPE_SHIFT 16
+#define CSM_DP_MMAP_TYPE_MASK 0xFF
+
+#define CSM_DP_MMAP_COOKIE(type, target) \
+ ((((type) & CSM_DP_MMAP_MEM_TYPE_MASK) << CSM_DP_MMAP_MEM_TYPE_SHIFT) | \
+ (((target) & CSM_DP_MMAP_TYPE_MASK) << CSM_DP_MMAP_TYPE_SHIFT))
+
+#define CSM_DP_MMAP_COOKIE_TO_MEM_TYPE(cookie) \
+ (((cookie) >> CSM_DP_MMAP_MEM_TYPE_SHIFT) & CSM_DP_MMAP_MEM_TYPE_MASK)
+
+#define CSM_DP_MMAP_COOKIE_TO_TYPE(cookie) \
+ (((cookie) >> CSM_DP_MMAP_TYPE_SHIFT) & CSM_DP_MMAP_TYPE_MASK)
+
+#define CSM_DP_MMAP_RX_COOKIE(type) \
+ CSM_DP_MMAP_COOKIE((type) + CSM_DP_MEM_TYPE_LAST, CSM_DP_MMAP_TYPE_RING)
+
+#define CSM_DP_MMAP_RX_COOKIE_TO_TYPE(cookie) \
+ (CSM_DP_MMAP_COOKIE_TO_MEM_TYPE(cookie) - CSM_DP_MEM_TYPE_LAST)
+
+#define CSM_DP_TX_FLAG_SG 0x01
+
+#define CSM_DP_MAX_NUM_BUSES 12 /* max supported QDU100 devices connected to this Host */
+#define CSM_DP_MAX_NUM_VFS 4 /* max Virtual Functions that single QDU100 device can expose */
+#define CSM_DP_MAX_NUM_DEVS (CSM_DP_MAX_NUM_BUSES * CSM_DP_MAX_NUM_VFS)
+
+#define ch_name(ch) ((ch) == CSM_DP_CH_CONTROL) ? "CONTROL" : "DATA"
+
+/*
+ * vma mapping for mempool which includes
+ * - buffer memory region
+ * - ring buffer shared between kernel and user space
+ * for buffer management
+ */
+struct csm_dp_mempool_vma {
+ struct csm_dp_mempool **pp_mempool;
+ struct vm_area_struct *vma[CSM_DP_MMAP_TYPE_LAST];
+ atomic_t refcnt[CSM_DP_MMAP_TYPE_LAST];
+ bool usr_alloc; /* allocated by user using ioctl */
+};
+
+/* vma mapping for receive queue */
+struct csm_dp_rxqueue_vma {
+ enum csm_dp_rx_type type;
+ struct vm_area_struct *vma;
+ atomic_t refcnt;
+};
+
+/* RX queue using ring buffer */
+struct csm_dp_rxqueue {
+ enum csm_dp_rx_type type;
+ struct csm_dp_ring *ring;
+ wait_queue_head_t wq;
+ atomic_t refcnt;
+ bool inited;
+};
+
+struct csm_dp_cdev {
+ struct list_head list;
+ struct csm_dp_dev *pdev;
+ pid_t pid;
+
+ /* vma mapping for memory pool */
+ struct csm_dp_mempool_vma mempool_vma[CSM_DP_MEM_TYPE_LAST];
+
+ /* vma mapping for receiving queue */
+ struct csm_dp_rxqueue_vma rxqueue_vma[CSM_DP_RX_TYPE_LAST];
+};
+
+struct csm_dp_core_mem_stats {
+ unsigned long mempool_mem_in_use[CSM_DP_MEM_TYPE_LAST];
+ unsigned long mempool_mem_dma_mapped[CSM_DP_MEM_TYPE_LAST];
+ unsigned long mempool_ring_in_use[CSM_DP_MEM_TYPE_LAST];
+ unsigned long rxq_ring_in_use[CSM_DP_RX_TYPE_LAST];
+};
+
+struct csm_dp_core_stats {
+ unsigned long tx_cnt;
+ unsigned long tx_err;
+ unsigned long tx_drop;
+
+ unsigned long rx_cnt;
+ unsigned long rx_badmsg;
+ unsigned long rx_drop;
+ unsigned long rx_int;
+ unsigned long rx_budget_overflow;
+ unsigned long rx_poll_ignore;
+ unsigned long rx_pending_pkts;
+
+ struct csm_dp_core_mem_stats mem_stats;
+};
+
+struct csm_dp_dev {
+ struct csm_dp_drv *pdrv; /* parent */
+ struct csm_dp_mhi mhi_control_dev; /* control path Tx/Rx */
+ struct csm_dp_mhi mhi_data_dev; /* data path Tx/Rx */
+ bool cdev_inited;
+ pid_t pid;
+ char pid_name[TASK_COMM_LEN + 1];
+ unsigned int bus_num;
+ unsigned int vf_num;
+ struct cdev cdev;
+ struct net_device *dummy_dev;
+ struct napi_struct napi;
+ /* Lock for each VF */
+ struct mutex cdev_lock;
+ struct list_head cdev_head;
+ /* Lock for each Mempool */
+ struct mutex mempool_lock;
+ struct csm_dp_mempool *mempool[CSM_DP_MEM_TYPE_LAST];
+ struct csm_dp_rxqueue rxq[CSM_DP_RX_TYPE_LAST];
+ struct csm_dp_core_stats stats;
+ struct work_struct alloc_work;
+ unsigned int csm_dp_prev_ul_prod_tail;
+
+ struct csm_dp_buf_cntrl *pending_packets;
+};
+
+struct csm_dp_drv {
+ struct device *dev;
+ struct class *dev_class;
+ dev_t devno;
+ struct dentry *dent;
+ struct csm_dp_dev dp_devs[CSM_DP_MAX_NUM_DEVS];
+};
+
+int csm_dp_cdev_init(struct csm_dp_drv *pdrv);
+void csm_dp_cdev_cleanup(struct csm_dp_drv *pdrv);
+int csm_dp_cdev_add(struct csm_dp_dev *pdev, struct device *mhi_dev);
+void csm_dp_cdev_del(struct csm_dp_dev *pdev);
+
+void csm_dp_debugfs_init(struct csm_dp_drv *pdrv);
+void csm_dp_debugfs_cleanup(struct csm_dp_drv *pdrv);
+
+int csm_dp_rx_init(struct csm_dp_dev *pdev);
+void csm_dp_rx_cleanup(struct csm_dp_dev *pdev);
+int csm_dp_tx(struct csm_dp_dev *pdev, enum csm_dp_channel ch,
+ struct iovec *iov, unsigned int iov_nr,
+ unsigned int flag, dma_addr_t dma_addr[]);
+int csm_dp_rx_poll(struct csm_dp_dev *pdev, struct iovec *iov, size_t iov_nr);
+void csm_dp_rx(struct csm_dp_dev *pdev, struct csm_dp_buf_cntrl *buf_cntrl, unsigned int length);
+int csm_dp_get_stats(struct csm_dp_dev *pdev, struct csm_dp_ioctl_getstats *stats);
+void csm_dp_mempool_put(struct csm_dp_mempool *mempool);
+
+#endif /* __QCOM_CSM_DP__ */
diff --git a/drivers/char/qcom_csm_dp/qcom_csm_dp_cdev.c b/drivers/char/qcom_csm_dp/qcom_csm_dp_cdev.c
new file mode 100644
index 000000000000..94fc69a8903a
--- /dev/null
+++ b/drivers/char/qcom_csm_dp/qcom_csm_dp_cdev.c
@@ -0,0 +1,941 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/cdev.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+
+#include "qcom_csm_dp.h"
+
+static bool csm_dp_is_rxqueue_mmap_cookie(unsigned int cookie)
+{
+ unsigned int type, mmap_type;
+
+ mmap_type = CSM_DP_MMAP_COOKIE_TO_TYPE(cookie);
+ type = CSM_DP_MMAP_COOKIE_TO_MEM_TYPE(cookie);
+ if (mmap_type == CSM_DP_MMAP_TYPE_RING && type >= CSM_DP_MEM_TYPE_LAST)
+ return true;
+ return false;
+}
+
+static void *csm_dp_usr_to_kern_vaddr(struct csm_dp_mempool_vma *mempool_vma,
+ void __user *addr, unsigned int *cluster,
+ unsigned int *c_offset)
+{
+ struct csm_dp_mempool *mempool = *mempool_vma->pp_mempool;
+ unsigned long offset = (unsigned long)addr -
+ mempool_vma->vma[CSM_DP_MMAP_TYPE_MEM]->vm_start;
+
+ *cluster = offset >> CSM_DP_MEMPOOL_CLUSTER_SHIFT;
+ *c_offset = offset & CSM_DP_MEMPOOL_CLUSTER_MASK;
+
+ return ((void *)mempool->mem.loc.cluster_kernel_addr[*cluster] +
+ *c_offset);
+}
+
+static struct csm_dp_rxqueue *csm_dp_rxqueue_vma_to_rxqueue(struct csm_dp_rxqueue_vma *rxq_vma)
+{
+ struct csm_dp_cdev *cdev = container_of(rxq_vma,
+ struct csm_dp_cdev,
+ rxqueue_vma[rxq_vma->type]);
+ return &cdev->pdev->rxq[rxq_vma->type];
+}
+
+static void csm_dp_cdev_init_mempool_vma(struct csm_dp_cdev *cdev)
+{
+ struct csm_dp_dev *pdev = cdev->pdev;
+ int type;
+
+ memset(cdev->mempool_vma, 0, sizeof(cdev->mempool_vma));
+
+ for (type = 0; type < CSM_DP_MEM_TYPE_LAST; type++)
+ cdev->mempool_vma[type].pp_mempool = &pdev->mempool[type];
+}
+
+static struct csm_dp_mempool_vma *csm_dp_find_mempool_vma(struct csm_dp_cdev *cdev,
+ void __user *addr,
+ unsigned int len)
+{
+ struct csm_dp_mempool_vma *mempool_vma = NULL;
+ struct vm_area_struct *vma;
+ unsigned int mem_type;
+
+ for (mem_type = 0; mem_type < CSM_DP_MEM_TYPE_LAST; mem_type++) {
+ mempool_vma = &cdev->mempool_vma[mem_type];
+ vma = mempool_vma->vma[CSM_DP_MMAP_TYPE_MEM];
+ if (vma && csm_dp_vaddr_in_vma_range(addr, len, vma))
+ return mempool_vma;
+ }
+ return NULL;
+}
+
+static int csm_dp_cdev_tx(struct csm_dp_cdev *cdev,
+ enum csm_dp_channel ch,
+ struct iovec __user *uiov,
+ unsigned int iov_nr,
+ unsigned int ioctl_flags,
+ bool sg)
+{
+ struct csm_dp_dev *pdev = cdev->pdev;
+ struct csm_dp_mempool_vma *mempool_vma;
+ struct iovec iov[CSM_DP_MAX_IOV_SIZE];
+ dma_addr_t dma_addr[CSM_DP_MAX_IOV_SIZE];
+ unsigned int n;
+ int ret;
+ unsigned int flag = 0;
+ struct csm_dp_mempool *mempool;
+ u32 iov_off_array[CSM_DP_MAX_IOV_SIZE];
+ unsigned int c_offset;
+ unsigned int cluster;
+ struct csm_dp_buf_cntrl *buf_cntrl = NULL, *prev_buf_cntrl = NULL;
+
+ pr_debug("ch %s bus %d VF %d iov_nr %u sg %d\n",
+ ch_name(ch), pdev->bus_num, pdev->vf_num, iov_nr, sg);
+ if (iov_nr > CSM_DP_MAX_IOV_SIZE)
+ return -E2BIG;
+
+ if (copy_from_user(iov, (void __user *)uiov,
+ sizeof(struct iovec) * iov_nr))
+ return -EFAULT;
+
+ for (n = 0; n < iov_nr; n++) {
+ mempool_vma = csm_dp_find_mempool_vma(cdev,
+ iov[n].iov_base,
+ iov[n].iov_len);
+ if (!mempool_vma) {
+ pr_debug("cannot find mempool addr=%p, len=%lu\n",
+ iov[n].iov_base, iov[n].iov_len);
+ return -EINVAL;
+ }
+ mempool = *mempool_vma->pp_mempool;
+
+ /* User passes in the pointer to message payload */
+ iov[n].iov_base = csm_dp_usr_to_kern_vaddr(mempool_vma, iov[n].iov_base,
+ &cluster, &c_offset);
+
+ unsigned long b_backtrack;
+ struct csm_dp_buf_cntrl *p;
+
+ b_backtrack = c_offset %
+ csm_dp_buf_true_size(&mempool->mem);
+ iov_off_array[n] = b_backtrack;
+ p = (struct csm_dp_buf_cntrl *)
+ (iov[n].iov_base - b_backtrack);
+ if (p->signature != CSM_DP_BUFFER_SIG) {
+ pr_err("mempool type %d buffer at kernel addr %p corrupted, %x, exp %x\n",
+ (*mempool_vma->pp_mempool)->type,
+ iov[n].iov_base, p->signature,
+ CSM_DP_BUFFER_SIG);
+ return -EINVAL;
+ }
+ if (p->fence != CSM_DP_BUFFER_FENCE_SIG) {
+ pr_err("mempool type %d buffer at kernel addr %p corrupted",
+ (*mempool_vma->pp_mempool)->type,
+ iov[n].iov_base);
+ return -EINVAL;
+ }
+ p->state = CSM_DP_BUF_STATE_KERNEL_XMIT_DMA;
+ p->xmit_status = CSM_DP_XMIT_IN_PROGRESS;
+
+ /* link SG fragments */
+ b_backtrack = c_offset % csm_dp_buf_true_size(&mempool->mem);
+ buf_cntrl = (struct csm_dp_buf_cntrl *)(iov[n].iov_base - b_backtrack);
+ if (sg) {
+ if (prev_buf_cntrl)
+ prev_buf_cntrl->next = buf_cntrl;
+ prev_buf_cntrl = buf_cntrl;
+ if (n == iov_nr - 1)
+ buf_cntrl->next = NULL;
+ } else {
+ buf_cntrl->next = NULL;
+ }
+
+ atomic_inc(&mempool->out_xmit);
+ if (mempool->mem.loc.dma_mapped) {
+ /*
+ * set to indicate iov_base is
+ * dma handle instead of
+ * kernel virtual addr
+ */
+ dma_addr[n] =
+ mempool->mem.loc.cluster_dma_addr[cluster] +
+ c_offset;
+ } else {
+ dma_addr[n] = 0;
+ }
+
+ pr_debug("start tx iov[%d], kaddr=%p len=%lu\n",
+ n, iov[n].iov_base, iov[n].iov_len);
+ }
+
+ buf_cntrl->next = NULL;
+
+ if (sg)
+ flag |= CSM_DP_TX_FLAG_SG;
+
+ ret = csm_dp_tx(pdev, ch, iov, iov_nr, flag, dma_addr);
+
+ if (ret) {
+ struct csm_dp_buf_cntrl *p;
+
+ for (n = 0; n < iov_nr; n++) {
+ p = (struct csm_dp_buf_cntrl *)
+ (iov[n].iov_base - iov_off_array[n]);
+ p->state = CSM_DP_BUF_STATE_KERNEL_XMIT_DMA_COMP;
+ atomic_dec(&mempool->out_xmit);
+ p->xmit_status = ret;
+ }
+ } else {
+ ret = iov_nr;
+ }
+
+ /* Ensure all the data are written */
+ wmb();
+ return ret;
+}
+
+static int __cdev_rx_poll(struct csm_dp_cdev *cdev,
+ struct iovec __user *uiov, size_t iov_nr)
+{
+ struct csm_dp_dev *pdev = cdev->pdev;
+ struct iovec iov[CSM_DP_MAX_IOV_SIZE];
+ int ret;
+
+ ret = csm_dp_rx_poll(pdev, iov, iov_nr);
+
+ if (ret > 0 && copy_to_user((void __user *)uiov, iov, sizeof(struct iovec) * ret))
+ ret = -EFAULT;
+
+ return ret;
+}
+
+static int csm_dp_cdev_ioctl_mempool_alloc(struct csm_dp_cdev *cdev,
+ unsigned long ioarg)
+{
+ struct csm_dp_dev *pdev = cdev->pdev;
+ struct csm_dp_ioctl_mempool_alloc req;
+ struct csm_dp_mempool *mempool;
+
+ if (copy_from_user(&req, (void __user *)ioarg, sizeof(req)))
+ return -EFAULT;
+
+ mempool = csm_dp_mempool_alloc(pdev, req.type, req.buf_sz, req.buf_num,
+ true);
+ if (!mempool)
+ return -ENOMEM;
+
+ cdev->mempool_vma[req.type].usr_alloc = true;
+
+ if (req.cfg) {
+ struct csm_dp_mempool_cfg cfg;
+
+ csm_dp_mempool_get_cfg(mempool, &cfg);
+ if (copy_to_user((void __user *)req.cfg, &cfg, sizeof(cfg)))
+ return -EFAULT;
+ }
+ return 0;
+}
+
+static int csm_dp_cdev_ioctl_mempool_getcfg(struct csm_dp_cdev *cdev, unsigned long ioarg)
+{
+ struct csm_dp_dev *pdev = cdev->pdev;
+ struct csm_dp_ioctl_getcfg req;
+ struct csm_dp_mempool_cfg cfg;
+
+ if (copy_from_user(&req, (void __user *)ioarg, sizeof(req)))
+ return -EFAULT;
+
+ if (!csm_dp_mem_type_is_valid(req.type))
+ return -EINVAL;
+
+ if (csm_dp_mempool_get_cfg(pdev->mempool[req.type], &cfg))
+ return -EAGAIN;
+
+ if (copy_to_user((void __user *)req.cfg, &cfg, sizeof(cfg)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int csm_dp_cdev_ioctl_tx(struct csm_dp_cdev *cdev, unsigned long ioarg)
+{
+ struct csm_dp_ioctl_tx arg;
+
+ if (copy_from_user(&arg, (void __user *)ioarg, sizeof(arg)))
+ return -EFAULT;
+
+ if (!arg.iov.iov_len || arg.iov.iov_len > CSM_DP_MAX_IOV_SIZE)
+ return -EINVAL;
+
+ return (csm_dp_cdev_tx(cdev, arg.ch, arg.iov.iov_base,
+ arg.iov.iov_len, arg.flags, false));
+}
+
+static int csm_dp_cdev_ioctl_sg_tx(struct csm_dp_cdev *cdev, unsigned long ioarg)
+{
+ struct csm_dp_ioctl_tx arg;
+
+ if (copy_from_user(&arg, (void __user *)ioarg, sizeof(arg)))
+ return -EFAULT;
+
+ if (!arg.iov.iov_len || arg.iov.iov_len > CSM_DP_MAX_IOV_SIZE)
+ return -EINVAL;
+
+ return (csm_dp_cdev_tx(cdev, arg.ch, arg.iov.iov_base,
+ arg.iov.iov_len, arg.flags, true));
+}
+
+static int csm_dp_cdev_ioctl_rx_getcfg(struct csm_dp_cdev *cdev, unsigned long ioarg)
+{
+ struct csm_dp_dev *pdev = cdev->pdev;
+ struct csm_dp_ioctl_getcfg req;
+ struct csm_dp_ring_cfg cfg;
+
+ if (copy_from_user(&req, (void __user *)ioarg, sizeof(req)))
+ return -EFAULT;
+
+ if (!csm_dp_rx_type_is_valid(req.type) || !req.cfg)
+ return -EINVAL;
+
+ csm_dp_ring_get_cfg(pdev->rxq[req.type].ring, &cfg);
+ if (copy_to_user((void __user *)req.cfg, &cfg, sizeof(cfg)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int csm_dp_cdev_ioctl_rx_poll(struct csm_dp_cdev *cdev, unsigned long ioarg)
+{
+ struct iovec iov;
+
+ if (copy_from_user(&iov, (void __user *)ioarg, sizeof(iov)))
+ return -EFAULT;
+
+ if (!iov.iov_len || iov.iov_len > CSM_DP_MAX_IOV_SIZE)
+ return -EINVAL;
+
+ return __cdev_rx_poll(cdev, iov.iov_base, iov.iov_len);
+}
+
+static int csm_dp_cdev_ioctl_get_stats(struct csm_dp_cdev *cdev, unsigned long ioarg)
+{
+ struct csm_dp_ioctl_getstats req;
+ struct csm_dp_dev *pdev = cdev->pdev;
+ int ret;
+
+ if (copy_from_user(&req, (void __user *)ioarg, sizeof(req)))
+ return -EFAULT;
+
+ mutex_lock(&pdev->cdev_lock);
+ ret = csm_dp_get_stats(pdev, &req);
+ mutex_unlock(&pdev->cdev_lock);
+ if (ret)
+ return ret;
+
+ if (copy_to_user((void __user *)ioarg, &req, sizeof(req)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static unsigned int csm_dp_cdev_poll(struct file *file, poll_table *wait)
+{
+ struct csm_dp_cdev *cdev = file->private_data;
+ struct csm_dp_dev *pdev = cdev->pdev;
+ struct csm_dp_rxqueue *rxq;
+ unsigned int mask = 0;
+ int type, n;
+
+ for (type = 0, n = 0; type < CSM_DP_RX_TYPE_LAST; type++) {
+ if (cdev->rxqueue_vma[type].vma) {
+ rxq = &pdev->rxq[type];
+ if (!rxq->inited)
+ continue;
+
+ poll_wait(file, &rxq->wq, wait);
+ n++;
+ }
+ }
+ if (unlikely(!n))
+ return POLLERR;
+
+ for (type = 0; type < CSM_DP_RX_TYPE_LAST; type++) {
+ if (cdev->rxqueue_vma[type].vma) {
+ rxq = &pdev->rxq[type];
+ if (!rxq->inited)
+ continue;
+
+ if (!csm_dp_ring_is_empty(rxq->ring)) {
+ mask |= POLLIN | POLLRDNORM;
+ break;
+ }
+ }
+ }
+ return mask;
+}
+
+static long csm_dp_cdev_ioctl(struct file *file,
+ unsigned int iocmd,
+ unsigned long ioarg)
+{
+ struct csm_dp_cdev *cdev = file->private_data;
+ int ret = -EINVAL;
+
+ if (in_compat_syscall())
+ return ret;
+
+ switch (iocmd) {
+ case CSM_DP_IOCTL_MEMPOOL_ALLOC:
+ ret = csm_dp_cdev_ioctl_mempool_alloc(cdev, ioarg);
+ break;
+ case CSM_DP_IOCTL_MEMPOOL_GET_CONFIG:
+ ret = csm_dp_cdev_ioctl_mempool_getcfg(cdev, ioarg);
+ break;
+ case CSM_DP_IOCTL_RX_GET_CONFIG:
+ ret = csm_dp_cdev_ioctl_rx_getcfg(cdev, ioarg);
+ break;
+ case CSM_DP_IOCTL_TX:
+ ret = csm_dp_cdev_ioctl_tx(cdev, ioarg);
+ break;
+ case CSM_DP_IOCTL_SG_TX:
+ ret = csm_dp_cdev_ioctl_sg_tx(cdev, ioarg);
+ break;
+ case CSM_DP_IOCTL_RX_POLL:
+ ret = csm_dp_cdev_ioctl_rx_poll(cdev, ioarg);
+ break;
+ case CSM_DP_IOCTL_GET_STATS:
+ ret = csm_dp_cdev_ioctl_get_stats(cdev, ioarg);
+ break;
+ default:
+ break;
+ }
+ return ret;
+}
+
+static void csm_dp_mempool_mem_vma_open(struct vm_area_struct *vma)
+{
+ struct csm_dp_mempool_vma *mempool_vma = vma->vm_private_data;
+ atomic_t *refcnt = &mempool_vma->refcnt[CSM_DP_MMAP_TYPE_MEM];
+
+ if (atomic_add_return(1, refcnt) == 1) {
+ struct csm_dp_mempool *mempool = *mempool_vma->pp_mempool;
+
+ mempool_vma->vma[CSM_DP_MMAP_TYPE_MEM] = vma;
+ if (!csm_dp_mempool_hold(mempool))
+ atomic_dec(refcnt);
+ }
+}
+
+static void csm_dp_mempool_mem_vma_close(struct vm_area_struct *vma)
+{
+ struct csm_dp_mempool_vma *mempool_vma = vma->vm_private_data;
+ atomic_t *refcnt = &mempool_vma->refcnt[CSM_DP_MMAP_TYPE_MEM];
+
+ if (atomic_dec_and_test(refcnt)) {
+ struct csm_dp_mempool *mempool = *mempool_vma->pp_mempool;
+
+ mempool_vma->vma[CSM_DP_MMAP_TYPE_MEM] = NULL;
+ csm_dp_mempool_put(mempool);
+ }
+}
+
+static const struct vm_operations_struct csm_dp_mempool_mem_vma_ops = {
+ .open = csm_dp_mempool_mem_vma_open,
+ .close = csm_dp_mempool_mem_vma_close,
+};
+
+static int csm_dp_mempool_mem_mmap(struct csm_dp_mempool_vma *mempool_vma,
+ struct vm_area_struct *vma)
+{
+ struct csm_dp_mempool *mempool = *mempool_vma->pp_mempool;
+ struct csm_dp_mem *mem;
+ unsigned long size;
+ int ret;
+ unsigned long addr = vma->vm_start;
+ int i;
+ unsigned long remainder;
+
+ if (mempool_vma->vma[CSM_DP_MMAP_TYPE_MEM]) {
+ pr_err("memory already mapped\n");
+ return -EBUSY;
+ }
+ if (!csm_dp_mempool_hold(mempool)) {
+ pr_err("mempool does not exist, mempool %p\n", mempool);
+ return -EAGAIN;
+ }
+
+ mem = &mempool->mem;
+ size = vma->vm_end - vma->vm_start;
+ remainder = mem->loc.size;
+ if (size < remainder) {
+ ret = -EINVAL;
+ pr_err("size(0x%lx) too small, expect at least 0x%lx\n",
+ size, remainder);
+ goto out;
+ }
+
+ /* Reset pgoff */
+ vma->vm_pgoff = 0;
+
+ for (i = 0; i < mem->loc.num_cluster; i++) {
+ unsigned long len;
+
+ if (i == mem->loc.num_cluster - 1)
+ len = remainder;
+ else
+ len = CSM_DP_MEMPOOL_CLUSTER_SIZE;
+
+ ret = remap_pfn_range(vma, addr,
+ page_to_pfn(mem->loc.page[i]),
+ len,
+ vma->vm_page_prot);
+ if (ret) {
+ pr_err("dma mmap failed\n");
+ goto out;
+ }
+ addr += len;
+ remainder -= len;
+ }
+
+ vma->vm_private_data = mempool_vma;
+ vma->vm_ops = &csm_dp_mempool_mem_vma_ops;
+ csm_dp_mempool_mem_vma_open(vma);
+
+out:
+ csm_dp_mempool_put(mempool);
+ return ret;
+}
+
+static void csm_dp_mempool_ring_vma_open(struct vm_area_struct *vma)
+{
+ struct csm_dp_mempool_vma *mempool_vma = vma->vm_private_data;
+ atomic_t *refcnt = &mempool_vma->refcnt[CSM_DP_MMAP_TYPE_RING];
+
+ if (atomic_add_return(1, refcnt) == 1) {
+ struct csm_dp_mempool *mempool = *mempool_vma->pp_mempool;
+
+ mempool_vma->vma[CSM_DP_MMAP_TYPE_RING] = vma;
+ __csm_dp_mempool_hold(mempool);
+ }
+}
+
+static void csm_dp_mempool_ring_vma_close(struct vm_area_struct *vma)
+{
+ struct csm_dp_mempool_vma *mempool_vma = vma->vm_private_data;
+ atomic_t *refcnt = &mempool_vma->refcnt[CSM_DP_MMAP_TYPE_RING];
+
+ if (atomic_dec_and_test(refcnt)) {
+ struct csm_dp_mempool *mempool = *mempool_vma->pp_mempool;
+
+ mempool_vma->vma[CSM_DP_MMAP_TYPE_RING] = NULL;
+ csm_dp_mempool_put(mempool);
+ }
+}
+
+static const struct vm_operations_struct csm_dp_mempool_ring_vma_ops = {
+ .open = csm_dp_mempool_ring_vma_open,
+ .close = csm_dp_mempool_ring_vma_close,
+};
+
+static int csm_dp_mempool_ring_mmap(struct csm_dp_mempool_vma *mempool_vma,
+ struct vm_area_struct *vma)
+{
+ struct csm_dp_mempool *mempool = *mempool_vma->pp_mempool;
+ struct csm_dp_ring *ring;
+ unsigned long size;
+ int ret;
+
+ if (mempool_vma->vma[CSM_DP_MMAP_TYPE_RING]) {
+ pr_err("ring already mapped, mem_type=%u\n", mempool->type);
+ ret = -EBUSY;
+ }
+
+ if (!csm_dp_mempool_hold(mempool)) {
+ pr_err("mempool not exist\n");
+ return -EAGAIN;
+ }
+
+ ring = &mempool->ring;
+ size = vma->vm_end - vma->vm_start;
+ if (size < csm_dp_mem_loc_mmap_size(&ring->loc)) {
+ pr_err("size(0x%lx) too small, expect at least 0x%lx\n",
+ size, csm_dp_mem_loc_mmap_size(&ring->loc));
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ret = remap_pfn_range(vma,
+ vma->vm_start,
+ page_to_pfn(ring->loc.page[0]),
+ ring->loc.size,
+ vma->vm_page_prot);
+ if (ret) {
+ pr_err("remap_pfn_range failed\n");
+ goto out;
+ }
+
+ /* Reset pgoff */
+ vma->vm_pgoff = 0;
+ vma->vm_private_data = mempool_vma;
+ vma->vm_ops = &csm_dp_mempool_ring_vma_ops;
+ csm_dp_mempool_ring_vma_open(vma);
+
+out:
+ csm_dp_mempool_put(mempool);
+ return ret;
+}
+
+/* mmap mempool into user space */
+static int csm_dp_cdev_mempool_mmap(struct csm_dp_cdev *cdev,
+ struct vm_area_struct *vma)
+{
+ struct csm_dp_mempool_vma *mempool_vma;
+ unsigned int mem_type, type, cookie;
+ int ret = 0;
+
+ /* use vm_pgoff to distinguish different area to map */
+ cookie = vma->vm_pgoff << PAGE_SHIFT;
+ type = CSM_DP_MMAP_COOKIE_TO_TYPE(cookie);
+ mem_type = CSM_DP_MMAP_COOKIE_TO_MEM_TYPE(cookie);
+
+ if (!csm_dp_mem_type_is_valid(mem_type) ||
+ !csm_dp_mmap_type_is_valid(type)) {
+ pr_err("invalid cookie(0x%x)\n", cookie);
+ return -EINVAL;
+ }
+
+ mempool_vma = &cdev->mempool_vma[mem_type];
+ switch (type) {
+ case CSM_DP_MMAP_TYPE_RING:
+ /* map ring for buffer management */
+ ret = csm_dp_mempool_ring_mmap(mempool_vma, vma);
+ break;
+ case CSM_DP_MMAP_TYPE_MEM:
+ /* map buffer memory */
+ ret = csm_dp_mempool_mem_mmap(mempool_vma, vma);
+ break;
+ }
+
+ return ret;
+}
+
+static void csm_dp_rxqueue_vma_open(struct vm_area_struct *vma)
+{
+ struct csm_dp_rxqueue_vma *rxq_vma = vma->vm_private_data;
+
+ if (atomic_add_return(1, &rxq_vma->refcnt) == 1) {
+ struct csm_dp_rxqueue *rxq;
+
+ rxq_vma->vma = vma;
+ rxq_vma->type = CSM_DP_MMAP_RX_COOKIE_TO_TYPE(vma->vm_pgoff << PAGE_SHIFT);
+
+ rxq = csm_dp_rxqueue_vma_to_rxqueue(rxq_vma);
+ atomic_inc(&rxq->refcnt);
+ }
+}
+
+static void csm_dp_rxqueue_vma_close(struct vm_area_struct *vma)
+{
+ struct csm_dp_rxqueue_vma *rxq_vma = vma->vm_private_data;
+ struct csm_dp_rxqueue *rxq = csm_dp_rxqueue_vma_to_rxqueue(rxq_vma);
+
+ if (!atomic_dec_and_test(&rxq_vma->refcnt))
+ return;
+ rxq_vma->vma = NULL;
+ atomic_dec(&rxq->refcnt);
+}
+
+static const struct vm_operations_struct csm_dp_rxqueue_vma_ops = {
+ .open = csm_dp_rxqueue_vma_open,
+ .close = csm_dp_rxqueue_vma_close,
+};
+
+/* mmap RXQ into user space */
+static int csm_dp_cdev_rxqueue_mmap(struct csm_dp_cdev *cdev,
+ struct vm_area_struct *vma)
+{
+ struct csm_dp_dev *pdev = cdev->pdev;
+ struct csm_dp_rxqueue_vma *rxq_vma = cdev->rxqueue_vma;
+ struct csm_dp_ring *ring;
+ unsigned int type, cookie;
+ unsigned long size;
+ int ret = 0;
+
+ cookie = vma->vm_pgoff << PAGE_SHIFT;
+
+ type = CSM_DP_MMAP_RX_COOKIE_TO_TYPE(cookie);
+ if (!csm_dp_rx_type_is_valid(type)) {
+ pr_err("invalid rx queue type, cookie=0x%x, type=%u\n",
+ cookie, type);
+ return -EINVAL;
+ }
+
+ if (!pdev->rxq[type].inited) {
+ pr_err("rx queue type %d not initialized\n", type);
+ return -EINVAL;
+ }
+
+ if (rxq_vma[type].vma) {
+ pr_err("rxqueue already mapped\n");
+ return -EBUSY;
+ }
+
+ ring = pdev->rxq[type].ring;
+ size = vma->vm_end - vma->vm_start;
+ if (size < csm_dp_mem_loc_mmap_size(&ring->loc)) {
+ pr_err("size(0x%lx) too small, expect at least 0x%lx\n",
+ size, csm_dp_mem_loc_mmap_size(&ring->loc));
+ return -EINVAL;
+ }
+ ret = remap_pfn_range(vma,
+ vma->vm_start,
+ page_to_pfn(ring->loc.page[0]),
+ ring->loc.size,
+ vma->vm_page_prot);
+ if (ret) {
+ pr_err("rxqueue mmap failed, error=%d\n", ret);
+ return ret;
+ }
+
+ vma->vm_private_data = &rxq_vma[type];
+ vma->vm_ops = &csm_dp_rxqueue_vma_ops;
+ csm_dp_rxqueue_vma_open(vma);
+
+ return 0;
+}
+
+static int csm_dp_cdev_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ struct csm_dp_cdev *cdev = file->private_data;
+ struct csm_dp_dev *pdev = cdev->pdev;
+ unsigned int cookie;
+ int ret;
+
+ mutex_lock(&pdev->cdev_lock);
+
+ cookie = vma->vm_pgoff << PAGE_SHIFT;
+
+ if (csm_dp_is_rxqueue_mmap_cookie(cookie))
+ ret = csm_dp_cdev_rxqueue_mmap(cdev, vma);
+ else
+ ret = csm_dp_cdev_mempool_mmap(cdev, vma);
+
+ mutex_unlock(&pdev->cdev_lock);
+
+ return ret;
+}
+
+static int csm_dp_cdev_open(struct inode *inode, struct file *file)
+{
+ struct csm_dp_dev *pdev = container_of(inode->i_cdev,
+ struct csm_dp_dev, cdev);
+ struct csm_dp_cdev *cdev;
+ struct csm_dp_mempool *mempool;
+ struct csm_dp_mempool_vma *mempool_vma;
+
+ cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
+ if (IS_ERR_OR_NULL(cdev)) {
+ pr_err("failed to alloc memory\n!");
+ return -ENOMEM;
+ }
+
+ cdev->pid = current->tgid;
+ pdev->pid = cdev->pid;
+ strscpy(pdev->pid_name, current->comm, TASK_COMM_LEN);
+ cdev->pdev = pdev;
+
+ csm_dp_cdev_init_mempool_vma(cdev);
+
+ mutex_lock(&pdev->cdev_lock);
+
+ mempool_vma = cdev->mempool_vma;
+
+ mempool = &(*mempool_vma->pp_mempool[CSM_DP_MEM_TYPE_UL_DATA]);
+ /* Free all the pending packets in Rx for UL data channel */
+ free_rx_ring_buffers(mempool, false);
+
+ mempool = &(*mempool_vma->pp_mempool[CSM_DP_MEM_TYPE_UL_CONTROL]);
+ /* Free all the pending packets in Rx for UL control channel */
+ free_rx_ring_buffers(mempool, false);
+
+ list_add_tail(&cdev->list, &pdev->cdev_head);
+ mutex_unlock(&pdev->cdev_lock);
+
+ file->private_data = cdev;
+
+ return 0;
+}
+
+static int csm_dp_cdev_release(struct inode *inode, struct file *file)
+{
+ struct csm_dp_cdev *cdev = file->private_data;
+ struct csm_dp_mempool *mempool;
+ struct csm_dp_mempool_vma *mempool_vma = cdev->mempool_vma;
+ struct csm_dp_dev *pdev = cdev->pdev;
+ int type;
+
+ pdev->pid = -EINVAL;
+ mutex_lock(&pdev->cdev_lock);
+ list_del(&cdev->list);
+
+ mempool = &(*mempool_vma->pp_mempool[CSM_DP_MEM_TYPE_UL_DATA]);
+ /* Free all the pending packets in Rx for UL data channel */
+ free_rx_ring_buffers(mempool, false);
+
+ mempool = &(*mempool_vma->pp_mempool[CSM_DP_MEM_TYPE_UL_CONTROL]);
+ /* Free all the pending packets in Rx for UL control channel */
+ free_rx_ring_buffers(mempool, false);
+
+ for (type = 0; type < CSM_DP_MEM_TYPE_LAST; type++, mempool_vma++) {
+ if (mempool_vma->usr_alloc)
+ csm_dp_mempool_put(*mempool_vma->pp_mempool);
+ }
+
+ kfree(cdev);
+ mutex_unlock(&pdev->cdev_lock);
+
+ return 0;
+}
+
+static const struct file_operations csm_dp_cdev_fops = {
+ .owner = THIS_MODULE,
+ .poll = csm_dp_cdev_poll,
+ .unlocked_ioctl = csm_dp_cdev_ioctl,
+ .compat_ioctl = compat_ptr_ioctl,
+ .mmap = csm_dp_cdev_mmap,
+ .open = csm_dp_cdev_open,
+ .release = csm_dp_cdev_release
+};
+
+int csm_dp_cdev_init(struct csm_dp_drv *pdrv)
+{
+ int ret;
+
+ pdrv->dev_class = class_create(CSM_DP_DEV_CLASS_NAME);
+ if (IS_ERR_OR_NULL(pdrv->dev_class)) {
+ pr_err("class_create failed\n");
+ return -ENOMEM;
+ }
+
+ ret = alloc_chrdev_region(&pdrv->devno, 0, CSM_DP_MAX_NUM_DEVS, CSM_DP_CDEV_NAME);
+ if (ret) {
+ pr_err("alloc_chrdev_region failed\n");
+ class_destroy(pdrv->dev_class);
+ pdrv->dev_class = NULL;
+ pr_err("CSM-DP: failed to initialize cdev\n");
+ return ret;
+ }
+
+ pr_info("CSM-DP: cdev initialized\n");
+ return 0;
+}
+
+void csm_dp_cdev_cleanup(struct csm_dp_drv *pdrv)
+{
+ int i;
+
+ if (!pdrv->dev_class)
+ return;
+
+ for (i = 0; i < CSM_DP_MAX_NUM_DEVS; i++)
+ csm_dp_cdev_del(&pdrv->dp_devs[i]);
+
+ unregister_chrdev_region(pdrv->devno, CSM_DP_MAX_NUM_DEVS);
+ class_destroy(pdrv->dev_class);
+ pdrv->dev_class = NULL;
+}
+
+/* Called from MHI probe for each VF */
+int csm_dp_cdev_add(struct csm_dp_dev *pdev, struct device *mhi_dev)
+{
+ struct device *dev;
+ int ret, new_devno;
+ struct csm_dp_drv *pdrv = pdev->pdrv;
+ unsigned int index = pdev - pdrv->dp_devs;
+
+ mutex_lock(&pdev->cdev_lock);
+
+ if (pdev->cdev_inited) {
+ pr_err("cdev already initialized\n");
+ mutex_unlock(&pdev->cdev_lock);
+ return -EINVAL;
+ }
+
+ ret = csm_dp_rx_init(pdev);
+ if (ret) {
+ mutex_unlock(&pdev->cdev_lock);
+ return ret;
+ }
+
+ cdev_init(&pdev->cdev, &csm_dp_cdev_fops);
+ new_devno = MKDEV(MAJOR(pdrv->devno), index);
+ ret = cdev_add(&pdev->cdev, new_devno, 1);
+ if (ret) {
+ pr_err("cdev_add failed!\n");
+ goto err;
+ }
+
+ dev = device_create(pdrv->dev_class, NULL, new_devno, pdrv, "csm%d-dp%d",
+ pdev->bus_num, pdev->vf_num);
+ if (IS_ERR_OR_NULL(dev)) {
+ pr_err("device_create failed\n");
+ ret = PTR_ERR(dev);
+ cdev_del(&pdev->cdev);
+ goto err;
+ }
+
+ pdev->cdev_inited = true;
+
+ mutex_unlock(&pdev->cdev_lock);
+
+ return 0;
+
+err:
+ csm_dp_rx_cleanup(pdev);
+ mutex_unlock(&pdev->cdev_lock);
+ return ret;
+}
+
+void csm_dp_cdev_del(struct csm_dp_dev *pdev)
+{
+ struct csm_dp_drv *pdrv = pdev->pdrv;
+
+ mutex_lock(&pdev->cdev_lock);
+ if (!pdev->cdev_inited) {
+ mutex_unlock(&pdev->cdev_lock);
+ return;
+ }
+ if (!list_empty(&pdev->cdev_head)) {
+ pr_err("Device file already open; skipping device deletion\n");
+ mutex_unlock(&pdev->cdev_lock);
+ return;
+ }
+
+ pdev->cdev_inited = false;
+
+ device_destroy(pdrv->dev_class, pdev->cdev.dev);
+ cdev_del(&pdev->cdev);
+
+ /* wait for idle mempools before Rx cleanup */
+ while (1) {
+ int control_ref = atomic_read(&pdev->mempool[CSM_DP_MEM_TYPE_UL_CONTROL]->ref);
+ int data_ref = atomic_read(&pdev->mempool[CSM_DP_MEM_TYPE_UL_DATA]->ref);
+
+ pr_debug("UL_CONTROL ref %d UL_DATA ref %d\n", control_ref, data_ref);
+ if (control_ref == 1 && data_ref == 1)
+ break;
+ mutex_unlock(&pdev->cdev_lock);
+ msleep(100);
+ mutex_lock(&pdev->cdev_lock);
+ }
+ csm_dp_rx_cleanup(pdev);
+
+ mutex_unlock(&pdev->cdev_lock);
+}
diff --git a/drivers/char/qcom_csm_dp/qcom_csm_dp_core.c b/drivers/char/qcom_csm_dp/qcom_csm_dp_core.c
new file mode 100644
index 000000000000..c0c106e28423
--- /dev/null
+++ b/drivers/char/qcom_csm_dp/qcom_csm_dp_core.c
@@ -0,0 +1,571 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+
+#include "qcom_csm_dp.h"
+
+#define CSM_DP_RX_QUEUE_SIZE 1024
+#define CSM_DP_MEMPOOL_PUT_SLEEP 10
+#define CSM_DP_MEMPOOL_PUT_ITER 2
+
+static struct csm_dp_drv *csm_dp_pdrv;
+
+static struct csm_dp_mhi *get_dp_mhi(struct csm_dp_dev *pdev, enum csm_dp_channel ch)
+{
+ switch (ch) {
+ case CSM_DP_CH_CONTROL:
+ return &pdev->mhi_control_dev;
+ case CSM_DP_CH_DATA:
+ return &pdev->mhi_data_dev;
+ default:
+ pr_err("invalid ch\n");
+ return NULL;
+ }
+}
+
+static int csm_dp_rxqueue_init(struct csm_dp_rxqueue *rxq,
+ enum csm_dp_rx_type rx_type,
+ unsigned int size)
+{
+ unsigned int ring_size;
+ int ret;
+
+ if (!csm_dp_rx_type_is_valid(rx_type))
+ return -EINVAL;
+
+ if (rxq->inited) {
+ pr_err("rx queue already initialized!\n");
+ return -EINVAL;
+ }
+
+ ring_size = csm_dp_calc_ring_size(size);
+ if (!ring_size)
+ return -EINVAL;
+
+ rxq->ring = kzalloc(sizeof(*rxq->ring), GFP_KERNEL);
+ if (!rxq->ring)
+ return -ENOMEM;
+
+ ret = csm_dp_ring_init(rxq->ring, ring_size, CSM_DP_MMAP_RX_COOKIE(rx_type));
+ if (ret) {
+ pr_debug("failed to initialize rx ring!\n");
+ kfree(rxq->ring);
+ rxq->ring = NULL;
+
+ return ret;
+ }
+
+ init_waitqueue_head(&rxq->wq);
+ rxq->type = rx_type;
+ rxq->inited = true;
+ atomic_set(&rxq->refcnt, 0);
+
+ return 0;
+}
+
+static void csm_dp_rxqueue_cleanup(struct csm_dp_rxqueue *rxq)
+{
+ unsigned long size;
+ struct csm_dp_dev *dev = container_of(rxq,
+ struct csm_dp_dev, rxq[rxq->type]);
+ if (rxq->inited) {
+ size = ((unsigned int)(1) << rxq->ring->loc.last_cl_order) * PAGE_SIZE;
+ rxq->inited = false;
+ wake_up(&rxq->wq);
+ csm_dp_ring_cleanup(rxq->ring);
+ dev->stats.mem_stats.rxq_ring_in_use[rxq->type] -= size;
+ kfree(rxq->ring);
+ rxq->ring = NULL;
+ }
+}
+
+void csm_dp_mempool_put(struct csm_dp_mempool *mempool)
+{
+ if (mempool && atomic_dec_and_test(&mempool->ref)) {
+ struct csm_dp_mhi *mhi = &mempool->dp_dev->mhi_data_dev;
+
+ /* wait for any pending mempool buffers on DATA channel */
+ if (csm_dp_mhi_is_ready(mhi)) {
+ int counter = CSM_DP_MEMPOOL_PUT_ITER;
+
+ while (counter-- && atomic_read(&mempool->out_xmit)) {
+ csm_dp_mhi_tx_poll(mhi);
+ msleep(CSM_DP_MEMPOOL_PUT_SLEEP);
+ }
+ }
+
+ csm_dp_mempool_free(mempool);
+ }
+}
+
+void csm_dp_rx(struct csm_dp_dev *pdev, struct csm_dp_buf_cntrl *buf_cntrl, unsigned int length)
+{
+ struct csm_dp_mempool *mempool;
+ struct csm_dp_rxqueue *rxq;
+ struct csm_dp_buf_cntrl *packet_start = buf_cntrl, *packet_next;
+ unsigned int offset, cl, i;
+ void *addr = buf_cntrl + 1;
+
+ if (unlikely(!pdev || !addr || !length)) {
+ pr_err("invalid argument\n");
+ return;
+ }
+
+ mempool = csm_dp_get_mempool(pdev, buf_cntrl, &cl);
+ if (!mempool) {
+ pr_err("not UL address, addr=%p\n", addr);
+ return;
+ }
+
+ for (i = 0; i < buf_cntrl->buf_count; i++) {
+ packet_next = packet_start->next;
+ packet_start->state = CSM_DP_BUF_STATE_KERNEL_RECVCMP_MSGQ_TO_APP;
+ packet_start = packet_next;
+ }
+ packet_start = buf_cntrl;
+ packet_next = NULL;
+
+ if (mempool->type == CSM_DP_MEM_TYPE_UL_DATA) {
+ struct csm_dp_buf_cntrl **p = &pdev->pending_packets;
+
+ while (*p)
+ p = &((*p)->next_packet);
+
+ *p = buf_cntrl;
+ return;
+ }
+
+ /* only one Rx queue */
+ rxq = &pdev->rxq[CSM_DP_RX_TYPE_FAPI];
+
+ if (!atomic_read(&rxq->refcnt)) {
+ pr_debug("rxq not active, drop message\n");
+ goto free_rxbuf;
+ }
+
+ offset = csm_dp_get_mem_offset(addr, &mempool->mem.loc, cl);
+ if (csm_dp_ring_write(rxq->ring, offset)) {
+ pr_err("failed to enqueue rx packet\n");
+ goto free_rxbuf;
+ }
+ wake_up(&rxq->wq);
+ pdev->stats.rx_cnt++;
+ return;
+free_rxbuf:
+ for (i = 0; i < buf_cntrl->buf_count; i++) {
+ addr = packet_start + 1;
+ packet_next = packet_start->next;
+ csm_dp_mempool_put_buf(mempool, addr);
+ pdev->stats.rx_drop++;
+ packet_start = packet_next;
+ }
+}
+
+int csm_dp_rx_init(struct csm_dp_dev *pdev)
+{
+ unsigned int type;
+ int ret;
+ unsigned int csm_dp_ul_buf_size = CSM_DP_DEFAULT_UL_BUF_SIZE;
+ unsigned int csm_dp_ul_buf_cnt = CSM_DP_DEFAULT_UL_BUF_CNT;
+ unsigned int rx_queue_size = CSM_DP_RX_QUEUE_SIZE;
+
+ if (csm_dp_ul_buf_size > CSM_DP_MAX_UL_MSG_LEN) {
+ pr_err("UL buffer size %d exceeds limit %d\n",
+ csm_dp_ul_buf_size,
+ CSM_DP_MAX_UL_MSG_LEN);
+ return -ENOMEM;
+ }
+
+ pdev->mempool[CSM_DP_MEM_TYPE_UL_CONTROL] = csm_dp_mempool_alloc(pdev,
+ CSM_DP_MEM_TYPE_UL_CONTROL,
+ csm_dp_ul_buf_size,
+ csm_dp_ul_buf_cnt,
+ false);
+ if (!pdev->mempool[CSM_DP_MEM_TYPE_UL_CONTROL]) {
+ pr_err("failed to allocate UL_CONTROL memory pool!\n");
+ return -ENOMEM;
+ }
+
+ pdev->mempool[CSM_DP_MEM_TYPE_UL_DATA] = csm_dp_mempool_alloc(pdev, CSM_DP_MEM_TYPE_UL_DATA,
+ csm_dp_ul_buf_size,
+ csm_dp_ul_buf_cnt,
+ false);
+ if (!pdev->mempool[CSM_DP_MEM_TYPE_UL_DATA]) {
+ pr_err("failed to allocate UL_DATA memory pool!\n");
+ return -ENOMEM;
+ }
+
+ for (type = 0; type < CSM_DP_RX_TYPE_LAST; type++) {
+ ret = csm_dp_rxqueue_init(&pdev->rxq[type], type, rx_queue_size);
+ if (ret) {
+ pr_err("failed to init rxqueue!\n");
+ return ret;
+ }
+ pdev->stats.mem_stats.rxq_ring_in_use[type] +=
+ pdev->rxq[type].ring->loc.true_alloc_size;
+ }
+
+ return 0;
+}
+
+void csm_dp_rx_cleanup(struct csm_dp_dev *pdev)
+{
+ unsigned int type;
+
+ if (pdev->mempool[CSM_DP_MEM_TYPE_UL_CONTROL])
+ csm_dp_mempool_free(pdev->mempool[CSM_DP_MEM_TYPE_UL_CONTROL]);
+ if (pdev->mempool[CSM_DP_MEM_TYPE_UL_DATA])
+ csm_dp_mempool_free(pdev->mempool[CSM_DP_MEM_TYPE_UL_DATA]);
+
+ for (type = 0; type < CSM_DP_RX_TYPE_LAST; type++)
+ csm_dp_rxqueue_cleanup(&pdev->rxq[type]);
+}
+
+int csm_dp_tx(struct csm_dp_dev *pdev,
+ enum csm_dp_channel ch,
+ struct iovec *iov,
+ unsigned int iov_nr,
+ unsigned int flag,
+ dma_addr_t dma_addr_array[])
+{
+ int ret = 0, n;
+ unsigned int num, to_send;
+ int j;
+ struct csm_dp_mhi *mhi;
+
+ if (unlikely(!pdev || !iov || !iov_nr))
+ return -EINVAL;
+
+ mhi = get_dp_mhi(pdev, ch);
+
+ if (flag & CSM_DP_TX_FLAG_SG) {
+ if (iov_nr > CSM_DP_MAX_SG_IOV_SIZE) {
+ pr_err("sg iov size too big!\n");
+ return -EINVAL;
+ }
+ }
+
+ atomic_inc(&mhi->mhi_dev_refcnt);
+ if (!csm_dp_mhi_is_ready(mhi)) {
+ atomic_dec(&mhi->mhi_dev_refcnt);
+ if (pdev->stats.tx_drop % 1024 == 0)
+ pr_err("mhi is not ready!\n");
+ pdev->stats.tx_drop++;
+ return -ENODEV;
+ }
+
+ mutex_lock(&mhi->tx_mutex);
+ to_send = 0;
+ for (n = 0, to_send = iov_nr; to_send > 0; ) {
+ if (to_send > CSM_DP_MAX_IOV_SIZE)
+ num = CSM_DP_MAX_IOV_SIZE;
+ else
+ num = to_send;
+ for (j = 0; j < num; j++) {
+ if ((flag & CSM_DP_TX_FLAG_SG) && n != (iov_nr - 1))
+ mhi->dl_flag_array[j] = MHI_CHAIN;
+ else
+ mhi->dl_flag_array[j] = MHI_EOT;
+ mhi->dl_buf_array[j].len = iov[n].iov_len;
+
+ if (flag & CSM_DP_TX_FLAG_SG) {
+ mhi->dl_flag_array[j] |= MHI_SG;
+ mhi->dl_buf_array[j].buf = iov[0].iov_base;
+ } else {
+ mhi->dl_buf_array[j].buf = iov[n].iov_base;
+ }
+
+ if (ch == CSM_DP_CH_DATA)
+ mhi->dl_flag_array[j] |= MHI_BEI;
+
+ if (dma_addr_array[n])
+ mhi->dl_buf_array[j].dma_addr =
+ dma_addr_array[n];
+ else
+ mhi->dl_buf_array[j].dma_addr = 0;
+
+ mhi->dl_buf_array[j].streaming_dma = true;
+ n++;
+ }
+ ret = csm_dp_mhi_n_tx(mhi, num);
+ if (ret) {
+ pdev->stats.tx_err++;
+ break;
+ }
+ to_send -= num;
+ }
+
+ if (!(flag & CSM_DP_TX_FLAG_SG))
+ pdev->stats.tx_cnt += (iov_nr - to_send);
+ else if (!to_send)
+ pdev->stats.tx_cnt++;
+ mutex_unlock(&mhi->tx_mutex);
+
+ if (ch == CSM_DP_CH_DATA)
+ csm_dp_mhi_tx_poll(mhi);
+
+ atomic_dec(&mhi->mhi_dev_refcnt);
+ return ret;
+}
+
+int csm_dp_rx_poll(struct csm_dp_dev *pdev, struct iovec *iov, size_t iov_nr)
+{
+ int ret;
+ struct csm_dp_buf_cntrl *cur_packet;
+ size_t n = 0, remain = iov_nr;
+
+ atomic_inc(&pdev->mhi_data_dev.mhi_dev_refcnt);
+ if (!csm_dp_mhi_is_ready(&pdev->mhi_data_dev)) {
+ pdev->stats.rx_poll_ignore++;
+ atomic_dec(&pdev->mhi_data_dev.mhi_dev_refcnt);
+ return -ENODEV;
+ }
+
+ /*
+ * poll to get packets from MHI. This will cause dl_xfer (RX callback) to get called which
+ * will then link the Rx packets into pdrv->pending_packets
+ */
+ ret = mhi_poll(pdev->mhi_data_dev.mhi_dev, CSM_DP_NAPI_WEIGHT, DMA_FROM_DEVICE);
+ if (ret < 0) {
+ pr_err_ratelimited("Error:%d rx polling for bus:%d VF:%d %s\n",
+ ret, pdev->bus_num, pdev->vf_num, ch_name(CSM_DP_CH_DATA));
+ atomic_dec(&pdev->mhi_data_dev.mhi_dev_refcnt);
+ return ret;
+ }
+
+ ret = csm_dp_mhi_rx_replenish(&pdev->mhi_data_dev);
+ if (ret < 0)
+ pr_err_ratelimited("Error:%d rx replenish for bus:%d VF:%d %s\n",
+ ret, pdev->bus_num, pdev->vf_num, ch_name(CSM_DP_CH_DATA));
+
+ atomic_dec(&pdev->mhi_data_dev.mhi_dev_refcnt);
+
+ /* fill iov with the received packets */
+ cur_packet = pdev->pending_packets;
+ while (cur_packet) {
+ struct csm_dp_buf_cntrl *cur_buf, *tmp;
+
+ if (cur_packet->buf_count > remain) {
+ if (cur_packet == pdev->pending_packets)
+ return -EINVAL; /* provided iov is too short even for 1st packet */
+ /* no more room in iov, we're done */
+ break;
+ }
+
+ for (cur_buf = cur_packet; cur_buf; cur_buf = cur_buf->next) {
+ unsigned int cl;
+ struct csm_dp_mempool *mempool = pdev->mempool[CSM_DP_MEM_TYPE_UL_DATA];
+
+ if (!mempool) {
+ pr_err_ratelimited("not UL address\n");
+ continue;
+ }
+
+ cl = csm_dp_mem_get_cluster(&mempool->mem, cur_buf->buf_index);
+ iov[n].iov_base = (void *)csm_dp_get_mem_offset(cur_buf + 1,
+ &mempool->mem.loc, cl);
+ iov[n].iov_len = cur_buf->len;
+ n++;
+ remain--;
+ }
+
+ tmp = cur_packet;
+ cur_packet = cur_packet->next_packet;
+ tmp->next_packet = NULL;
+ }
+
+ pdev->pending_packets = cur_packet;
+
+ return n;
+}
+
+int csm_dp_get_stats(struct csm_dp_dev *pdev, struct csm_dp_ioctl_getstats *stats)
+{
+ struct csm_dp_mhi *mhi = NULL;
+
+ switch (stats->ch) {
+ case CSM_DP_CH_CONTROL:
+ mhi = &pdev->mhi_control_dev;
+ break;
+ case CSM_DP_CH_DATA:
+ mhi = &pdev->mhi_data_dev;
+ break;
+ }
+
+ if (!mhi)
+ return -EINVAL;
+
+ stats->tx_cnt = mhi->stats.tx_cnt;
+ stats->tx_acked = mhi->stats.tx_acked;
+ stats->rx_cnt = mhi->stats.rx_cnt;
+
+ return 0;
+}
+
+/* napi function to replenish control channel */
+static int csm_dp_poll(struct napi_struct *napi, int budget)
+{
+ int rx_work = 0;
+ struct csm_dp_dev *pdev;
+ int ret;
+
+ pdev = container_of(napi, struct csm_dp_dev, napi);
+ atomic_inc(&pdev->mhi_control_dev.mhi_dev_refcnt);
+ if (!csm_dp_mhi_is_ready(&pdev->mhi_control_dev)) {
+ pdev->stats.rx_poll_ignore++;
+ atomic_dec(&pdev->mhi_control_dev.mhi_dev_refcnt);
+ return -ENODEV;
+ }
+
+ rx_work = mhi_poll(pdev->mhi_control_dev.mhi_dev, budget, DMA_FROM_DEVICE);
+ if (rx_work < 0) {
+ pr_err("Error Rx polling ret:%d\n", rx_work);
+ rx_work = 0;
+ napi_complete(napi);
+ goto exit_poll;
+ }
+
+ ret = csm_dp_mhi_rx_replenish(&pdev->mhi_control_dev);
+ if (ret == -ENOMEM)
+ schedule_work(&pdev->alloc_work); /* later */
+ if (rx_work < budget)
+ napi_complete(napi);
+ else
+ pdev->stats.rx_budget_overflow++;
+exit_poll:
+ atomic_dec(&pdev->mhi_control_dev.mhi_dev_refcnt);
+ return rx_work;
+}
+
+/* worker function to replenish control channel, in case replenish failed in napi function */
+static void csm_dp_alloc_work(struct work_struct *work)
+{
+ struct csm_dp_dev *pdev;
+ const int sleep_ms = 1000;
+ int retry = 60;
+ int ret;
+
+ pdev = container_of(work, struct csm_dp_dev, alloc_work);
+
+ do {
+ ret = csm_dp_mhi_rx_replenish(&pdev->mhi_control_dev);
+ /* sleep and try again */
+ if (ret == -ENOMEM) {
+ msleep(sleep_ms);
+ retry--;
+ }
+ } while (ret == -ENOMEM && retry);
+}
+
+static int csm_dp_core_init(struct csm_dp_drv *pdrv)
+{
+ int i;
+
+ for (i = 0; i < CSM_DP_MAX_NUM_DEVS; i++) {
+ struct csm_dp_dev *pdev = &pdrv->dp_devs[i];
+
+ pdev->pdrv = pdrv;
+ mutex_init(&pdev->mempool_lock);
+ mutex_init(&pdev->cdev_lock);
+ INIT_LIST_HEAD(&pdev->cdev_head);
+ pdev->dummy_dev = alloc_netdev_dummy(0);
+ netif_napi_add(pdev->dummy_dev, &pdev->napi, csm_dp_poll);
+ napi_enable(&pdev->napi);
+ INIT_WORK(&pdev->alloc_work, csm_dp_alloc_work);
+ }
+
+ return 0;
+}
+
+static void csm_dp_core_cleanup(struct csm_dp_drv *pdrv)
+{
+ int i;
+
+ for (i = 0; i < CSM_DP_MAX_NUM_DEVS; i++) {
+ struct csm_dp_dev *pdev = &pdrv->dp_devs[i];
+
+ flush_work(&pdev->alloc_work);
+ napi_disable(&pdev->napi);
+ netif_napi_del(&pdev->napi);
+
+ csm_dp_rx_cleanup(pdev);
+ }
+
+ kfree(pdrv);
+}
+
+static int csm_dp_init(void)
+{
+ struct csm_dp_drv *pdrv;
+ int ret;
+
+ pr_info("CSM-DP: probing CSM\n");
+
+ pdrv = kzalloc(sizeof(*pdrv), GFP_KERNEL);
+ if (IS_ERR_OR_NULL(pdrv))
+ return -ENOMEM;
+ csm_dp_pdrv = pdrv;
+
+ ret = csm_dp_core_init(pdrv);
+ if (ret)
+ goto cleanup;
+
+ ret = csm_dp_cdev_init(pdrv);
+ if (ret)
+ goto cleanup_cdev;
+
+ csm_dp_debugfs_init(pdrv);
+
+ ret = csm_dp_mhi_init(pdrv);
+ if (ret)
+ goto cleanup_debugfs;
+
+ pr_info("CSM-DP: module initialized now\n");
+ return 0;
+
+cleanup_debugfs:
+ csm_dp_debugfs_cleanup(pdrv);
+cleanup_cdev:
+ csm_dp_cdev_cleanup(pdrv);
+cleanup:
+ csm_dp_core_cleanup(pdrv);
+ csm_dp_pdrv = NULL;
+ pr_err("CSM-DP: module init failed!\n");
+ return ret;
+}
+
+static int csm_dp_remove(void)
+{
+ struct csm_dp_drv *pdrv = csm_dp_pdrv;
+
+ if (pdrv) {
+ csm_dp_mhi_cleanup(pdrv);
+ csm_dp_cdev_cleanup(pdrv);
+ csm_dp_debugfs_cleanup(pdrv);
+ csm_dp_core_cleanup(pdrv);
+ }
+ csm_dp_pdrv = NULL;
+
+ return 0;
+}
+
+static int __init csm_dp_module_init(void)
+{
+ return csm_dp_init();
+}
+module_init(csm_dp_module_init);
+
+static void __exit csm_dp_module_exit(void)
+{
+ csm_dp_remove();
+}
+module_exit(csm_dp_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("QUALCOMM CSM DP driver");
diff --git a/drivers/char/qcom_csm_dp/qcom_csm_dp_debugfs.c b/drivers/char/qcom_csm_dp/qcom_csm_dp_debugfs.c
new file mode 100644
index 000000000000..9d2d158cb30f
--- /dev/null
+++ b/drivers/char/qcom_csm_dp/qcom_csm_dp_debugfs.c
@@ -0,0 +1,993 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "qcom_csm_dp.h"
+#include "qcom_csm_dp_mhi.h"
+#ifdef CONFIG_DEBUG_FS
+
+#include <linux/atomic.h>
+#include <linux/debugfs.h>
+#include <linux/kernel.h>
+#include <linux/mod_devicetable.h>
+#include <linux/slab.h>
+
+#define CSM_DP_MEM_DUMP_COL_WIDTH 16
+#define CSM_DP_MAX_MEM_DUMP_SIZE 256
+
+#define CSM_DP_DEFINE_DEBUGFS_OPS(name, __read, __write) \
+static int name ##_open(struct inode *inode, struct file *file) \
+{ \
+ return single_open(file, __read, inode->i_private); \
+} \
+static const struct file_operations name ##_ops = { \
+ .open = name ## _open, \
+ .read = seq_read, \
+ .write = __write, \
+ .llseek = seq_lseek, \
+ .release = single_release, \
+}
+
+static int __csm_dp_rxqueue_vma_dump(struct seq_file *s,
+ struct csm_dp_rxqueue_vma *rxq_vma)
+{
+ if (rxq_vma->vma) {
+ struct vm_area_struct *vma = rxq_vma->vma;
+
+ seq_printf(s, " Type: %s\n",
+ csm_dp_rx_type_to_str(rxq_vma->type));
+ seq_printf(s, " RefCnt: %d\n",
+ atomic_read(&rxq_vma->refcnt));
+ seq_printf(s,
+ " vm_start: %lx\n"
+ " vm_end: %lx\n"
+ " vm_pgoff: %lx\n"
+ " vm_flags: %lx\n",
+ vma->vm_start,
+ vma->vm_end,
+ vma->vm_pgoff,
+ vma->vm_flags);
+ }
+ return 0;
+}
+
+static int __csm_dp_mempool_vma_dump(struct seq_file *s,
+ struct csm_dp_mempool_vma *mempool_vma)
+{
+ struct csm_dp_mempool *mempool = *mempool_vma->pp_mempool;
+ struct vm_area_struct *vma;
+ int i;
+
+ if (mempool)
+ seq_printf(s, " Type: %s\n",
+ csm_dp_mem_type_to_str(mempool->type));
+
+ for (i = 0; i < CSM_DP_MMAP_TYPE_LAST; i++) {
+ if (mempool_vma->vma[i]) {
+ vma = mempool_vma->vma[i];
+ seq_printf(s, " VMA[%d]: %s\n",
+ i, csm_dp_mmap_type_to_str(i));
+ seq_printf(s,
+ " vm_start: %lx\n"
+ " vm_end: %lx\n"
+ " vm_pgoff: %lx\n"
+ " vm_flags: %lx\n",
+ vma->vm_start,
+ vma->vm_end,
+ vma->vm_pgoff,
+ vma->vm_flags);
+ seq_printf(s, " refcnt: %d\n",
+ atomic_read(&mempool_vma->refcnt[i]));
+ }
+ }
+ return 0;
+}
+
+static int __csm_dp_ring_opstats_dump(struct seq_file *s,
+ struct csm_dp_ring_opstats *stats)
+{
+ seq_puts(s, "Read:\n");
+ seq_printf(s, " Ok: %u\n", atomic_read(&stats->read_ok));
+ seq_printf(s, " Empty: %u\n", atomic_read(&stats->read_empty));
+ seq_puts(s, "Write:\n");
+ seq_printf(s, " Ok: %u\n", atomic_read(&stats->write_ok));
+ seq_printf(s, " Full: %u\n", atomic_read(&stats->write_full));
+ return 0;
+}
+
+static int __csm_dp_ring_runtime_dump(struct seq_file *s,
+ struct csm_dp_ring *ring)
+{
+ seq_printf(s, "ProdHdr: %u\n", *ring->prod_head);
+ seq_printf(s, "ProdTail: %u\n", *ring->prod_tail);
+ seq_printf(s, "ConsHdr: %u\n", *ring->cons_head);
+ seq_printf(s, "ConsTail: %u\n", *ring->cons_tail);
+ seq_printf(s, "NumOfElementAvail: %u\n",
+ (*ring->prod_head - *ring->cons_tail) & (ring->size - 1));
+ return 0;
+}
+
+static int __csm_dp_ring_config_dump(struct seq_file *s,
+ struct csm_dp_ring *ring)
+{
+ seq_printf(s, "Ring %llx MemoryAlloc:\n", (u64)ring);
+ seq_printf(s, " AllocAddr: %llx\n", (u64)ring->loc.base);
+ seq_printf(s, " AllocSize: 0x%08lx\n", ring->loc.size);
+ seq_printf(s, " MmapCookie: 0x%08x\n", ring->loc.cookie);
+ seq_printf(s, "Size: 0x%x\n", ring->size);
+ seq_printf(s, "ProdHdr: %llx\n", (u64)ring->prod_head);
+ seq_printf(s, "ProdTail: %llx\n", (u64)ring->prod_tail);
+ seq_printf(s, "ConsHdr: %llx\n", (u64)ring->cons_head);
+ seq_printf(s, "ConsTail: %llx\n", (u64)ring->cons_tail);
+ seq_printf(s, "RingBuf: %llx\n", (u64)ring->element);
+ return 0;
+}
+
+static int csm_dp_debugfs_rxq_refcnt_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_rxqueue *rxq = (struct csm_dp_rxqueue *)s->private;
+
+ if (rxq->inited)
+ seq_printf(s, "%d\n", atomic_read(&rxq->refcnt));
+
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_rxq_refcnt,
+ csm_dp_debugfs_rxq_refcnt_read, NULL);
+
+static int csm_dp_debugfs_rxq_opstats_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_rxqueue *rxq = (struct csm_dp_rxqueue *)s->private;
+
+ if (rxq->inited)
+ __csm_dp_ring_opstats_dump(s, &rxq->ring->opstats);
+
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_rxq_opstats,
+ csm_dp_debugfs_rxq_opstats_read, NULL);
+
+static int csm_dp_debugfs_rxq_config_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_rxqueue *rxq = (struct csm_dp_rxqueue *)s->private;
+
+ if (rxq->inited) {
+ seq_printf(s, "Type: %s\n",
+ csm_dp_rx_type_to_str(rxq->type));
+ __csm_dp_ring_config_dump(s, rxq->ring);
+ }
+
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_rxq_config,
+ csm_dp_debugfs_rxq_config_read, NULL);
+
+static int csm_dp_debugfs_rxq_runtime_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_rxqueue *rxq = (struct csm_dp_rxqueue *)s->private;
+
+ if (rxq->inited)
+ __csm_dp_ring_runtime_dump(s, rxq->ring);
+
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_rxq_runtime,
+ csm_dp_debugfs_rxq_runtime_read, NULL);
+
+static unsigned int __mem_dump_size[CSM_DP_MEM_TYPE_LAST];
+static unsigned int __mem_offset[CSM_DP_MEM_TYPE_LAST];
+
+static int csm_dp_debugfs_mem_data_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool) {
+ struct csm_dp_mem *mem = &mempool->mem;
+ unsigned int n = __mem_dump_size[mempool->type];
+ unsigned int offset = __mem_offset[mempool->type];
+ unsigned int i, j;
+ unsigned int cluster, c_offset;
+ unsigned char *data = (unsigned char *)mem->loc.base + offset;
+
+ data = csm_dp_mem_offset_addr(mem, offset, &cluster, &c_offset);
+ if (!data)
+ return 0;
+ if (n > (mem->loc.size - offset))
+ n = mem->loc.size - offset;
+
+ for (i = 0; i < offset % CSM_DP_MEM_DUMP_COL_WIDTH; i++)
+ seq_puts(s, " ");
+
+ for (j = 0; j < n; j++, i++) {
+ if (i && !(i % CSM_DP_MEM_DUMP_COL_WIDTH))
+ seq_puts(s, "\n");
+ seq_printf(s, "%02x ", *data);
+ data++;
+ c_offset++;
+ if (c_offset >= CSM_DP_MEMPOOL_CLUSTER_SIZE) {
+ c_offset = 0;
+ cluster++;
+ data = mem->loc.cluster_kernel_addr[cluster];
+ }
+ }
+ seq_puts(s, "\n");
+ }
+ return 0;
+}
+
+static ssize_t csm_dp_debugfs_mem_data_write(struct file *fp,
+ const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct csm_dp_mempool *mempool = *((struct csm_dp_mempool **)
+ (((struct seq_file *)fp->private_data)->private));
+
+ if (mempool) {
+ struct csm_dp_mem *mem = &mempool->mem;
+ unsigned int value = 0;
+ unsigned int *data;
+ unsigned int offset = __mem_offset[mempool->type];
+ unsigned int cluster, c_offset;
+
+ if (kstrtouint_from_user(buf, count, 0, &value))
+ return -EFAULT;
+ data = (unsigned int *)csm_dp_mem_offset_addr(mem, offset,
+ &cluster, &c_offset);
+ if (!data)
+ return count;
+ *data = value;
+ }
+ return count;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_mem_data,
+ csm_dp_debugfs_mem_data_read,
+ csm_dp_debugfs_mem_data_write);
+
+static int csm_dp_debugfs_mem_dump_size_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool)
+ seq_printf(s, "%u\n", __mem_dump_size[mempool->type]);
+ return 0;
+}
+
+static ssize_t csm_dp_debugfs_mem_dump_size_write(struct file *fp,
+ const char __user *buf,
+ size_t count, loff_t *ppos)
+{
+ struct csm_dp_mempool *mempool = *((struct csm_dp_mempool **)
+ (((struct seq_file *)fp->private_data)->private));
+ unsigned int value = 0;
+
+ if (!mempool)
+ goto done;
+
+ if (kstrtouint_from_user(buf, count, 0, &value))
+ return -EFAULT;
+
+ if (value > CSM_DP_MAX_MEM_DUMP_SIZE)
+ return -EINVAL;
+
+ __mem_dump_size[mempool->type] = value;
+done:
+ return count;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_mem_dump_size,
+ csm_dp_debugfs_mem_dump_size_read,
+ csm_dp_debugfs_mem_dump_size_write);
+
+static int csm_dp_debugfs_mem_offset_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool)
+ seq_printf(s, "0x%08x\n", __mem_offset[mempool->type]);
+ return 0;
+}
+
+static ssize_t csm_dp_debugfs_mem_offset_write(struct file *fp,
+ const char __user *buf,
+ size_t count,
+ loff_t *ppos)
+{
+ struct csm_dp_mempool *mempool = *((struct csm_dp_mempool **)
+ (((struct seq_file *)fp->private_data)->private));
+
+ if (mempool) {
+ struct csm_dp_mem *mem = &mempool->mem;
+ unsigned int value = 0;
+
+ if (kstrtouint_from_user(buf, count, 0, &value))
+ return -EFAULT;
+
+ if (value >= mem->loc.size)
+ return -EINVAL;
+ if (value & 3)
+ return -EINVAL;
+
+ __mem_offset[mempool->type] = value;
+ }
+ return count;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_mem_offset,
+ csm_dp_debugfs_mem_offset_read,
+ csm_dp_debugfs_mem_offset_write);
+
+static int csm_dp_debugfs_mem_config_show(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+ int i;
+
+ if (mempool) {
+ struct csm_dp_mem *mem = &mempool->mem;
+
+ seq_puts(s, "MemoryAlloc:\n");
+ seq_printf(s, " AllocSize: 0x%08lx\n",
+ mem->loc.size);
+ seq_printf(s, " Total Cluster: %d\n",
+ mem->loc.num_cluster);
+ seq_printf(s, " Cluster Size: 0x%x\n",
+ CSM_DP_MEMPOOL_CLUSTER_SIZE);
+ for (i = 0; i < mem->loc.num_cluster; i++)
+ seq_printf(s, " Cluster %d Addr: %llx\n", i,
+ (u64)mem->loc.cluster_kernel_addr[i]);
+ seq_printf(s, " Buffer Per Cluster: %d\n",
+ mem->loc.buf_per_cluster);
+ seq_printf(s, " Last Cluster Order: %d\n",
+ mem->loc.last_cl_order);
+ seq_printf(s, " MmapCookie: %08x\n",
+ mem->loc.cookie);
+ seq_printf(s, "BufSize: 0x%x\n", mem->buf_sz);
+ seq_printf(s, "BufCount: 0x%x\n", mem->buf_cnt);
+ seq_printf(s, "BufTrueSize: 0x%x\n",
+ csm_dp_buf_true_size(mem));
+ }
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_mem_config,
+ csm_dp_debugfs_mem_config_show, NULL);
+
+static int csm_dp_debugfs_mem_buffer_state_show(struct seq_file *s, void *unused)
+{
+ int i, j;
+ int k_free = 0;
+ int k_alloc_dma = 0;
+ int k_recv_msgq_app = 0;
+ int k_xmit_dma = 0;
+ int k_xmit_dma_comp = 0;
+ int u_free = 0;
+ int u_alloc = 0;
+ int u_recv = 0;
+ char *cl_start;
+ unsigned int cl_buf_cnt;
+ struct csm_dp_buf_cntrl *p;
+
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool) {
+ struct csm_dp_mem *mem = &mempool->mem;
+
+ if (!csm_dp_mem_type_is_valid(mempool->type))
+ return 0;
+
+ for (j = 0; j < mem->loc.num_cluster; j++) {
+ cl_start = mem->loc.cluster_kernel_addr[j];
+ if (j == mem->loc.num_cluster - 1)
+ cl_buf_cnt = mem->buf_cnt -
+ (mem->loc.buf_per_cluster * j);
+ else
+ cl_buf_cnt = mem->loc.buf_per_cluster;
+ for (i = 0; i < cl_buf_cnt; i++) {
+ p = (struct csm_dp_buf_cntrl *)(cl_start +
+ (i * csm_dp_buf_true_size(mem)));
+
+ if (!p)
+ break;
+ if (p->state == CSM_DP_BUF_STATE_KERNEL_FREE)
+ k_free++;
+ if (p->state == CSM_DP_BUF_STATE_KERNEL_ALLOC_RECV_DMA)
+ k_alloc_dma++;
+ if (p->state == CSM_DP_BUF_STATE_KERNEL_RECVCMP_MSGQ_TO_APP)
+ k_recv_msgq_app++;
+ if (p->state == CSM_DP_BUF_STATE_KERNEL_XMIT_DMA)
+ k_xmit_dma++;
+ if (p->state == CSM_DP_BUF_STATE_KERNEL_XMIT_DMA_COMP)
+ k_xmit_dma_comp++;
+ if (p->state == CSM_DP_BUF_STATE_USER_FREE)
+ u_free++;
+ if (p->state == CSM_DP_BUF_STATE_USER_ALLOC)
+ u_alloc++;
+ if (p->state == CSM_DP_BUF_STATE_USER_RECV)
+ u_recv++;
+ }
+ }
+
+ seq_puts(s, "MemoryBufferState:\n");
+ seq_printf(s, "MemoryType: %s\n", csm_dp_mem_type_to_str(mempool->type));
+ seq_printf(s, " KERNEL_FREE: %d\n",
+ k_free);
+ seq_printf(s, " KERNEL_ALLOC_RECV_DMA: %d\n",
+ k_alloc_dma);
+ seq_printf(s, " KERNEL_RECVCMP_MSGQ_TO_APP: %d\n",
+ k_recv_msgq_app);
+ seq_printf(s, " KERNEL_XMIT_DMA: %d\n",
+ k_xmit_dma);
+ seq_printf(s, " KERNEL_XMIT_DMA_COMP: %d\n",
+ k_xmit_dma_comp);
+ seq_printf(s, " USER_FREE: %d\n",
+ u_free);
+ seq_printf(s, " USER_ALLOC: %d\n",
+ u_alloc);
+ seq_printf(s, " USER_RECV: %d\n",
+ u_recv);
+ }
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_mem_buffer_state,
+ csm_dp_debugfs_mem_buffer_state_show, NULL);
+
+static int csm_dp_debugfs_ring_config_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool)
+ __csm_dp_ring_config_dump(s, &mempool->ring);
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_ring_config,
+ csm_dp_debugfs_ring_config_read, NULL);
+
+static int csm_dp_debugfs_ring_runtime_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool)
+ __csm_dp_ring_runtime_dump(s, &mempool->ring);
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_ring_runtime,
+ csm_dp_debugfs_ring_runtime_read, NULL);
+
+static int csm_dp_debugfs_ring_opstats_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool)
+ __csm_dp_ring_opstats_dump(s, &mempool->ring.opstats);
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_ring_opstats,
+ csm_dp_debugfs_ring_opstats_read, NULL);
+
+unsigned long __ring_index[CSM_DP_MEM_TYPE_LAST];
+
+static int csm_dp_debugfs_ring_index_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool)
+ seq_printf(s, "%lu\n", __ring_index[mempool->type]);
+ return 0;
+}
+
+static ssize_t csm_dp_debugfs_ring_index_write(struct file *fp,
+ const char __user *buf,
+ size_t count,
+ loff_t *ppos)
+{
+ struct csm_dp_mempool *mempool = *((struct csm_dp_mempool **)
+ (((struct seq_file *)fp->private_data)->private));
+ unsigned int value = 0;
+
+ if (!mempool)
+ goto done;
+
+ if (kstrtouint_from_user(buf, count, 0, &value))
+ return -EFAULT;
+
+ if (value >= mempool->ring.size)
+ return -EINVAL;
+
+ __ring_index[mempool->type] = value;
+done:
+ return count;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_ring_index,
+ csm_dp_debugfs_ring_index_read,
+ csm_dp_debugfs_ring_index_write);
+
+static int csm_dp_debugfs_ring_data_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool) {
+ struct csm_dp_ring_element *elem_p;
+
+ elem_p = (mempool->ring.element + __ring_index[mempool->type]);
+
+ seq_printf(s, "0x%lx\n", elem_p->element_data);
+ }
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_ring_data,
+ csm_dp_debugfs_ring_data_read, NULL);
+
+static int csm_dp_debugfs_mempool_status_show(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool) {
+ seq_printf(s, "BufPut: %lu\n",
+ mempool->stats.buf_put);
+ seq_printf(s, "InvalidBufPut: %lu\n",
+ mempool->stats.invalid_buf_put);
+ seq_printf(s, "ErrBufPut: %lu\n",
+ mempool->stats.buf_put_err);
+ seq_printf(s, "BufGet: %lu\n",
+ mempool->stats.buf_get);
+ seq_printf(s, "InvalidBufGet: %lu\n",
+ mempool->stats.invalid_buf_get);
+ seq_printf(s, "ErrBufGet: %lu\n",
+ mempool->stats.buf_get_err);
+ }
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_mempool_status,
+ csm_dp_debugfs_mempool_status_show, NULL);
+
+static int csm_dp_debugfs_mempool_state_show(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+ unsigned long state_cnt[CSM_DP_BUF_STATE_LAST];
+ unsigned long buf_bad = 0;
+ unsigned long unknown_state = 0;
+ int i;
+
+ memset(state_cnt, 0, sizeof(state_cnt));
+ if (mempool) {
+ struct csm_dp_mem *mem = &mempool->mem;
+ struct csm_dp_buf_cntrl *p;
+
+ for (i = 0; i < mem->buf_cnt; i++) {
+ p = (struct csm_dp_buf_cntrl *)
+ csm_dp_mem_rec_addr(mem, i);
+ if (!p)
+ return 0;
+ if (p->signature != CSM_DP_BUFFER_SIG ||
+ p->fence != CSM_DP_BUFFER_FENCE_SIG ||
+ p->buf_index != i)
+ buf_bad++;
+ else if (p->state >= CSM_DP_BUF_STATE_LAST)
+ unknown_state++;
+ else
+ state_cnt[p->state]++;
+ }
+
+ seq_printf(s, "Total Buf: %u\n",
+ mem->buf_cnt);
+ seq_printf(s, "Buf Real Size: %u\n",
+ mem->buf_sz + mem->buf_overhead_sz);
+ seq_printf(s, "Buf Corrupted: %lu\n",
+ buf_bad);
+ seq_printf(s, "Buf Unknown State: %lu\n",
+ unknown_state);
+
+ for (i = 0; i < CSM_DP_BUF_STATE_LAST; i++) {
+ if (state_cnt[i]) {
+ seq_printf(s, "Buf State %s: ",
+ csm_dp_buf_state_to_str(i));
+ seq_printf(s, " %lu\n",
+ state_cnt[i]);
+ }
+ }
+ }
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_mempool_state,
+ csm_dp_debugfs_mempool_state_show, NULL);
+
+static int csm_dp_debugfs_mempool_active_show(struct seq_file *s, void *unused)
+{
+ struct csm_dp_dev *pdev = (struct csm_dp_dev *)s->private;
+ unsigned int type;
+
+ for (type = 0; type < CSM_DP_MEM_TYPE_LAST; type++) {
+ if (pdev->mempool[type])
+ seq_printf(s, "%s ", csm_dp_mem_type_to_str(type));
+ }
+ seq_puts(s, "\n");
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_mempool_active,
+ csm_dp_debugfs_mempool_active_show, NULL);
+
+static int csm_dp_debugfs_mempool_info_show(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mempool *mempool =
+ *((struct csm_dp_mempool **)s->private);
+
+ if (mempool) {
+ seq_printf(s, "Driver: %llx\n",
+ (u64)mempool->dp_dev);
+ seq_printf(s, "MemPool: %llx\n",
+ (u64)mempool);
+ seq_printf(s, "Type: %s\n",
+ csm_dp_mem_type_to_str(mempool->type));
+ seq_printf(s, "Ref: %d\n",
+ atomic_read(&mempool->ref));
+ }
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_mempool_info,
+ csm_dp_debugfs_mempool_info_show, NULL);
+
+static int csm_dp_debugfs_start_recovery_dev_read(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mhi *mhi = (struct csm_dp_mhi *)s->private;
+
+ if (mhi)
+ seq_printf(s, "mhi_dev_suspended: %u\n",
+ mhi->mhi_dev_suspended);
+
+ return 0;
+}
+
+static ssize_t csm_dp_debugfs_start_recovery_dev_write(struct file *fp,
+ const char __user *buf,
+ size_t count,
+ loff_t *ppos)
+{
+ struct csm_dp_mhi *mhi = ((struct csm_dp_mhi *)
+ (((struct seq_file *)fp->private_data)->private));
+ int retry = 10;
+
+ if (mhi->mhi_dev_suspended) {
+ pr_info("MHI channel is already suspended\n");
+ return count;
+ }
+
+ /* Start the recovery operation */
+ mhi->mhi_dev_suspended = true;
+ pr_info("Recovering MHI channel..\n");
+ /* Allocating work to queue */
+ queue_work(mhi->mhi_dev_workqueue, &mhi->alloc_work);
+
+ while (mhi->mhi_dev_suspended && retry) {
+ msleep(50);
+ retry--;
+ }
+
+ if (!mhi->mhi_dev_suspended)
+ pr_info("MHI channel recovery is complete\n");
+ return count;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_start_recovery_control_dev,
+ csm_dp_debugfs_start_recovery_dev_read,
+ csm_dp_debugfs_start_recovery_dev_write);
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_start_recovery_data_dev,
+ csm_dp_debugfs_start_recovery_dev_read,
+ csm_dp_debugfs_start_recovery_dev_write);
+
+static int csm_dp_debugfs_mhi_show(struct seq_file *s, void *unused)
+{
+ struct csm_dp_mhi *mhi = (struct csm_dp_mhi *)s->private;
+ struct mhi_device *mhi_dev = mhi->mhi_dev;
+
+ seq_printf(s, "MHIDevice: %llx\n", (u64)mhi->mhi_dev);
+ seq_puts(s, "Stats:\n");
+ seq_printf(s, " TX: %lu\n", mhi->stats.tx_cnt);
+ seq_printf(s, " TX_ACKED: %lu\n", mhi->stats.tx_acked);
+ seq_printf(s, " TX_ERR: %lu\n", mhi->stats.tx_err);
+ seq_printf(s, " RX: %lu\n", mhi->stats.rx_cnt);
+
+ if (mhi_dev) {
+ struct csm_dp_dev *pdev = dev_get_drvdata(&mhi_dev->dev);
+ struct csm_dp_core_stats *stats = &pdev->stats;
+ bool is_control = (mhi_dev->id->driver_data == CSM_DP_CH_CONTROL);
+
+ if (stats && is_control)
+ seq_printf(s, " RX_DROP: %lu\n", stats->rx_drop);
+ }
+
+ seq_printf(s, " RX_OUT_OF_BUF: %lu\n",
+ mhi->stats.rx_out_of_buf);
+ seq_printf(s, " RX_REPLENISH: %lu\n",
+ mhi->stats.rx_replenish);
+ seq_printf(s, " RX_REPLENISH_ERR: %lu\n",
+ mhi->stats.rx_replenish_err);
+ seq_printf(s, " CHANNEL_ERR_COUNT: %lu\n",
+ mhi->stats.ch_err_cnt);
+ if (mhi_dev) {
+ seq_printf(s, " MHI_TX_RING_LAST_REQ_COUNT: %d\n",
+ mhi_get_free_desc_count(mhi->mhi_dev, DMA_TO_DEVICE));
+ seq_printf(s, " MHI_RX_RING_LAST_REQ_COUNT: %d\n",
+ mhi_get_free_desc_count(mhi->mhi_dev, DMA_FROM_DEVICE));
+ }
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_mhi, csm_dp_debugfs_mhi_show, NULL);
+
+static int csm_dp_debugfs_cdev_show(struct seq_file *s, void *unused)
+{
+ struct csm_dp_dev *pdev = (struct csm_dp_dev *)s->private;
+ struct csm_dp_cdev *cdev;
+ int n = 0;
+ int i;
+
+ if (!pdev->cdev_inited)
+ return 0;
+
+ mutex_lock(&pdev->cdev_lock);
+ list_for_each_entry(cdev, &pdev->cdev_head, list) {
+ seq_printf(s, "CDEV(%d)\n", n++);
+ seq_printf(s, "Driver: %llx\n",
+ (u64)cdev->pdev);
+ seq_printf(s, "Cdev: %llx\n",
+ (u64)cdev);
+ seq_printf(s, "PID: %d\n",
+ cdev->pid);
+
+ for (i = 0; i < CSM_DP_MEM_TYPE_LAST; i++) {
+ seq_printf(s, "MemPoolVMA[%d]\n", i);
+ __csm_dp_mempool_vma_dump(s, &cdev->mempool_vma[i]);
+ }
+ seq_puts(s, "RxQueue\n");
+ for (i = 0; i < CSM_DP_RX_TYPE_LAST; i++)
+ __csm_dp_rxqueue_vma_dump(s, &cdev->rxqueue_vma[i]);
+ }
+ mutex_unlock(&pdev->cdev_lock);
+
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_cdev,
+ csm_dp_debugfs_cdev_show, NULL);
+
+static int csm_dp_debugfs_dev_status_show(struct seq_file *s, void *unused)
+{
+ struct csm_dp_dev *pdev = (struct csm_dp_dev *)s->private;
+ struct csm_dp_core_stats *stats = &pdev->stats;
+ int i;
+
+ seq_printf(s, "TX: %lu\n", stats->tx_cnt);
+ seq_printf(s, "TX_ERR: %lu\n", stats->tx_err);
+ seq_printf(s, "TX_DROP: %lu\n", stats->tx_drop);
+ seq_printf(s, "RX: %lu\n", stats->rx_cnt);
+ seq_printf(s, "RX_BADMSG: %lu\n", stats->rx_badmsg);
+ seq_printf(s, "RX_DROP: %lu\n", stats->rx_drop);
+ seq_printf(s, "RX_INT: %lu\n", stats->rx_int);
+ seq_printf(s, "RX_BUDGET_OVF: %lu\n", stats->rx_budget_overflow);
+ seq_printf(s, "RX_IGNORE: %lu\n", stats->rx_poll_ignore);
+ for (i = 0; i < CSM_DP_MEM_TYPE_LAST; i++) {
+ seq_printf(s, "Mempool[%d]\n", i);
+ seq_printf(s, "MEM_POOL_IN_USE: %lu\n",
+ stats->mem_stats.mempool_mem_in_use[i]);
+ seq_printf(s, "MEM_DMA_MAPPED: %lu\n",
+ stats->mem_stats.mempool_mem_dma_mapped[i]);
+ seq_printf(s, "MEM_RING_IN_USE: %lu\n",
+ stats->mem_stats.mempool_ring_in_use[i]);
+ }
+ for (i = 0; i < CSM_DP_RX_TYPE_LAST; i++) {
+ seq_printf(s, "RXQ[%d]\n", i);
+ seq_printf(s, "MEM_RXQ_IN_USE: %lu\n",
+ stats->mem_stats.rxq_ring_in_use[i]);
+ }
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_dev_status,
+ csm_dp_debugfs_dev_status_show, NULL);
+
+static int csm_dp_debugfs_drv_show(struct seq_file *s, void *unused)
+{
+ struct csm_dp_drv *drv = (struct csm_dp_drv *)s->private;
+
+ seq_printf(s, "Driver: %llx\n", (u64)drv);
+ return 0;
+}
+
+CSM_DP_DEFINE_DEBUGFS_OPS(csm_dp_debugfs_drv,
+ csm_dp_debugfs_drv_show, NULL);
+
+static void csm_dp_debugfs_create_rxq_dir(struct dentry *parent,
+ struct csm_dp_dev *pdev)
+{
+ struct dentry *dentry = NULL, *root = NULL;
+ unsigned int type;
+
+ root = debugfs_create_dir("rxque", parent);
+
+ for (type = 0; type < CSM_DP_RX_TYPE_LAST; type++) {
+ dentry = debugfs_create_dir(csm_dp_rx_type_to_str(type),
+ root);
+
+ debugfs_create_file("config", 0444, dentry,
+ &pdev->rxq[type],
+ &csm_dp_debugfs_rxq_config_ops);
+
+ debugfs_create_file("runtime", 0444, dentry,
+ &pdev->rxq[type],
+ &csm_dp_debugfs_rxq_runtime_ops);
+
+ debugfs_create_file("opstats", 0444, dentry,
+ &pdev->rxq[type],
+ &csm_dp_debugfs_rxq_opstats_ops);
+
+ debugfs_create_file("refcnt", 0444, dentry,
+ &pdev->rxq[type],
+ &csm_dp_debugfs_rxq_refcnt_ops);
+ }
+}
+
+static void csm_dp_debugfs_create_ring_dir(struct dentry *parent,
+ struct csm_dp_mempool **mempool)
+{
+ struct dentry *dentry = NULL;
+
+ dentry = debugfs_create_dir("ring", parent);
+
+ debugfs_create_file("config", 0444, dentry, mempool,
+ &csm_dp_debugfs_ring_config_ops);
+
+ debugfs_create_file("runtime", 0444, dentry, mempool,
+ &csm_dp_debugfs_ring_runtime_ops);
+
+ debugfs_create_file("index", 0644, dentry, mempool,
+ &csm_dp_debugfs_ring_index_ops);
+
+ debugfs_create_file("data", 0644, dentry, mempool,
+ &csm_dp_debugfs_ring_data_ops);
+
+ debugfs_create_file("opstats", 0444, dentry, mempool,
+ &csm_dp_debugfs_ring_opstats_ops);
+}
+
+static void csm_dp_debugfs_create_mem_dir(struct dentry *parent,
+ struct csm_dp_mempool **mempool)
+{
+ struct dentry *dentry = NULL;
+
+ dentry = debugfs_create_dir("mem", parent);
+
+ debugfs_create_file("config", 0444, dentry, mempool,
+ &csm_dp_debugfs_mem_config_ops);
+
+ debugfs_create_file("offset", 0444, dentry, mempool,
+ &csm_dp_debugfs_mem_offset_ops);
+
+ debugfs_create_file("dump_size", 0644, dentry, mempool,
+ &csm_dp_debugfs_mem_dump_size_ops);
+
+ debugfs_create_file("data", 0644, dentry, mempool,
+ &csm_dp_debugfs_mem_data_ops);
+
+ debugfs_create_file("buffer_state", 0444, dentry, mempool,
+ &csm_dp_debugfs_mem_buffer_state_ops);
+}
+
+static void csm_dp_debugfs_create_mempool_dir(struct dentry *parent,
+ struct csm_dp_dev *pdev)
+{
+ struct dentry *dentry = NULL, *root = NULL;
+ unsigned int type;
+
+ root = debugfs_create_dir("mempool", parent);
+
+ debugfs_create_file("active", 0444, root, pdev,
+ &csm_dp_debugfs_mempool_active_ops);
+
+ for (type = 0; type < CSM_DP_MEM_TYPE_LAST; type++) {
+ dentry = debugfs_create_dir(csm_dp_mem_type_to_str(type), root);
+
+ csm_dp_debugfs_create_ring_dir(dentry, &pdev->mempool[type]);
+
+ csm_dp_debugfs_create_mem_dir(dentry, &pdev->mempool[type]);
+
+ debugfs_create_file("info", 0444, dentry,
+ &pdev->mempool[type],
+ &csm_dp_debugfs_mempool_info_ops);
+
+ debugfs_create_file("status", 0444, dentry,
+ &pdev->mempool[type],
+ &csm_dp_debugfs_mempool_status_ops);
+
+ debugfs_create_file("state", 0444, dentry,
+ &pdev->mempool[type],
+ &csm_dp_debugfs_mempool_state_ops);
+ }
+}
+
+void csm_dp_debugfs_init(struct csm_dp_drv *drv)
+{
+ struct dentry *dp_dev_entry;
+ int i;
+
+ drv->dent = debugfs_create_dir(CSM_DP_MODULE_NAME, 0);
+ debugfs_create_file("driver", 0444, drv->dent, drv,
+ &csm_dp_debugfs_drv_ops);
+
+ for (i = 0; i < CSM_DP_MAX_NUM_DEVS; i++) {
+ char buf[10];
+ struct csm_dp_dev *pdev = &drv->dp_devs[i];
+
+ snprintf(buf, sizeof(buf), "dev%d", i);
+ dp_dev_entry = debugfs_create_dir(buf, drv->dent);
+
+ debugfs_create_file("cdev", 0444, dp_dev_entry, pdev,
+ &csm_dp_debugfs_cdev_ops);
+
+ debugfs_create_file("mhi_control_dev", 0444, dp_dev_entry,
+ &pdev->mhi_control_dev, &csm_dp_debugfs_mhi_ops);
+
+ debugfs_create_file("mhi_data_dev", 0444, dp_dev_entry,
+ &pdev->mhi_data_dev, &csm_dp_debugfs_mhi_ops);
+
+ debugfs_create_file("status", 0444, dp_dev_entry, pdev,
+ &csm_dp_debugfs_dev_status_ops);
+
+ debugfs_create_file("start_recovery_mhi_control_dev",
+ 0644, dp_dev_entry, &pdev->mhi_control_dev,
+ &csm_dp_debugfs_start_recovery_control_dev_ops);
+
+ debugfs_create_file("start_recovery_mhi_data_dev",
+ 0644, dp_dev_entry, &pdev->mhi_data_dev,
+ &csm_dp_debugfs_start_recovery_data_dev_ops);
+
+ csm_dp_debugfs_create_mempool_dir(dp_dev_entry, pdev);
+
+ csm_dp_debugfs_create_rxq_dir(dp_dev_entry, pdev);
+ }
+}
+
+void csm_dp_debugfs_cleanup(struct csm_dp_drv *drv)
+{
+ debugfs_remove_recursive(drv->dent);
+ drv->dent = NULL;
+}
+
+#else
+
+int csm_dp_debugfs_init(struct csm_dp_drv *drv)
+{
+ return 0;
+}
+
+void csm_dp_debugfs_cleanup(struct csm_dp_drv *drv)
+{
+}
+#endif
diff --git a/drivers/char/qcom_csm_dp/qcom_csm_dp_mem.c b/drivers/char/qcom_csm_dp/qcom_csm_dp_mem.c
new file mode 100644
index 000000000000..6d3f4e2299a7
--- /dev/null
+++ b/drivers/char/qcom_csm_dp/qcom_csm_dp_mem.c
@@ -0,0 +1,1078 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/list.h>
+#include <linux/slab.h>
+
+#include "qcom_csm_dp.h"
+#include "qcom_csm_dp_mem.h"
+
+static struct csm_dp_mempool *csm_dp_mem_to_mempool(struct csm_dp_mem *mem)
+{
+ struct csm_dp_mempool *mempool = container_of(mem,
+ struct csm_dp_mempool, mem);
+ return mempool;
+}
+
+static struct csm_dp_mempool *csm_dp_mem_loc_to_mempool(struct csm_dp_mem_loc *loc)
+{
+ struct csm_dp_mem *mem = container_of(loc,
+ struct csm_dp_mem, loc);
+ return csm_dp_mem_to_mempool(mem);
+}
+
+static void csm_dp_mem_loc_set(struct csm_dp_mem_loc *loc, size_t size,
+ unsigned int mmap_cookie)
+{
+ loc->base = loc->cluster_kernel_addr[0];
+ loc->size = size;
+ loc->cookie = mmap_cookie;
+ loc->dma_mapped = false;
+}
+
+static int csm_dp_alloc_ring(size_t size, unsigned int mmap_cookie,
+ struct csm_dp_mem_loc *loc)
+{
+ unsigned int order;
+ struct page *page;
+
+ order = get_order(size);
+ if (order > get_order(CSM_DP_MEMPOOL_CLUSTER_SIZE)) {
+ pr_err("failed to allocate memory. Too Big %ld\n", size);
+ return -ENOMEM;
+ }
+ page = alloc_pages(GFP_KERNEL, order);
+ loc->last_cl_order = order;
+ loc->num_cluster = 1;
+ if (page) {
+ loc->page[0] = page;
+ loc->cluster_kernel_addr[0] = page_address(page);
+ csm_dp_mem_loc_set(loc, size, mmap_cookie);
+ loc->true_alloc_size += ((unsigned int)(1) << loc->last_cl_order) * PAGE_SIZE;
+ pr_info("Allocated ring memory of size %lu\n", loc->true_alloc_size);
+ return 0;
+ }
+ return -ENOMEM;
+}
+
+static void csm_dp_free_ring(struct csm_dp_mem_loc *loc)
+{
+ if (loc && loc->page[0]) {
+ __free_pages(loc->page[0], loc->last_cl_order);
+ pr_info("Free ring memory of size %lu\n",
+ ((unsigned int)(1) << loc->last_cl_order) * PAGE_SIZE);
+ memset(loc, 0, sizeof(*loc));
+ }
+}
+
+static struct page *csm_dp_alloc_zeroed_pages(unsigned int gfp_mask, unsigned int order)
+{
+ int i;
+ struct page *page = alloc_pages(gfp_mask, order);
+ void *page_addr;
+
+ if (!page)
+ return NULL;
+
+ page_addr = page_address(page);
+ for (i = 0; i < 1 << order; i++)
+ clear_page(page_addr + i * PAGE_SIZE);
+
+ return page;
+}
+
+static int csm_dp_buf_mem_alloc(size_t size,
+ unsigned int mmap_cookie,
+ struct csm_dp_mem_loc *loc)
+{
+ unsigned int order;
+ struct page *page;
+ int i;
+ unsigned long rem = size;
+ unsigned long len;
+ struct csm_dp_mempool *mempool = csm_dp_mem_loc_to_mempool(loc);
+
+ for (i = 0; i < loc->num_cluster; i++) {
+ if (i == loc->num_cluster - 1)
+ len = rem;
+ else
+ len = CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ order = get_order(len);
+ if (i == loc->num_cluster - 1)
+ loc->last_cl_order = order;
+ page = csm_dp_alloc_zeroed_pages(GFP_KERNEL, order);
+ if (!page)
+ goto error;
+ loc->page[i] = page;
+ loc->cluster_kernel_addr[i] = page_address(page);
+ rem -= len;
+ loc->true_alloc_size +=
+ (i == (loc->num_cluster - 1)) ?
+ ((unsigned int)(1) << order) *
+ PAGE_SIZE : CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ }
+ csm_dp_mem_loc_set(loc, size, mmap_cookie);
+ mempool->dp_dev->stats.mem_stats.mempool_mem_in_use[mempool->type] +=
+ loc->true_alloc_size;
+ pr_info("Allocated Mempool %u of size %lu\n",
+ mempool->type, loc->true_alloc_size);
+ return 0;
+error:
+ for (i = 0; i < loc->num_cluster; i++) {
+ if (loc->page[i]) {
+ if (i == loc->num_cluster - 1)
+ order = loc->last_cl_order;
+ else
+ order = get_order(CSM_DP_MEMPOOL_CLUSTER_SIZE);
+ __free_pages(loc->page[i], order);
+ loc->page[i] = NULL;
+ loc->true_alloc_size -=
+ (i == (loc->num_cluster - 1)) ? ((unsigned int)(1) << order) *
+ PAGE_SIZE : CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ }
+ }
+ loc->num_cluster = 0;
+ return -ENOMEM;
+}
+
+static void csm_dp_buf_mem_free(struct csm_dp_mem_loc *loc)
+{
+ int i;
+ struct csm_dp_mempool *mempool;
+ unsigned long to_free = loc->true_alloc_size;
+ unsigned int order = get_order(CSM_DP_MEMPOOL_CLUSTER_SIZE);
+
+ if (loc) {
+ mempool = csm_dp_mem_loc_to_mempool(loc);
+ for (i = 0; i < loc->num_cluster; i++) {
+ if (loc->page[i]) {
+ if (i == loc->num_cluster - 1)
+ order = loc->last_cl_order;
+ __free_pages(loc->page[i], order);
+ loc->true_alloc_size -=
+ (i == (loc->num_cluster - 1)) ?
+ ((unsigned int)(1) << order) * PAGE_SIZE :
+ CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ }
+ }
+ mempool->dp_dev->stats.mem_stats.mempool_mem_in_use[mempool->type] -=
+ (to_free - loc->true_alloc_size);
+ pr_info("Free Mempool %u of size %lu\n",
+ mempool->type, (to_free - loc->true_alloc_size));
+ memset(loc, 0, sizeof(*loc));
+ }
+}
+
+/*
+ * Get the MHI controller dev - needed for dma operations. Control and data
+ * channels refer to same MHI controller dev.
+ */
+static struct device *get_mhi_cntrl_dev(struct csm_dp_dev *pdev)
+{
+ atomic_inc(&pdev->mhi_control_dev.mhi_dev_refcnt);
+ atomic_inc(&pdev->mhi_data_dev.mhi_dev_refcnt);
+
+ if (csm_dp_mhi_is_ready(&pdev->mhi_control_dev))
+ return pdev->mhi_control_dev.mhi_dev->mhi_cntrl->cntrl_dev;
+ else if (csm_dp_mhi_is_ready(&pdev->mhi_data_dev))
+ return pdev->mhi_data_dev.mhi_dev->mhi_cntrl->cntrl_dev;
+
+ atomic_dec(&pdev->mhi_control_dev.mhi_dev_refcnt);
+ atomic_dec(&pdev->mhi_data_dev.mhi_dev_refcnt);
+
+ return NULL;
+}
+
+static void put_mhi_cntrl_dev(struct csm_dp_dev *pdev)
+{
+ atomic_dec(&pdev->mhi_control_dev.mhi_dev_refcnt);
+ atomic_dec(&pdev->mhi_data_dev.mhi_dev_refcnt);
+}
+
+int csm_dp_ring_init(struct csm_dp_ring *ring,
+ unsigned int ringsz,
+ unsigned int mmap_cookie)
+{
+ unsigned int allocsz = ringsz * sizeof(*ring->element);
+ char *aligned_ptr;
+ struct csm_dp_ring_element *elem_p;
+
+ /* cons and prod index space, aligned to cache line */
+ allocsz += 4 * cache_line_size();
+ allocsz = ALIGN(allocsz, cache_line_size());
+
+ if (csm_dp_alloc_ring(allocsz, mmap_cookie, &ring->loc)) {
+ pr_err("failed to allocate ring memory\n");
+ return -ENOMEM;
+ }
+
+ aligned_ptr = (char *)ALIGN((unsigned long)ring->loc.base,
+ cache_line_size());
+ ring->prod_head = (unsigned int *)aligned_ptr;
+ aligned_ptr += cache_line_size();
+ ring->prod_tail = (unsigned int *)aligned_ptr;
+ aligned_ptr += cache_line_size();
+ ring->cons_head = (unsigned int *)aligned_ptr;
+ aligned_ptr += cache_line_size();
+ ring->cons_tail = (unsigned int *)aligned_ptr;
+ aligned_ptr += cache_line_size();
+ elem_p = (struct csm_dp_ring_element *)aligned_ptr;
+ ring->element = elem_p;
+ ring->size = ringsz;
+ *ring->cons_tail = 0;
+ *ring->cons_head = 0;
+ *ring->prod_tail = 0;
+ *ring->prod_head = 0;
+
+ return 0;
+}
+
+void csm_dp_ring_cleanup(struct csm_dp_ring *ring)
+{
+ if (ring) {
+ csm_dp_free_ring(&ring->loc);
+ memset(ring, 0, sizeof(*ring));
+ }
+}
+
+int csm_dp_ring_get_cfg(struct csm_dp_ring *ring, struct csm_dp_ring_cfg *cfg)
+{
+ if (unlikely(!ring || !cfg))
+ return -EINVAL;
+ cfg->mmap.length = ring->loc.size;
+ cfg->mmap.cookie = ring->loc.cookie;
+
+ cfg->size = ring->size;
+ cfg->prod_head_off = csm_dp_vaddr_offset((void *)ring->prod_head,
+ ring->loc.base);
+ cfg->prod_tail_off = csm_dp_vaddr_offset((void *)ring->prod_tail,
+ ring->loc.base);
+ cfg->cons_head_off = csm_dp_vaddr_offset((void *)ring->cons_head,
+ ring->loc.base);
+ cfg->cons_tail_off = csm_dp_vaddr_offset((void *)ring->cons_tail,
+ ring->loc.base);
+ cfg->ringbuf_off = csm_dp_vaddr_offset((void *)ring->element,
+ ring->loc.base);
+ return 0;
+}
+
+int csm_dp_ring_read(struct csm_dp_ring *ring,
+ unsigned long *element_ptr)
+{
+ register unsigned int cons_head, cons_next;
+ register unsigned int prod_tail, mask;
+ unsigned long data;
+
+ if (unlikely(!ring))
+ return -EINVAL;
+
+ mask = ring->size - 1;
+
+again:
+ /*
+ * Test to see if the ring is empty.
+ * If not, advance cons_head and read the data
+ */
+ cons_head = *ring->cons_head;
+ prod_tail = *ring->prod_tail;
+ /* Get current cons_head and prod_tail */
+ rmb();
+ if ((cons_head & mask) == (prod_tail & mask)) {
+ /* Load ring elements */
+ rmb();
+ if (cons_head == *ring->cons_head && prod_tail == *ring->prod_tail) {
+ atomic_inc(&ring->opstats.read_empty);
+ return -EAGAIN;
+ }
+ goto again;
+ }
+ cons_next = cons_head + 1;
+ if (atomic_cmpxchg((atomic_t *)ring->cons_head,
+ cons_head,
+ cons_next) != cons_head)
+ goto again;
+
+ /* Read the ring */
+ data = ring->element[(cons_head & mask)].element_data;
+ /* Get current element */
+ rmb();
+
+ if (element_ptr)
+ *element_ptr = data;
+
+ atomic_inc(&ring->opstats.read_ok);
+
+ /* Potential two consumer is updating */
+ while (atomic_cmpxchg((atomic_t *)ring->cons_tail,
+ cons_head, cons_next) != cons_head)
+ ;
+
+ return 0;
+}
+
+int csm_dp_ring_write(struct csm_dp_ring *ring, unsigned long data)
+{
+ register unsigned int prod_head, prod_next;
+ register unsigned int cons_tail, mask;
+
+ if (unlikely(!ring))
+ return -EINVAL;
+
+ mask = ring->size - 1;
+
+again:
+ /*
+ * Test to see if the ring is full.
+ * If not, advance prod_head and write the data
+ */
+ prod_head = *ring->prod_head;
+ cons_tail = *ring->cons_tail;
+ /* Get current prod_head and cons_tail */
+ rmb();
+ prod_next = prod_head + 1;
+ if ((prod_next & mask) == (cons_tail & mask)) {
+ /* Load ring elements */
+ rmb();
+ if (prod_head == *ring->prod_head && cons_tail == *ring->cons_tail) {
+ atomic_inc(&ring->opstats.write_full);
+ return -EAGAIN;
+ }
+ goto again;
+ }
+ if (atomic_cmpxchg((atomic_t *)ring->prod_head,
+ prod_head,
+ prod_next) != prod_head)
+ goto again;
+
+ ring->element[(prod_head & mask)].element_data = data;
+ /* Ensure element is written */
+ wmb();
+
+ atomic_inc(&ring->opstats.write_ok);
+
+ /* Potential two producer is updating */
+ while (atomic_cmpxchg((atomic_t *)ring->prod_tail,
+ prod_head, prod_next) != prod_head)
+ ;
+
+ return 0;
+}
+
+bool csm_dp_ring_is_empty(struct csm_dp_ring *ring)
+{
+ unsigned int prod_tail, cons_tail;
+
+ prod_tail = *ring->prod_tail;
+ cons_tail = *ring->cons_tail;
+ if (prod_tail == cons_tail)
+ return true;
+ return false;
+}
+
+static int csm_dp_mem_init(struct csm_dp_mem *mem,
+ unsigned int bufcnt,
+ unsigned int bufsz,
+ unsigned int cookie)
+{
+ unsigned int num_buf_cl; /* number of buffers per cluster */
+ unsigned int num_cl; /* number of clusters */
+ unsigned long size;
+ unsigned long rem_size = 0;
+ unsigned int rem_buf;
+
+ mem->buf_cnt = bufcnt;
+ mem->buf_sz = ALIGN(bufsz, cache_line_size());
+ mem->buf_overhead_sz = CSM_DP_L1_CACHE_BYTES;
+
+ if (csm_dp_buf_true_size(mem) > CSM_DP_MEMPOOL_CLUSTER_SIZE) {
+ pr_err("buf_true_size too big %d (CLUSTER_SIZE %d)\n",
+ csm_dp_buf_true_size(mem), CSM_DP_MEMPOOL_CLUSTER_SIZE);
+ return -ENOMEM;
+ }
+
+ num_buf_cl = CSM_DP_MEMPOOL_CLUSTER_SIZE / csm_dp_buf_true_size(mem);
+ num_cl = bufcnt / num_buf_cl;
+ size = (long)num_cl * CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ rem_buf = bufcnt % num_buf_cl;
+ if (rem_buf) {
+ num_cl++;
+ rem_size = csm_dp_buf_true_size(mem) * rem_buf;
+ }
+ if (num_cl > MAX_CSM_DP_MEMPOOL_CLUSTERS) {
+ pr_err("mempool size too big. num_cl %d\n", num_cl);
+ return -ENOMEM;
+ }
+ if (ULONG_MAX / CSM_DP_MEMPOOL_CLUSTER_SIZE < num_cl) {
+ pr_err("mempool size too big. num_cl %d CLUSTER_SIZE %d\n",
+ num_cl, CSM_DP_MEMPOOL_CLUSTER_SIZE);
+ return -ENOMEM;
+ }
+ mem->loc.num_cluster = num_cl;
+ mem->loc.buf_per_cluster = num_buf_cl;
+ size += rem_size;
+ if (csm_dp_buf_mem_alloc(size, cookie, &mem->loc)) {
+ pr_err("failed to allocate DMA memory\n");
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static void csm_dp_mem_cleanup(struct csm_dp_mem *mem)
+{
+ struct csm_dp_mempool *mempool = NULL;
+ struct csm_dp_dev *pdev = NULL;
+ int i;
+ unsigned long size;
+
+ if (!mem) {
+ pr_err("csm_dp_mem is NULL!\n");
+ return;
+ }
+ mempool = csm_dp_mem_to_mempool(mem);
+
+ if (!mempool) {
+ pr_err("csm_dp_mempool is NULL!\n");
+ return;
+ }
+ pdev = mempool->dp_dev;
+
+ spin_lock(&mempool->lock);
+ if (mem->loc.dma_mapped && mempool->dev) {
+ size = mem->loc.size;
+ for (i = 0; i < mem->loc.num_cluster; i++) {
+ if (i == mem->loc.num_cluster - 1) {
+ dma_unmap_single(mempool->dev,
+ mem->loc.cluster_dma_addr[i],
+ size,
+ mem->loc.direction);
+
+ if (pdev)
+ pdev->stats.mem_stats.mempool_mem_dma_mapped[mempool->type]
+ -=
+ size;
+ } else {
+ dma_unmap_single(mempool->dev,
+ mem->loc.cluster_dma_addr[i],
+ CSM_DP_MEMPOOL_CLUSTER_SIZE,
+ mem->loc.direction);
+
+ if (pdev)
+ pdev->stats.mem_stats.mempool_mem_dma_mapped[mempool->type]
+ -=
+ CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ size -= CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ }
+ }
+ pr_info("DMA Unmap Mempool %u of size %lu\n", mempool->type, mem->loc.size);
+ }
+ mem->loc.dma_mapped = false;
+ spin_unlock(&mempool->lock);
+
+ csm_dp_buf_mem_free(&mem->loc);
+ memset(mem, 0, sizeof(*mem));
+
+ mempool->dev = NULL;
+
+ put_mhi_cntrl_dev(pdev);
+}
+
+static int csm_dp_mem_get_cfg(struct csm_dp_mem *mem,
+ struct csm_dp_mem_cfg *cfg)
+{
+ cfg->mmap.length = mem->loc.size;
+ cfg->mmap.cookie = mem->loc.cookie;
+
+ cfg->buf_sz = mem->buf_sz;
+ cfg->buf_cnt = mem->buf_cnt;
+ cfg->buf_overhead_sz = CSM_DP_L1_CACHE_BYTES;
+ cfg->cluster_size = CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ cfg->num_cluster = mem->loc.num_cluster;
+ cfg->buf_per_cluster = mem->loc.buf_per_cluster;
+
+ pr_debug("buf_sz %d buf_cnt %d buf_overhead_sz %d cluster_size %d num_cluster %d buf_per_cluster %d\n",
+ cfg->buf_sz, cfg->buf_cnt, cfg->buf_overhead_sz,
+ cfg->cluster_size, cfg->num_cluster, cfg->buf_per_cluster);
+
+ return 0;
+}
+
+static void csm_dp_mempool_init(struct csm_dp_mempool *mempool)
+{
+ struct csm_dp_mem *mem = &mempool->mem;
+ struct csm_dp_ring *ring = &mempool->ring;
+ unsigned long element_data;
+ int i, j;
+ struct csm_dp_buf_cntrl *p;
+ unsigned int cl_buf_cnt;
+ unsigned int buf_index = 0;
+ char *cl_start;
+
+ if (!csm_dp_mem_type_is_valid(mempool->type))
+ return;
+
+ for (j = 0; j < mem->loc.num_cluster; j++) {
+ element_data = (long)j * CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ cl_start = mem->loc.cluster_kernel_addr[j];
+ if (j == mem->loc.num_cluster - 1)
+ cl_buf_cnt = mem->buf_cnt -
+ (mem->loc.buf_per_cluster * j);
+ else
+ cl_buf_cnt = mem->loc.buf_per_cluster;
+ for (i = 0; i < cl_buf_cnt; i++) {
+ p = (struct csm_dp_buf_cntrl *)(cl_start +
+ (i * csm_dp_buf_true_size(mem)));
+ p->signature = CSM_DP_BUFFER_SIG;
+ p->fence = CSM_DP_BUFFER_FENCE_SIG;
+ p->state = CSM_DP_BUF_STATE_KERNEL_FREE;
+ p->mem_type = mempool->type;
+ p->buf_index = buf_index;
+ p->next_packet = NULL;
+ if (!csm_dp_mem_type_is_ul(mempool->type))
+ p->xmit_status = CSM_DP_XMIT_OK;
+ /* pointing to start of user data */
+ ring->element[buf_index].element_data =
+ element_data + mem->buf_overhead_sz;
+ element_data += csm_dp_buf_true_size(mem);
+ buf_index++;
+ }
+ }
+ *ring->cons_head = *ring->cons_tail = 0;
+ *ring->prod_head = *ring->prod_tail = mem->buf_cnt - 1;
+ /* Ensure all the data are written */
+ wmb();
+}
+
+static struct csm_dp_mempool *__csm_dp_mempool_alloc(struct csm_dp_dev *pdev,
+ enum csm_dp_mem_type type,
+ unsigned int buf_sz,
+ unsigned int buf_cnt,
+ unsigned int ring_sz,
+ bool may_map)
+{
+ struct csm_dp_mempool *mempool;
+ unsigned int cookie;
+
+ mempool = kzalloc(sizeof(*mempool), GFP_KERNEL);
+ if (IS_ERR_OR_NULL(mempool)) {
+ pr_err("failed to allocate mempool\n");
+ return NULL;
+ }
+
+ mempool->dp_dev = pdev;
+ mempool->type = type;
+ mempool->signature = CSM_DP_MEMPOOL_SIG;
+
+ cookie = CSM_DP_MMAP_COOKIE(type, CSM_DP_MMAP_TYPE_MEM);
+ if (csm_dp_mem_init(&mempool->mem, buf_cnt, buf_sz, cookie)) {
+ pr_err("failed to initialize memory\n");
+ goto cleanup;
+ }
+
+ if (may_map) {
+ struct device *dev = get_mhi_cntrl_dev(pdev);
+
+ if (dev) {
+ int ret = csm_dp_mempool_dma_map(dev, mempool);
+
+ put_mhi_cntrl_dev(pdev);
+ if (ret)
+ goto cleanup_mem;
+ }
+ }
+ cookie = CSM_DP_MMAP_COOKIE(type, CSM_DP_MMAP_TYPE_RING);
+ if (csm_dp_ring_init(&mempool->ring, ring_sz, cookie)) {
+ pr_err("failed to initialize ring\n");
+ goto cleanup_mem;
+ }
+ mempool->dp_dev->stats.mem_stats.mempool_ring_in_use[mempool->type] +=
+ mempool->ring.loc.true_alloc_size;
+ csm_dp_mempool_init(mempool);
+
+ pr_debug("mempool is created, type=%u bufsz=%u bufcnt=%u\n",
+ type, buf_sz, buf_cnt);
+
+ return mempool;
+
+cleanup_mem:
+ csm_dp_mem_cleanup(&mempool->mem);
+cleanup:
+ kfree(mempool);
+ return NULL;
+}
+
+static void csm_dp_mempool_release(struct csm_dp_mempool *mempool)
+{
+ unsigned long to_release;
+
+ if (mempool) {
+ enum csm_dp_mem_type type = mempool->type;
+
+ to_release = ((unsigned int)(1) << mempool->ring.loc.last_cl_order) * PAGE_SIZE;
+
+ mempool->signature = CSM_DP_MEMPOOL_SIG_BAD;
+ /* Ensure all the data are written before cleanup*/
+ wmb();
+ csm_dp_mem_cleanup(&mempool->mem);
+ csm_dp_ring_cleanup(&mempool->ring);
+ mempool->dp_dev->stats.mem_stats.mempool_ring_in_use[mempool->type] -= to_release;
+ kfree(mempool);
+ pr_debug("mempool is freed, type=%u\n", type);
+ }
+}
+
+#define CSM_DP_MEMPOOL_RELEASE_SLEEP 200 /* 200 ms */
+void csm_dp_mempool_release_no_delay(struct csm_dp_mempool *mempool)
+{
+ unsigned int out_xmit, out_xmit1;
+
+ if (!mempool)
+ return;
+ /* wait for all tx buffers done */
+
+ out_xmit1 = atomic_read(&mempool->out_xmit);
+ if (out_xmit1) {
+ msleep(CSM_DP_MEMPOOL_RELEASE_SLEEP);
+ out_xmit = atomic_read(&mempool->out_xmit);
+ if (out_xmit)
+ pr_err("mempool %p out_xmit changed from %d to %d after %d ms\n",
+ mempool, out_xmit1, out_xmit, CSM_DP_MEMPOOL_RELEASE_SLEEP);
+ }
+ csm_dp_mempool_release(mempool);
+}
+
+int csm_dp_mempool_dma_map(struct device *dev, /* device for iommu ops */
+ struct csm_dp_mempool *mpool)
+{
+ enum dma_data_direction direction;
+ int i, k, err;
+ unsigned long size;
+ struct csm_dp_mem_loc *loc;
+
+ loc = &mpool->mem.loc;
+ if (loc->dma_mapped)
+ return 0;
+ if (csm_dp_mem_type_is_ul(mpool->type))
+ direction = DMA_BIDIRECTIONAL; /* rx, tx for rx loopback */
+ else
+ direction = DMA_TO_DEVICE;
+ size = loc->size;
+
+ err = dma_set_mask(dev, DMA_BIT_MASK(CSM_DP_DMA_MASK));
+ if (err) {
+ pr_err("Cannot set proper DMA mask\n");
+ return err;
+ }
+
+ for (i = 0; i < loc->num_cluster; i++) {
+ if (i == loc->num_cluster - 1) {
+ loc->cluster_dma_addr[i] =
+ dma_map_single(dev,
+ loc->cluster_kernel_addr[i],
+ size,
+ direction);
+ mpool->dp_dev->stats.mem_stats.mempool_mem_dma_mapped[mpool->type] += size;
+ } else {
+ loc->cluster_dma_addr[i] =
+ dma_map_single(dev,
+ loc->cluster_kernel_addr[i],
+ CSM_DP_MEMPOOL_CLUSTER_SIZE,
+ direction);
+ mpool->dp_dev->stats.mem_stats.mempool_mem_dma_mapped[mpool->type] +=
+ CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ size -= CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ }
+ if (dma_mapping_error(dev, loc->cluster_dma_addr[i]))
+ goto error;
+ }
+ pr_info("DMA map Mempool %u of size %lu\n", mpool->type, loc->size);
+ mpool->mem.loc.dma_mapped = true;
+ mpool->mem.loc.direction = direction;
+ mpool->dev = dev;
+ return 0;
+error:
+ for (k = 0; k < i - 1; k++) {
+ dma_unmap_single(dev, loc->cluster_dma_addr[k],
+ CSM_DP_MEMPOOL_CLUSTER_SIZE,
+ loc->direction);
+ }
+ return -ENOMEM;
+}
+
+struct csm_dp_mempool *csm_dp_mempool_alloc(struct csm_dp_dev *pdev,
+ enum csm_dp_mem_type type,
+ unsigned int buf_sz,
+ unsigned int buf_cnt,
+ bool may_dma_map)
+{
+ struct csm_dp_mempool *mempool;
+ unsigned int ring_sz;
+
+ if (unlikely(!buf_sz || !buf_cnt || !csm_dp_mem_type_is_valid(type)))
+ return NULL;
+ if (unlikely(((ULONG_MAX) / (buf_sz + CSM_DP_L1_CACHE_BYTES) < buf_cnt)))
+ return NULL;
+ if (buf_sz > CSM_DP_MAX_DL_MSG_LEN) {
+ pr_err("mempool alloc buffer size %d exceeds limit %d\n",
+ buf_sz, CSM_DP_MAX_DL_MSG_LEN);
+ return NULL;
+ }
+
+ ring_sz = csm_dp_calc_ring_size(buf_cnt);
+ if (unlikely(!ring_sz))
+ return NULL;
+
+ mutex_lock(&pdev->mempool_lock);
+ mempool = pdev->mempool[type];
+ if (mempool) {
+ if (!csm_dp_mem_type_is_ul(type)) {
+ pr_err("can't use existing mempool, type=%u\n", type);
+ mempool = NULL;
+ goto done;
+ }
+ goto mempool_hold;
+ }
+
+ mempool = __csm_dp_mempool_alloc(pdev, type, buf_sz,
+ buf_cnt, ring_sz, may_dma_map);
+ if (!mempool)
+ goto done;
+ atomic_set(&mempool->ref, 1);
+ atomic_set(&mempool->out_xmit, 0);
+ spin_lock_init(&mempool->lock);
+ pdev->mempool[type] = mempool;
+ goto done;
+mempool_hold:
+ if (!csm_dp_mempool_hold(mempool))
+ mempool = NULL;
+done:
+ mutex_unlock(&pdev->mempool_lock);
+ return mempool;
+}
+
+void csm_dp_mempool_free(struct csm_dp_mempool *mempool)
+{
+ struct csm_dp_dev *pdev = mempool->dp_dev;
+
+ if (!mempool)
+ return;
+ pr_info("Free mempool type %d\n", mempool->type);
+ csm_dp_mempool_release_no_delay(mempool);
+ pdev->mempool[mempool->type] = NULL;
+ /* Ensure all the data are freed */
+ wmb();
+}
+
+int csm_dp_mempool_get_cfg(struct csm_dp_mempool *mempool,
+ struct csm_dp_mempool_cfg *cfg)
+{
+ if (unlikely(!mempool || !cfg))
+ return -EINVAL;
+
+ cfg->type = mempool->type;
+ csm_dp_mem_get_cfg(&mempool->mem, &cfg->mem);
+ csm_dp_ring_get_cfg(&mempool->ring, &cfg->ring);
+ return 0;
+}
+
+/* For CSM_DP_MEM_TYPE_UL_* pool only */
+int csm_dp_mempool_put_buf(struct csm_dp_mempool *mempool, void *vaddr)
+{
+ struct csm_dp_mem *mem;
+ unsigned long offset;
+ int ret;
+ struct csm_dp_buf_cntrl *p;
+ unsigned int buf_index;
+ unsigned int cluster;
+
+ if (unlikely(!mempool || !vaddr))
+ return -EINVAL;
+
+ mem = &mempool->mem;
+ p = (vaddr - mem->buf_overhead_sz);
+ buf_index = p->buf_index;
+ if (buf_index >= mem->buf_cnt) {
+ pr_err("buf_index %d exceed %d\n", buf_index, mem->buf_cnt);
+ return -EINVAL;
+ }
+ cluster = buf_index / mem->loc.buf_per_cluster;
+ offset = ((long)cluster * CSM_DP_MEMPOOL_CLUSTER_SIZE) +
+ (buf_index % mem->loc.buf_per_cluster) *
+ csm_dp_buf_true_size(mem);
+ if (p->signature != CSM_DP_BUFFER_SIG) {
+ mempool->stats.invalid_buf_put++;
+ pr_err("mempool %llx type %d buffer at offset %ld corrupted, sig %x, exp %x\n",
+ (u64)mempool, mempool->type,
+ offset, p->signature, CSM_DP_BUFFER_SIG);
+ return -EINVAL;
+ }
+ if (p->fence != CSM_DP_BUFFER_FENCE_SIG) {
+ mempool->stats.invalid_buf_put++;
+ pr_err("mempool %llx type %d buffer at offset %ld corrupted, fence %x, exp %x\n",
+ (u64)mempool, mempool->type,
+ offset, p->fence, CSM_DP_BUFFER_FENCE_SIG);
+ pr_err("vaddr %llx p %llx\n", (u64)vaddr, (u64)p);
+ return -EINVAL;
+ }
+ p->state = CSM_DP_BUF_STATE_KERNEL_FREE;
+ offset += sizeof(struct csm_dp_buf_cntrl);
+
+ ret = csm_dp_ring_write(&mempool->ring, (unsigned long)offset);
+ if (ret)
+ mempool->stats.buf_put_err++;
+ else
+ mempool->stats.buf_put++;
+
+ return ret;
+}
+
+/* For CSM_DP_MEM_TYPE_UL_* pool only */
+void *csm_dp_mempool_get_buf(struct csm_dp_mempool *mempool,
+ unsigned int *cluster, unsigned int *c_offset)
+{
+ struct csm_dp_mem *mem;
+ unsigned long val;
+ void *ptr;
+ struct csm_dp_buf_cntrl *p;
+
+ if (unlikely(!mempool))
+ return NULL;
+
+ if (csm_dp_ring_read(&mempool->ring, &val)) {
+ mempool->stats.buf_get_err++;
+ return NULL;
+ }
+
+ mem = &mempool->mem;
+ *cluster = val >> CSM_DP_MEMPOOL_CLUSTER_SHIFT;
+ *c_offset = val & CSM_DP_MEMPOOL_CLUSTER_MASK;
+ ptr = (char *)mempool->mem.loc.cluster_kernel_addr[*cluster] +
+ *c_offset;
+ if ((*c_offset - mem->buf_overhead_sz) % csm_dp_buf_true_size(mem)) {
+ mempool->stats.invalid_buf_get++;
+ pr_err("get unaligned buffer from ring, buf true size %d offset %d\n",
+ csm_dp_buf_true_size(mem), *c_offset);
+ return NULL;
+ }
+ p = ptr - mem->buf_overhead_sz;
+ if (p->signature != CSM_DP_BUFFER_SIG) {
+ mempool->stats.invalid_buf_get++;
+ pr_err("mempool type %d buffer at %d corrupted, %x, exp %x\n",
+ *c_offset, mempool->type,
+ p->signature, CSM_DP_BUFFER_SIG);
+ return NULL;
+ }
+ if (p->fence != CSM_DP_BUFFER_FENCE_SIG) {
+ mempool->stats.invalid_buf_get++;
+ pr_err("mempool type %d buffer at %d corrupted, fence %x, exp %x\n",
+ *c_offset, mempool->type,
+ p->fence, CSM_DP_BUFFER_FENCE_SIG);
+ return NULL;
+ }
+ mempool->stats.buf_get++;
+ return ptr;
+}
+
+void get_mempool_buf_status(struct csm_dp_mempool *mempool)
+{
+ struct csm_dp_dev *pdev = mempool->dp_dev;
+ struct csm_dp_mem *mem = &mempool->mem;
+ struct csm_dp_buf_cntrl *p = NULL;
+ char *cl_start = NULL;
+ unsigned int cl_buf_cnt;
+ unsigned int k_free = 0, u_free = 0, u_recev = 0, k_msg_q_app = 0, k_recev_dma = 0;
+ unsigned int k_tx_dma = 0, k_tx_dma_cmp = 0, u_alloc = 0;
+ void *buf = NULL;
+ int i, j;
+
+ if (mempool) {
+ if (!csm_dp_mem_type_is_valid(mempool->type))
+ return;
+
+ for (j = 0; j < mem->loc.num_cluster; j++) {
+ cl_start = mem->loc.cluster_kernel_addr[j];
+ if (j == mem->loc.num_cluster - 1)
+ cl_buf_cnt = mem->buf_cnt -
+ (mem->loc.buf_per_cluster * j);
+ else
+ cl_buf_cnt = mem->loc.buf_per_cluster;
+ for (i = 0; i < cl_buf_cnt; i++) {
+ p = (struct csm_dp_buf_cntrl *)(cl_start +
+ (i * csm_dp_buf_true_size(mem)));
+
+ if (!p)
+ break;
+ buf = (char *)p + CSM_DP_L1_CACHE_BYTES;
+ if (!buf)
+ break;
+ if (p->state == CSM_DP_BUF_STATE_USER_RECV)
+ u_recev++;
+ else if (p->state == CSM_DP_BUF_STATE_KERNEL_RECVCMP_MSGQ_TO_APP)
+ k_msg_q_app++;
+ else if (p->state == CSM_DP_BUF_STATE_KERNEL_FREE)
+ k_free++;
+ else if (p->state == CSM_DP_BUF_STATE_KERNEL_ALLOC_RECV_DMA)
+ k_recev_dma++;
+ else if (p->state == CSM_DP_BUF_STATE_USER_FREE)
+ u_free++;
+ else if (p->state == CSM_DP_BUF_STATE_KERNEL_XMIT_DMA)
+ k_tx_dma++;
+ else if (p->state == CSM_DP_BUF_STATE_KERNEL_XMIT_DMA_COMP)
+ k_tx_dma_cmp++;
+ else if (p->state == CSM_DP_BUF_STATE_USER_ALLOC)
+ u_alloc++;
+ }
+ }
+ pr_err("%s Buffer status for bus %d VF %d\n"
+ "KERNEL_FREE:%u\n"
+ "KERNEL_ALLOC_RECV_DMA:%u\n"
+ "KERNEL_RECVCMP_MSGQ_TO_APP:%u\n"
+ "KERNEL_XMIT_DMA:%u\n"
+ "KERNEL_XMIT_DMA_COMP:%u\n"
+ "USER_FREE:%u\n"
+ "USER_ALLOC:%u\n"
+ "USER_RECV:%u\n",
+ csm_dp_mem_type_to_str(mempool->type),
+ pdev->bus_num,
+ pdev->vf_num,
+ k_free,
+ k_recev_dma,
+ k_msg_q_app,
+ k_tx_dma,
+ k_tx_dma_cmp,
+ u_free,
+ u_alloc,
+ u_recev);
+ }
+}
+
+void free_rx_ring_buffers(struct csm_dp_mempool *mempool, bool probe)
+{
+ struct csm_dp_dev *pdev = mempool->dp_dev;
+ struct csm_dp_mem *mem = &mempool->mem;
+ struct csm_dp_buf_cntrl *p = NULL;
+ struct csm_dp_buf_cntrl *packet_start, *tmp;
+ struct csm_dp_rxqueue *rxq;
+ char *cl_start = NULL;
+ unsigned int cl_buf_cnt;
+ void *buf = NULL;
+ int i, j, free_count = 0;
+ struct task_struct *task;
+ bool task_active = true;
+ unsigned int cluster, c_offset;
+ unsigned long offset;
+
+ /* Check if L2 is running and has a valid PID on mhi_probe */
+ if (probe && pdev->pid != -EINVAL) {
+ task = pid_task(find_vpid(pdev->pid), PIDTYPE_PID);
+
+ if (!task || strncmp(pdev->pid_name, task->comm, TASK_COMM_LEN) != 0) {
+ task_active = false;
+ pr_info("pid %d l2 task not active\n", pdev->pid);
+ }
+ }
+
+ if (mempool) {
+ if (!csm_dp_mem_type_is_valid(mempool->type))
+ return;
+
+ if (mempool->type == CSM_DP_MEM_TYPE_UL_DATA) {
+ packet_start = pdev->pending_packets;
+ while (packet_start) {
+ tmp = packet_start->next_packet;
+ packet_start->next_packet = NULL;
+ csm_dp_mempool_put_buf(mempool, packet_start + 1);
+ packet_start = tmp;
+ free_count++;
+ }
+ }
+
+ if (mempool->type == CSM_DP_MEM_TYPE_UL_CONTROL) {
+ rxq = &pdev->rxq[CSM_DP_RX_TYPE_FAPI];
+ while (rxq && !csm_dp_ring_is_empty(rxq->ring)) {
+ if (csm_dp_ring_read(rxq->ring, &offset)) {
+ pr_err("RxQ ring read failed\n");
+ break;
+ }
+ buf = csm_dp_mem_offset_addr(&mempool->mem, offset,
+ &cluster, &c_offset);
+ csm_dp_mempool_put_buf(mempool, buf);
+ free_count++;
+ }
+ if (!csm_dp_ring_is_empty(rxq->ring))
+ pr_err("Not all RX control channel packets freed\n");
+ }
+
+ for (j = 0; j < mem->loc.num_cluster; j++) {
+ cl_start = mem->loc.cluster_kernel_addr[j];
+ if (j == mem->loc.num_cluster - 1)
+ cl_buf_cnt = mem->buf_cnt -
+ (mem->loc.buf_per_cluster * j);
+ else
+ cl_buf_cnt = mem->loc.buf_per_cluster;
+ for (i = 0; i < cl_buf_cnt; i++) {
+ p = (struct csm_dp_buf_cntrl *)(cl_start +
+ (i * csm_dp_buf_true_size(mem)));
+
+ if (!p)
+ break;
+ buf = (char *)p + CSM_DP_L1_CACHE_BYTES;
+ if (!buf)
+ break;
+
+ if (p->state == CSM_DP_BUF_STATE_KERNEL_RECVCMP_MSGQ_TO_APP) {
+ csm_dp_mempool_put_buf(mempool, buf);
+ free_count++;
+ }
+ if (list_empty(&pdev->cdev_head) || !task_active) {
+ if (p->state == CSM_DP_BUF_STATE_USER_RECV) {
+ csm_dp_mempool_put_buf(mempool, buf);
+ free_count++;
+ }
+ }
+ if (probe) {
+ if (p->state == CSM_DP_BUF_STATE_KERNEL_ALLOC_RECV_DMA) {
+ csm_dp_mempool_put_buf(mempool, buf);
+ free_count++;
+ }
+ }
+ }
+ }
+ pr_info("%s %d RX buffers freed for bus %d VF %d\n",
+ csm_dp_mem_type_to_str(mempool->type),
+ free_count, pdev->bus_num, pdev->vf_num);
+ }
+}
+
+struct csm_dp_mempool *csm_dp_get_mempool(struct csm_dp_dev *pdev,
+ struct csm_dp_buf_cntrl *buf_cntrl,
+ unsigned int *cluster)
+{
+ struct csm_dp_mempool *mempool;
+
+ if (!csm_dp_mem_type_is_valid(buf_cntrl->mem_type))
+ return NULL;
+
+ mempool = pdev->mempool[buf_cntrl->mem_type];
+ if (!mempool)
+ return NULL;
+
+ if (buf_cntrl->buf_index >= U16_MAX * mempool->mem.loc.buf_per_cluster)
+ return NULL;
+
+ if (cluster)
+ *cluster = csm_dp_mem_get_cluster(&mempool->mem, buf_cntrl->buf_index);
+
+ return mempool;
+}
+
+uint16_t csm_dp_mem_get_cluster(struct csm_dp_mem *mem, unsigned int buf_index)
+{
+ if (buf_index >= U16_MAX * mem->loc.buf_per_cluster) {
+ pr_err("invalid buf_index\n");
+ return U16_MAX;
+ }
+
+ return buf_index / mem->loc.buf_per_cluster;
+}
diff --git a/drivers/char/qcom_csm_dp/qcom_csm_dp_mem.h b/drivers/char/qcom_csm_dp/qcom_csm_dp_mem.h
new file mode 100644
index 000000000000..4abee1e8d6b6
--- /dev/null
+++ b/drivers/char/qcom_csm_dp/qcom_csm_dp_mem.h
@@ -0,0 +1,292 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __QCOM_CSM_DP_MEM_H__
+#define __QCOM_CSM_DP_MEM_H__
+
+#include <linux/types.h>
+#include <linux/qcom_csm_dp_ioctl.h>
+
+#define MAX_CSM_DP_MEMPOOL_SIZE (1024L * 1024 * 1024 * 16)
+#define CSM_DP_MEMPOOL_CLUSTER_SIZE (1024 * 1024 * 2) /* must be > CSM_DP_MAX_DL_MSG_LEN */
+#define CSM_DP_MEMPOOL_CLUSTER_SHIFT 21
+#define CSM_DP_MEMPOOL_CLUSTER_MASK (CSM_DP_MEMPOOL_CLUSTER_SIZE - 1)
+#define MAX_CSM_DP_MEMPOOL_CLUSTERS \
+ (MAX_CSM_DP_MEMPOOL_SIZE / CSM_DP_MEMPOOL_CLUSTER_SIZE)
+
+struct csm_dp_mem_loc {
+ size_t size; /* size of memory chunk */
+ void *base; /* virtual address of first cluster.
+ * for ring with one cluster only
+ */
+ unsigned int cookie; /* mmap cookie */
+ struct page *page[MAX_CSM_DP_MEMPOOL_CLUSTERS];
+ unsigned int last_cl_order;
+ unsigned int num_cluster; /* number of cluster, 1 for ring */
+ char *cluster_kernel_addr[MAX_CSM_DP_MEMPOOL_CLUSTERS];
+
+ /* for CSM_DP_MMAP_TYPE_MEM */
+ dma_addr_t cluster_dma_addr[MAX_CSM_DP_MEMPOOL_CLUSTERS];
+ enum dma_data_direction direction;
+ bool dma_mapped;
+ unsigned int buf_per_cluster;
+ unsigned long true_alloc_size;
+};
+
+struct csm_dp_mem {
+ struct csm_dp_mem_loc loc; /* location */
+ unsigned int buf_cnt; /* buffer counter */
+ unsigned int buf_sz; /* buffer size */
+ unsigned int buf_headroom_sz; /* headroom unused for now */
+ unsigned int buf_overhead_sz; /* buffer overhead size */
+};
+
+struct csm_dp_ring_opstats {
+ atomic_t read_ok;
+ atomic_t read_empty;
+
+ atomic_t write_ok;
+ atomic_t write_full;
+};
+
+struct csm_dp_ring {
+ struct csm_dp_mem_loc loc; /* location */
+ unsigned int size; /* size of ring(power of 2) */
+ unsigned int *cons_head; /* consumer index header */
+ unsigned int *cons_tail; /* consumer index tail */
+ unsigned int *prod_head; /* producer index header */
+ unsigned int *prod_tail; /* producer index tail */
+ struct csm_dp_ring_element *element; /* ring element */
+ struct csm_dp_ring_opstats opstats;
+};
+
+struct csm_dp_mempool_stats {
+ unsigned long buf_put;
+ unsigned long buf_get;
+ unsigned long invalid_buf_put;
+ unsigned long invalid_buf_get;
+ unsigned long buf_put_err;
+ unsigned long buf_get_err;
+};
+
+#define CSM_DP_MEMPOOL_SIG 0xdeadbeef
+#define CSM_DP_MEMPOOL_SIG_BAD 0xbeefdead
+
+struct csm_dp_mempool {
+ unsigned int signature;
+ struct csm_dp_dev *dp_dev;
+ enum csm_dp_mem_type type;
+ struct csm_dp_ring ring;
+ struct csm_dp_mem mem;
+ atomic_t ref;
+ atomic_t out_xmit;
+ struct csm_dp_mempool_stats stats;
+ /* Lock for mempools */
+ spinlock_t lock;
+ struct device *dev; /* device for iommu ops */
+};
+
+struct csm_dp_mempool *csm_dp_mempool_alloc(struct csm_dp_dev *pdev,
+ enum csm_dp_mem_type type,
+ unsigned int buf_sz,
+ unsigned int buf_cnt,
+ bool may_dma_map);
+
+void csm_dp_mempool_free(struct csm_dp_mempool *mempool);
+
+int csm_dp_mempool_get_cfg(struct csm_dp_mempool *mempool,
+ struct csm_dp_mempool_cfg *cfg);
+
+int csm_dp_mempool_put_buf(struct csm_dp_mempool *mempool, void *vaddr);
+void *csm_dp_mempool_get_buf(struct csm_dp_mempool *mempool,
+ unsigned int *cluster, unsigned int *c_offset);
+void get_mempool_buf_status(struct csm_dp_mempool *mempool);
+void free_rx_ring_buffers(struct csm_dp_mempool *mempool, bool probe);
+
+static inline bool csm_dp_mempool_hold(struct csm_dp_mempool *mempool)
+{
+ bool ret = false;
+
+ if (!mempool)
+ return ret;
+
+ /* Update mempool ref count before incrementing */
+ smp_mb__before_atomic();
+ if (atomic_inc_not_zero(&mempool->ref))
+ ret = true;
+
+ /* Update mempool ref count after incrementing */
+ smp_mb__after_atomic();
+ return ret;
+}
+
+static inline void __csm_dp_mempool_hold(struct csm_dp_mempool *mempool)
+{
+ atomic_inc(&mempool->ref);
+}
+
+int csm_dp_ring_init(struct csm_dp_ring *ring,
+ unsigned int ringsz,
+ unsigned int mmap_cookie);
+
+void csm_dp_ring_cleanup(struct csm_dp_ring *ring);
+
+int csm_dp_ring_read(struct csm_dp_ring *ring, unsigned long *element_data);
+int csm_dp_ring_write(struct csm_dp_ring *ring, unsigned long element_data);
+
+bool csm_dp_ring_is_empty(struct csm_dp_ring *ring);
+
+int csm_dp_ring_get_cfg(struct csm_dp_ring *ring, struct csm_dp_ring_cfg *cfg);
+
+struct csm_dp_mempool *csm_dp_get_mempool(struct csm_dp_dev *pdev,
+ struct csm_dp_buf_cntrl *buf_cntrl,
+ unsigned int *cluster);
+uint16_t csm_dp_mem_get_cluster(struct csm_dp_mem *mem, unsigned int buf_index);
+
+void csm_dp_mempool_release_no_delay(struct csm_dp_mempool *mempool);
+
+int csm_dp_mempool_dma_map(struct device *dev, /* device for iommu ops */
+ struct csm_dp_mempool *mpool);
+
+static inline bool __csm_dp_ulong_in_range(unsigned long v,
+ unsigned long start,
+ unsigned long end)
+{
+ return (v >= start && v < end);
+}
+
+static inline bool csm_dp_ulong_in_range(unsigned long v,
+ unsigned long start,
+ size_t size)
+{
+ return __csm_dp_ulong_in_range(v, start, start + size - 1);
+}
+
+static inline bool __csm_dp_vaddr_in_range(void *addr, void *start, void *end)
+{
+ return __csm_dp_ulong_in_range((unsigned long)addr,
+ (unsigned long)start,
+ (unsigned long)end);
+}
+
+static inline bool csm_dp_vaddr_in_range(void *addr, void *start, size_t size)
+{
+ return __csm_dp_vaddr_in_range(addr, start, (char *)start + size - 1);
+}
+
+static inline bool __csm_dp_vaddr_in_vma_range(void __user *vaddr_start,
+ void __user *vaddr_end,
+ struct vm_area_struct *vma)
+{
+ unsigned long start = (unsigned long)vaddr_start;
+ unsigned long end = (unsigned long)vaddr_end;
+
+ return (start >= vma->vm_start && end < vma->vm_end);
+}
+
+static inline bool csm_dp_vaddr_in_vma_range(void __user *vaddr, size_t len,
+ struct vm_area_struct *vma)
+{
+ return __csm_dp_vaddr_in_vma_range(vaddr,
+ (void __user *)((char *)vaddr + len - 1),
+ vma);
+}
+
+/* Find offset, this function is used with a memory ring type */
+static inline unsigned long csm_dp_vaddr_offset(void *addr, void *base)
+{
+ return (unsigned long)addr - (unsigned long)base;
+}
+
+/* Find mmap size */
+static inline unsigned long csm_dp_mem_loc_mmap_size(struct csm_dp_mem_loc *loc)
+{
+ return loc->size;
+}
+
+static inline unsigned int csm_dp_calc_ring_size(unsigned int elements)
+{
+ unsigned int size = 1, shift = 0;
+
+ for (shift = 0; (shift < (sizeof(unsigned int) * 8 - 1)); shift++) {
+ if (size >= elements)
+ return size;
+ size <<= 1;
+ }
+ return 0;
+}
+
+/* set buffer state, ptr: pointing to beginging of buffer user data */
+static inline void csm_dp_set_buf_state(void *ptr, enum csm_dp_buf_state state)
+{
+ struct csm_dp_buf_cntrl *pf = (ptr - CSM_DP_L1_CACHE_BYTES);
+
+ pf->state = state;
+}
+
+/* get true buffer size which includes size for user space and control */
+static inline uint32_t csm_dp_buf_true_size(struct csm_dp_mem *mem)
+{
+ return (mem->buf_sz + mem->buf_overhead_sz);
+}
+
+static inline void *csm_dp_mem_rec_addr(struct csm_dp_mem *mem,
+ unsigned int rec)
+{
+ unsigned int cluster;
+ unsigned int offset;
+
+ if (rec >= mem->buf_cnt) {
+ pr_err("record %d exceed %d\n",
+ rec, mem->buf_cnt);
+ return NULL;
+ }
+ cluster = rec / mem->loc.buf_per_cluster;
+ offset = (rec % mem->loc.buf_per_cluster) * csm_dp_buf_true_size(mem);
+ return (void *)mem->loc.cluster_kernel_addr[cluster] + offset;
+}
+
+static inline long csm_dp_mem_rec_offset(struct csm_dp_mem *mem,
+ unsigned int rec)
+{
+ unsigned int cluster;
+ unsigned int offset;
+
+ if (rec >= mem->buf_cnt) {
+ pr_err("record %d exceed %d\n", rec, mem->buf_cnt);
+ return -EINVAL;
+ }
+ cluster = rec / mem->loc.buf_per_cluster;
+ offset = (rec % mem->loc.buf_per_cluster) * csm_dp_buf_true_size(mem);
+ return (long)cluster * CSM_DP_MEMPOOL_CLUSTER_SIZE + offset;
+}
+
+static inline void *csm_dp_mem_offset_addr(struct csm_dp_mem *mem,
+ unsigned long offset,
+ unsigned int *cluster,
+ unsigned int *c_offset)
+{
+ if (offset >= mem->loc.size) {
+ pr_err("offset 0x%lx exceed 0x%lx\n",
+ offset, mem->loc.size);
+ return NULL;
+ }
+ *cluster = offset >> CSM_DP_MEMPOOL_CLUSTER_SHIFT;
+ *c_offset = offset & CSM_DP_MEMPOOL_CLUSTER_MASK;
+ return (void *)(mem->loc.cluster_kernel_addr[*cluster] + *c_offset);
+}
+
+static inline unsigned long csm_dp_get_mem_offset(void *addr,
+ struct csm_dp_mem_loc *loc,
+ unsigned int cl)
+{
+ unsigned long offset;
+
+ offset = (char *)addr - loc->cluster_kernel_addr[cl];
+ offset += (long)cl * CSM_DP_MEMPOOL_CLUSTER_SIZE;
+ return offset;
+}
+
+#endif /* __QCOM_CSM_DP_MEM_H__ */
diff --git a/drivers/char/qcom_csm_dp/qcom_csm_dp_mhi.c b/drivers/char/qcom_csm_dp/qcom_csm_dp_mhi.c
new file mode 100644
index 000000000000..406f3c5e5c66
--- /dev/null
+++ b/drivers/char/qcom_csm_dp/qcom_csm_dp_mhi.c
@@ -0,0 +1,651 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+
+#include "qcom_csm_dp.h"
+#include "qcom_csm_dp_mhi.h"
+
+static struct csm_dp_drv *__pdrv;
+
+static int __mhi_rx_replenish(struct csm_dp_mhi *mhi)
+{
+ struct mhi_device *mhi_dev = mhi->mhi_dev;
+ struct csm_dp_dev *pdev = dev_get_drvdata(&mhi_dev->dev);
+ struct csm_dp_mempool *mempool;
+ int nr = mhi_get_free_desc_count(mhi_dev, DMA_FROM_DEVICE);
+ void *buf;
+ int ret, i, to_xfer;
+ bool is_control = (mhi_dev->id->driver_data == CSM_DP_CH_CONTROL);
+ unsigned int cluster, c_offset;
+ struct csm_dp_buf_cntrl *first_buf_cntrl = NULL, *buf_cntrl = NULL, *prev_buf_cntrl = NULL;
+
+ mempool = is_control ? pdev->mempool[CSM_DP_MEM_TYPE_UL_CONTROL] :
+ pdev->mempool[CSM_DP_MEM_TYPE_UL_DATA];
+
+ ret = 0;
+ if (nr < mhi_get_total_descriptors(mhi_dev, DMA_FROM_DEVICE) / 8)
+ return ret;
+ for (; nr > 0;) {
+ to_xfer = min(CSM_DP_MAX_IOV_SIZE, nr);
+ for (i = 0; i < to_xfer; i++) {
+ buf = csm_dp_mempool_get_buf(mempool, &cluster,
+ &c_offset);
+ if (!buf) {
+ mhi->stats.rx_out_of_buf++;
+ pr_debug("out of rx buffer (nr %d to_xfer %d)!\n",
+ nr, to_xfer);
+ to_xfer = i;
+ ret = -ENOMEM;
+ goto err;
+ }
+ csm_dp_set_buf_state(buf, CSM_DP_BUF_STATE_KERNEL_ALLOC_RECV_DMA);
+ /* link all buffers */
+ buf_cntrl = buf - sizeof(struct csm_dp_buf_cntrl);
+ if (!first_buf_cntrl)
+ first_buf_cntrl = buf_cntrl;
+ else
+ prev_buf_cntrl->next = buf_cntrl;
+ prev_buf_cntrl = buf_cntrl;
+
+ mhi->ul_buf_array[i].buf = buf;
+ mhi->ul_buf_array[i].len = mempool->mem.buf_sz;
+ mhi->ul_flag_array[i] = MHI_EOT | MHI_SG;
+ if (!is_control)
+ mhi->ul_flag_array[i] |= MHI_BEI;
+ if (mempool->mem.loc.dma_mapped)
+ mhi->ul_buf_array[i].dma_addr =
+ mempool->mem.loc.cluster_dma_addr
+ [cluster] + c_offset;
+ else
+ mhi->ul_buf_array[i].dma_addr = 0;
+
+ mhi->ul_buf_array[i].streaming_dma = true;
+ }
+ ret = mhi_queue_n_dma(mhi_dev,
+ DMA_FROM_DEVICE,
+ mhi->ul_buf_array,
+ mhi->ul_flag_array,
+ to_xfer);
+ if (ret)
+ goto err;
+
+ /* update rx head/tail */
+ if (!mhi->rx_tail_buf_cntrl) {
+ /* first repelenish (after probe) */
+ buf_cntrl->next = first_buf_cntrl;
+ mhi->rx_head_buf_cntrl = first_buf_cntrl;
+ } else {
+ mhi->rx_tail_buf_cntrl->next = first_buf_cntrl;
+ buf_cntrl->next = mhi->rx_head_buf_cntrl;
+ }
+ mhi->rx_tail_buf_cntrl = buf_cntrl;
+ first_buf_cntrl = NULL;
+
+ mhi->stats.rx_replenish++;
+ nr -= to_xfer;
+ }
+
+ return ret;
+
+err:
+ for (i = 0; i < to_xfer; i++) {
+ csm_dp_set_buf_state(mhi->ul_buf_array[i].buf,
+ CSM_DP_BUF_STATE_KERNEL_FREE);
+ csm_dp_mempool_put_buf(mempool,
+ mhi->ul_buf_array[i].buf);
+ }
+ mhi->stats.rx_replenish_err++;
+ pr_err("failed to load rx buf for bus:%d VF:%d %s\n",
+ pdev->bus_num, pdev->vf_num, csm_dp_mem_type_to_str(mempool->type));
+ get_mempool_buf_status(mempool);
+ return ret;
+}
+
+static struct csm_dp_mhi *get_dp_mhi(struct mhi_device *mhi_dev)
+{
+ struct csm_dp_dev *pdev = dev_get_drvdata(&mhi_dev->dev);
+
+ switch (mhi_dev->id->driver_data) {
+ case CSM_DP_CH_CONTROL:
+ return &pdev->mhi_control_dev;
+ case CSM_DP_CH_DATA:
+ return &pdev->mhi_data_dev;
+ default:
+ pr_err("invalid mhi_dev->id->driver_data\n");
+ return NULL;
+ }
+}
+
+/* TX complete */
+static void __mhi_ul_xfer_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *result)
+{
+ struct csm_dp_dev *pdev = dev_get_drvdata(&mhi_dev->dev);
+ struct csm_dp_mhi *mhi = get_dp_mhi(mhi_dev);
+ void *addr = result->buf_addr;
+ struct csm_dp_mempool *mempool;
+ struct csm_dp_buf_cntrl *buf_cntrl;
+
+ if ((result->transaction_status == -ENOTCONN) || mhi->mhi_dev_suspended) {
+ pr_debug("(TX Dropped) ch %s bus %d VF %d addr=%p bytes=%lu status=%d mhi->mhi_dev_suspended %d\n",
+ ch_name(mhi_dev->id->driver_data),
+ pdev->bus_num, pdev->vf_num, result->buf_addr,
+ result->bytes_xferd, result->transaction_status,
+ mhi->mhi_dev_suspended);
+
+ } else {
+ pr_debug("(TX complete) ch %s bus %d VF %d addr=%p bytes=%lu status=%d\n",
+ ch_name(mhi_dev->id->driver_data),
+ pdev->bus_num, pdev->vf_num, result->buf_addr,
+ result->bytes_xferd, result->transaction_status);
+
+ mhi->stats.tx_acked++;
+ }
+
+ buf_cntrl = addr - sizeof(struct csm_dp_buf_cntrl);
+ while (buf_cntrl) {
+ mempool = csm_dp_get_mempool(pdev, buf_cntrl, NULL);
+ if (unlikely(!mempool)) {
+ pr_err("cannot find mempool for ch %s bus %d VF %d, addr=0x%p\n",
+ ch_name(mhi_dev->id->driver_data),
+ pdev->bus_num, pdev->vf_num, addr);
+ return;
+ }
+
+ if (mempool->signature != CSM_DP_MEMPOOL_SIG) {
+ pr_err("mempool 0x%p signature 0x%x error, expect 0x%x for ch %s bus %d VF %d\n",
+ mempool, mempool->signature,
+ CSM_DP_MEMPOOL_SIG, ch_name(mhi_dev->id->driver_data),
+ pdev->bus_num, pdev->vf_num);
+ return;
+ }
+
+ if (atomic_read(&mempool->out_xmit) == 0) {
+ pr_err("mempool 0x%p out xmit cnt should not be zero for ch %s bus %d VF %d\n",
+ mempool, ch_name(mhi_dev->id->driver_data),
+ pdev->bus_num, pdev->vf_num);
+ return;
+ }
+
+ atomic_dec(&mempool->out_xmit);
+
+ switch (mempool->type) {
+ case CSM_DP_MEM_TYPE_UL_CONTROL:
+ case CSM_DP_MEM_TYPE_UL_DATA:
+ pr_err("unexpected mempool %d\n", mempool->type);
+ break;
+ default:
+ if (buf_cntrl->state == CSM_DP_BUF_STATE_KERNEL_XMIT_DMA)
+ buf_cntrl->state =
+ CSM_DP_BUF_STATE_KERNEL_XMIT_DMA_COMP;
+ buf_cntrl->xmit_status = CSM_DP_XMIT_OK;
+ /* make it visible to other CPU */
+ wmb();
+ break;
+ }
+
+ buf_cntrl = buf_cntrl->next;
+ addr = buf_cntrl + 1;
+ }
+}
+
+/* RX */
+static void __mhi_dl_xfer_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *result)
+{
+ struct csm_dp_dev *pdev = dev_get_drvdata(&mhi_dev->dev);
+ struct csm_dp_mhi *mhi = get_dp_mhi(mhi_dev);
+ struct csm_dp_mempool *mempool;
+ struct csm_dp_buf_cntrl *packet_start, *packet_end, *prev_buf_cntrl = NULL;
+ bool is_control = (mhi_dev->id->driver_data == CSM_DP_CH_CONTROL);
+ unsigned int buf_count = 0;
+
+ mempool = is_control ? pdev->mempool[CSM_DP_MEM_TYPE_UL_CONTROL] :
+ pdev->mempool[CSM_DP_MEM_TYPE_UL_DATA];
+ if (!mempool) {
+ /*
+ * Getting here with transaction_status == -ENOTCONN is an expected situation while
+ * mhi devices gets removed: mempool got released part of 1st mhi device removal
+ * (e.g. CONTROL) and we now get RX complete for 2nd mhi device (e.g. DATA).
+ */
+ if (result->transaction_status != -ENOTCONN)
+ pr_err_ratelimited("no mempool (ch %s bus %d VF %d status %d)\n",
+ ch_name(mhi_dev->id->driver_data),
+ pdev->bus_num, pdev->vf_num,
+ result->transaction_status);
+ return;
+ }
+
+ if ((result->transaction_status == -ENOTCONN) || mhi->mhi_dev_suspended) {
+ pr_debug("(RX) ch %s bus %d VF %d addr=%p bytes=%lu status=%d mhi->mhi_dev_suspended %d\n",
+ ch_name(mhi_dev->id->driver_data),
+ pdev->bus_num, pdev->vf_num, result->buf_addr,
+ result->bytes_xferd, result->transaction_status,
+ mhi->mhi_dev_suspended);
+ } else {
+ pr_debug("(RX) ch %s bus %d VF %d addr=%p bytes=%lu status=%d\n",
+ ch_name(mhi_dev->id->driver_data),
+ pdev->bus_num, pdev->vf_num, result->buf_addr,
+ result->bytes_xferd, result->transaction_status);
+ }
+
+ if (result->transaction_status == -EOVERFLOW) {
+ pr_debug("overflow event ignored\n");
+ return;
+ }
+
+ while (!mhi->rx_tail_buf_cntrl) {
+ pr_debug("waiting for probe to complete\n");
+ usleep_range(0, 100);
+ }
+
+ packet_start = mhi->rx_head_buf_cntrl;
+ packet_end = result->buf_addr - sizeof(struct csm_dp_buf_cntrl);
+ for (; ((mhi->rx_head_buf_cntrl != mhi->rx_tail_buf_cntrl) ||
+ (mhi->rx_head_buf_cntrl == packet_end));
+ mhi->rx_head_buf_cntrl = mhi->rx_head_buf_cntrl->next) {
+ buf_count++;
+ if (prev_buf_cntrl)
+ prev_buf_cntrl->next_buf_index = mhi->rx_head_buf_cntrl->buf_index;
+ prev_buf_cntrl = mhi->rx_head_buf_cntrl;
+ if (mhi->rx_head_buf_cntrl != packet_end) {
+ mhi->rx_head_buf_cntrl->len = 0; /* 0 indicates this is part of SG */
+ continue;
+ }
+
+ /* reached end of packet */
+ if (mhi->rx_head_buf_cntrl != mhi->rx_tail_buf_cntrl)
+ mhi->rx_head_buf_cntrl = packet_end->next;
+ packet_start->buf_count = buf_count;
+ packet_end->next = NULL;
+ packet_end->next_buf_index = CSM_DP_INVALID_BUF_INDEX;
+ packet_end->len = result->bytes_xferd;
+
+ if (result->transaction_status == -ENOTCONN) {
+ for (; packet_start; packet_start = packet_start->next)
+ csm_dp_mempool_put_buf(mempool, packet_start + 1);
+ return;
+ }
+
+ mhi->stats.rx_cnt++;
+ csm_dp_rx(pdev, packet_start, result->bytes_xferd);
+
+ return;
+ }
+
+ pr_err("couldn't find end of packet for bus:%d VF:%d %s, buf_addr 0x%p bytes:%lu rx_head_buf_cntrl 0x%p rx_tail_buf_cntrl 0x%p buf_count %d\n",
+ pdev->bus_num, pdev->vf_num, csm_dp_mem_type_to_str(mempool->type),
+ result->buf_addr, result->bytes_xferd, mhi->rx_head_buf_cntrl,
+ mhi->rx_tail_buf_cntrl, buf_count);
+ mhi->rx_head_buf_cntrl = packet_start;
+}
+
+/*
+ * Worker function to reset (unprepare and prepare)
+ * MHI channel when channel goes into error state
+ */
+static void csm_dp_mhi_alloc_work(struct work_struct *work)
+{
+ struct mhi_controller *mhi_cntrl;
+ struct pci_dev *pci_dev;
+ struct csm_dp_mhi *mhi;
+ const int sleep_us = 500;
+ int retry = 10;
+ unsigned int bus_num, vf_num;
+ int ret;
+
+ mhi = container_of(work, struct csm_dp_mhi, alloc_work);
+
+ if (!mhi || !mhi->mhi_dev)
+ return;
+
+ mhi_cntrl = mhi->mhi_dev->mhi_cntrl;
+ pci_dev = to_pci_dev(mhi_cntrl->cntrl_dev);
+ bus_num = mhi_cntrl->index;
+ vf_num = PCI_DEVFN(PCI_SLOT(pci_dev->devfn), PCI_FUNC(pci_dev->devfn));
+ pr_info("bus %d VF %d ch %s\n", bus_num, vf_num,
+ ch_name(mhi->mhi_dev->id->driver_data));
+
+ if (!mhi->mhi_dev_suspended) {
+ pr_err("mhi is not suspended\n");
+ return;
+ }
+ mhi->stats.ch_err_cnt++;
+ do {
+ if (atomic_read(&mhi->mhi_dev_refcnt) == 0) {
+ break;
+
+ usleep_range(sleep_us, 2 * sleep_us);
+ retry--;
+ }
+ } while (retry);
+
+ mhi_unprepare_from_transfer(mhi->mhi_dev);
+ pr_info("bus %d VF %d ch %s mhi_unprepare_from_transfer completed\n",
+ bus_num, vf_num, ch_name(mhi->mhi_dev->id->driver_data));
+
+ /*
+ * mhi_prepare_for_transfer is a blocking call that will return
+ * only after the mhi channel connection is restored
+ */
+ ret = mhi_prepare_for_transfer(mhi->mhi_dev);
+ if (ret) {
+ pr_err("mhi_prepare_for_transfer failed\n");
+ return;
+ }
+ pr_info("bus %d VF %d ch %s mhi_prepare_for_transfer completed\n",
+ bus_num, vf_num,
+ ch_name(mhi->mhi_dev->id->driver_data));
+ mhi->rx_head_buf_cntrl = NULL;
+ mhi->rx_tail_buf_cntrl = NULL;
+
+ ret = csm_dp_mhi_rx_replenish(mhi);
+ if (ret) {
+ pr_err("csm_dp_mhi_rx_replenish failed\n");
+ return;
+ }
+
+ mhi->mhi_dev_suspended = false;
+ pr_info("bus %d VF %d ch %s mhi channel reset completed\n",
+ bus_num, vf_num,
+ ch_name(mhi->mhi_dev->id->driver_data));
+}
+
+static void __mhi_status_cb(struct mhi_device *mhi_dev, enum mhi_callback mhi_cb)
+{
+ struct csm_dp_dev *pdev;
+ struct csm_dp_mhi *mhi;
+
+ switch (mhi_cb) {
+ case MHI_CB_PENDING_DATA:
+ pdev = dev_get_drvdata(&mhi_dev->dev);
+ if (napi_schedule_prep(&pdev->napi)) {
+ __napi_schedule(&pdev->napi);
+ pdev->stats.rx_int++;
+ }
+ break;
+ case MHI_CB_CHANNEL_ERROR:
+ mhi = get_dp_mhi(mhi_dev);
+ mhi->mhi_dev_suspended = true;
+ queue_work(mhi->mhi_dev_workqueue, &mhi->alloc_work);
+ break;
+ default:
+ break;
+ }
+}
+
+int csm_dp_mhi_rx_replenish(struct csm_dp_mhi *mhi)
+{
+ int ret;
+
+ spin_lock_bh(&mhi->rx_lock);
+
+ if (mhi->mhi_dev_destroyed) {
+ ret = -ENODEV;
+ pr_err_ratelimited("Replenish error:%d Device destroyed\n", ret);
+ } else {
+ ret = __mhi_rx_replenish(mhi);
+ }
+
+ spin_unlock_bh(&mhi->rx_lock);
+ return ret;
+}
+
+void csm_dp_mhi_tx_poll(struct csm_dp_mhi *mhi)
+{
+ int n;
+
+ do {
+ n = mhi_poll(mhi->mhi_dev, CSM_DP_NAPI_WEIGHT, DMA_TO_DEVICE);
+ if (n < 0)
+ pr_err_ratelimited("Error Tx polling n:%d\n", n);
+ } while (n == CSM_DP_NAPI_WEIGHT);
+}
+
+void csm_dp_mhi_rx_poll(struct csm_dp_mhi *mhi)
+{
+ int n;
+
+ do {
+ n = mhi_poll(mhi->mhi_dev, CSM_DP_NAPI_WEIGHT, DMA_FROM_DEVICE);
+ if (n < 0)
+ pr_err_ratelimited("Error Rx polling n:%d\n", n);
+
+ pr_info("Number of Rx poll %d\n", n);
+ } while (n == CSM_DP_NAPI_WEIGHT);
+}
+
+static void csm_dp_mhi_packet_stats_reset(struct csm_dp_mhi *mhi,
+ struct csm_dp_dev *pdev,
+ enum csm_dp_channel ch)
+{
+ struct csm_dp_core_stats *stats = &pdev->stats;
+ bool is_control = (ch == CSM_DP_CH_CONTROL);
+
+ if (mhi) {
+ mhi->stats.tx_cnt = 0;
+ mhi->stats.tx_acked = 0;
+ mhi->stats.tx_err = 0;
+ mhi->stats.rx_cnt = 0;
+ mhi->stats.rx_out_of_buf = 0;
+ mhi->stats.rx_replenish = 0;
+ mhi->stats.rx_replenish_err = 0;
+ mhi->stats.ch_err_cnt = 0;
+ if (stats && is_control)
+ stats->rx_drop = 0;
+ }
+}
+
+static int csm_dp_mhi_probe(struct mhi_device *mhi_dev,
+ const struct mhi_device_id *id)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct pci_dev *pci_dev = to_pci_dev(mhi_cntrl->cntrl_dev);
+ struct csm_dp_dev *pdev;
+ int ret;
+ struct csm_dp_mhi *mhi;
+ struct csm_dp_mempool *mempool;
+ unsigned int bus_num, vf_num;
+
+ bus_num = mhi_cntrl->index;
+ vf_num = PCI_DEVFN(PCI_SLOT(pci_dev->devfn), PCI_FUNC(pci_dev->devfn));
+ pr_info("probing bus:%d VF:%d mhi chan %s dp chan %s\n",
+ bus_num, vf_num, id->chan, ch_name(id->driver_data));
+
+ if (!__pdrv)
+ return -ENODEV;
+
+ if (vf_num < 0) {
+ /* SR-IOV disabled, create single device node */
+ pdev = &__pdrv->dp_devs[bus_num * CSM_DP_MAX_NUM_VFS];
+ } else if (vf_num == 0) {
+ /* SR-IOV enabled, PF device: ignore */
+ return 0;
+ } else if (bus_num >= CSM_DP_MAX_NUM_BUSES || vf_num > CSM_DP_MAX_NUM_VFS) {
+ /* invalid ids */
+ pr_err("invalid ids bus_num %d vf_num %d\n", bus_num, vf_num);
+ return -EINVAL;
+ }
+
+ /* SR-IOV enabled, VF device. bus_num is 0..11, vf_num is 1..4 */
+ if (vf_num > 0)
+ pdev = &__pdrv->dp_devs[vf_num];
+
+ if (!pdev->cdev_inited) {
+ pdev->bus_num = bus_num;
+ pdev->vf_num = vf_num;
+ ret = csm_dp_cdev_add(pdev, &mhi_dev->dev);
+ if (ret)
+ return ret;
+ }
+
+ switch (id->driver_data) {
+ case CSM_DP_CH_CONTROL:
+ mhi = &pdev->mhi_control_dev;
+ mempool = pdev->mempool[CSM_DP_MEM_TYPE_UL_CONTROL];
+ break;
+ case CSM_DP_CH_DATA:
+ mhi = &pdev->mhi_data_dev;
+ mempool = pdev->mempool[CSM_DP_MEM_TYPE_UL_DATA];
+ break;
+ default:
+ pr_err("unexpected driver_data %ld for bus:%d VF:%d\n",
+ id->driver_data, bus_num, vf_num);
+ ret = -EINVAL;
+ goto err;
+ }
+
+ free_rx_ring_buffers(mempool, true);
+ csm_dp_mhi_packet_stats_reset(mhi, pdev, id->driver_data);
+
+ dev_set_drvdata(&mhi_dev->dev, pdev);
+
+ ret = mhi_prepare_for_transfer(mhi_dev);
+ if (ret) {
+ pr_err("mhi_prepare_for_transfer failed for bus:%d VF:%d mhi chan %s dp chan %s\n",
+ bus_num, vf_num, id->chan, ch_name(id->driver_data));
+ goto err;
+ }
+
+ /* Creating workqueue */
+ mhi->mhi_dev_workqueue = alloc_workqueue("csm_dp_mhi_workqueue",
+ WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
+ if (!mhi->mhi_dev_workqueue) {
+ pr_err("Failed to allocate workqueue for bus:%d VF:%d mhi chan %s dp chan %s\n",
+ bus_num, vf_num, id->chan, ch_name(id->driver_data));
+ goto err;
+ }
+
+ INIT_WORK(&mhi->alloc_work, csm_dp_mhi_alloc_work);
+
+ mhi->mhi_dev = mhi_dev;
+ mhi->mhi_dev_suspended = false;
+ atomic_set(&mhi->mhi_dev_refcnt, 0);
+ spin_lock_init(&mhi->rx_lock);
+ mutex_init(&mhi->tx_mutex);
+ mhi->rx_head_buf_cntrl = NULL;
+ mhi->rx_tail_buf_cntrl = NULL;
+
+ mhi->mhi_dev_destroyed = false;
+ pr_debug("csm_dp_mhi_rx_replenish\n");
+ if (mempool) {
+ ret = csm_dp_mempool_dma_map(mhi_dev->mhi_cntrl->cntrl_dev, mempool);
+ if (ret) {
+ pr_err("dma_map failed for bus:%d VF:%d mhi chan %s dp chan %s, mempool type %d ret %d\n",
+ bus_num, vf_num, id->chan,
+ ch_name(id->driver_data), mempool->type, ret);
+ goto err;
+ }
+
+ ret = csm_dp_mhi_rx_replenish(mhi);
+ if (ret) {
+ pr_err("csm_dp_mhi_rx_replenish failed for bus:%d VF:%d mhi chan %s dp chan %s\n",
+ bus_num, vf_num, id->chan, ch_name(id->driver_data));
+ goto err;
+ }
+ }
+
+ pr_info("successful for bus:%d VF:%d mhi chan %s dp chan %s\n",
+ bus_num, vf_num, id->chan, ch_name(id->driver_data));
+
+ return 0;
+
+err:
+ mhi->mhi_dev_destroyed = true;
+ csm_dp_cdev_del(pdev);
+ return ret;
+}
+
+static void csm_dp_mhi_remove(struct mhi_device *mhi_dev)
+{
+ struct csm_dp_dev *pdev = dev_get_drvdata(&mhi_dev->dev);
+ struct csm_dp_mhi *mhi;
+
+ pr_info("mhi chan %s dp chan %s bus %d VF %d\n",
+ mhi_dev->id->chan,
+ ch_name(mhi_dev->id->driver_data),
+ pdev->bus_num, pdev->vf_num);
+
+ switch (mhi_dev->id->driver_data) {
+ case CSM_DP_CH_CONTROL:
+ mhi = &pdev->mhi_control_dev;
+ break;
+ case CSM_DP_CH_DATA:
+ mhi = &pdev->mhi_data_dev;
+ break;
+ default:
+ pr_err("unexpected driver_data %ld\n",
+ mhi_dev->id->driver_data);
+ return;
+ }
+
+ flush_work(&mhi->alloc_work);
+ destroy_workqueue(mhi->mhi_dev_workqueue);
+ mhi_unprepare_from_transfer(mhi_dev);
+
+ spin_lock_bh(&mhi->rx_lock);
+ mhi->mhi_dev_destroyed = true;
+ spin_unlock_bh(&mhi->rx_lock);
+
+ /* wait for idle mhi_dev */
+ while (atomic_read(&mhi->mhi_dev_refcnt) > 0) {
+ pr_debug("mhi_dev_refcnt %d\n",
+ atomic_read(&mhi->mhi_dev_refcnt));
+ usleep_range(0, 10 * 1000);
+ }
+ mhi->mhi_dev = NULL;
+}
+
+static struct mhi_device_id csm_dp_mhi_match_table[] = {
+ { .chan = "IP_HW1", .driver_data = CSM_DP_CH_CONTROL },
+ { .chan = "IP_HW2", .driver_data = CSM_DP_CH_DATA },
+ {},
+};
+
+static struct mhi_driver __csm_dp_mhi_drv = {
+ .id_table = csm_dp_mhi_match_table,
+ .remove = csm_dp_mhi_remove,
+ .probe = csm_dp_mhi_probe,
+ .ul_xfer_cb = __mhi_ul_xfer_cb,
+ .dl_xfer_cb = __mhi_dl_xfer_cb,
+ .status_cb = __mhi_status_cb,
+ .driver = {
+ .name = CSM_DP_MHI_NAME,
+ .owner = THIS_MODULE,
+ },
+};
+
+int csm_dp_mhi_init(struct csm_dp_drv *pdrv)
+{
+ int ret = -EBUSY;
+
+ if (!__pdrv) {
+ __pdrv = pdrv;
+ ret = mhi_driver_register(&__csm_dp_mhi_drv);
+ if (ret) {
+ __pdrv = NULL;
+ pr_err("CSM-DP: mhi registration failed!\n");
+ return ret;
+ }
+
+ pr_info("CSM-DP: Register MHI driver!\n");
+ }
+ return ret;
+}
+
+void csm_dp_mhi_cleanup(struct csm_dp_drv *pdrv)
+{
+ if (__pdrv) {
+ mhi_driver_unregister(&__csm_dp_mhi_drv);
+ __pdrv = NULL;
+ pr_info("CSM-DP: Unregister MHI driver\n");
+ }
+}
diff --git a/drivers/char/qcom_csm_dp/qcom_csm_dp_mhi.h b/drivers/char/qcom_csm_dp/qcom_csm_dp_mhi.h
new file mode 100644
index 000000000000..ad014956e6ec
--- /dev/null
+++ b/drivers/char/qcom_csm_dp/qcom_csm_dp_mhi.h
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __QCOM_CSM_DP_MHI_H__
+#define __QCOM_CSM_DP_MHI_H__
+
+#include <linux/dma-mapping.h>
+#include <linux/mhi.h>
+#include <linux/slab.h>
+
+#define CSM_DP_MHI_NAME "csm-dp-mhi"
+
+struct csm_dp_drv;
+
+struct csm_dp_mhi_stats {
+ unsigned long tx_cnt;
+ unsigned long tx_acked;
+ unsigned long tx_err;
+ unsigned long rx_cnt;
+ unsigned long rx_out_of_buf;
+
+ unsigned long rx_replenish;
+ unsigned long rx_replenish_err;
+ unsigned long ch_err_cnt;
+};
+
+/* Represents MHI channel pair - Tx and Rx */
+struct csm_dp_mhi {
+ struct mhi_device *mhi_dev;
+ bool mhi_dev_destroyed;
+ bool mhi_dev_suspended;
+ atomic_t mhi_dev_refcnt;
+ struct csm_dp_mhi_stats stats;
+ /* rx_lock for control and data channels */
+ spinlock_t rx_lock;
+ /* Mutex lock for TX's */
+ struct mutex tx_mutex;
+ struct workqueue_struct *mhi_dev_workqueue;
+ struct work_struct alloc_work;
+ /*
+ * The following are for needed storage
+ * for mhi_queue_n_transfer.
+ */
+ enum mhi_flags ul_flag_array[CSM_DP_MAX_IOV_SIZE];
+ enum mhi_flags dl_flag_array[CSM_DP_MAX_IOV_SIZE];
+ struct mhi_buf dl_buf_array[CSM_DP_MAX_IOV_SIZE];
+ struct mhi_buf ul_buf_array[CSM_DP_MAX_IOV_SIZE];
+
+ struct csm_dp_buf_cntrl *rx_head_buf_cntrl, *rx_tail_buf_cntrl;
+};
+
+int csm_dp_mhi_init(struct csm_dp_drv *pdrv);
+void csm_dp_mhi_cleanup(struct csm_dp_drv *pdrv);
+
+int csm_dp_mhi_rx_replenish(struct csm_dp_mhi *mhi);
+
+static inline int csm_dp_mhi_n_tx(struct csm_dp_mhi *mhi,
+ unsigned int num)
+{
+ int ret;
+
+ ret = mhi_queue_n_dma(mhi->mhi_dev, DMA_TO_DEVICE, mhi->dl_buf_array,
+ mhi->dl_flag_array, num);
+ if (!ret)
+ mhi->stats.tx_cnt += num;
+ else
+ mhi->stats.tx_err += num;
+ return ret;
+}
+
+static inline bool csm_dp_mhi_is_ready(struct csm_dp_mhi *mhi)
+{
+ return mhi->mhi_dev && !mhi->mhi_dev_destroyed && !mhi->mhi_dev_suspended;
+}
+
+void csm_dp_mhi_tx_poll(struct csm_dp_mhi *mhi);
+void csm_dp_mhi_rx_poll(struct csm_dp_mhi *mhi);
+
+#endif /* __QCOM_CSM_DP_MHI_H__ */
diff --git a/include/uapi/linux/qcom_csm_dp_ioctl.h b/include/uapi/linux/qcom_csm_dp_ioctl.h
new file mode 100644
index 000000000000..af04b9b13d34
--- /dev/null
+++ b/include/uapi/linux/qcom_csm_dp_ioctl.h
@@ -0,0 +1,306 @@
+/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#ifndef __QCOM_CSM_DP_IOCTL_H__
+#define __QCOM_CSM_DP_IOCTL_H__
+
+#include <linux/types.h>
+#ifdef __KERNEL__
+#include <linux/uio.h>
+#else
+#include <sys/uio.h>
+#endif
+
+#define CSM_DP_MAX_IOV_SIZE 128
+#define CSM_DP_MAX_SG_IOV_SIZE 128
+
+#define CSM_DP_IOCTL_BASE 0xDA
+
+#define CSM_DP_IOCTL_MEMPOOL_ALLOC \
+ _IOWR(CSM_DP_IOCTL_BASE, 1, struct csm_dp_ioctl_mempool_alloc)
+
+#define CSM_DP_IOCTL_MEMPOOL_GET_CONFIG \
+ _IOWR(CSM_DP_IOCTL_BASE, 2, struct csm_dp_ioctl_getcfg)
+
+#define CSM_DP_IOCTL_RX_GET_CONFIG \
+ _IOWR(CSM_DP_IOCTL_BASE, 3, struct csm_dp_ioctl_getcfg)
+
+#define CSM_DP_IOCTL_TX \
+ _IOWR(CSM_DP_IOCTL_BASE, 4, struct csm_dp_ioctl_tx)
+
+#define CSM_DP_IOCTL_SG_TX \
+ _IOWR(CSM_DP_IOCTL_BASE, 5, struct csm_dp_ioctl_tx)
+
+/* obsolete */
+#define CSM_DP_IOCTL_TX_MODE_CONFIG \
+ _IOWR(CSM_DP_IOCTL_BASE, 6, unsigned int)
+
+#define CSM_DP_IOCTL_RX_POLL \
+ _IOWR(CSM_DP_IOCTL_BASE, 7, struct iovec)
+
+#define CSM_DP_IOCTL_GET_STATS \
+ _IOWR(CSM_DP_IOCTL_BASE, 8, struct csm_dp_ioctl_getstats)
+
+enum csm_dp_mem_type {
+ CSM_DP_MEM_TYPE_DL_CONTROL,
+ CSM_DP_MEM_TYPE_DL_DATA,
+ CSM_DP_MEM_TYPE_UL_CONTROL,
+ CSM_DP_MEM_TYPE_UL_DATA,
+ CSM_DP_MEM_TYPE_LAST,
+};
+
+enum csm_dp_mmap_type {
+ CSM_DP_MMAP_TYPE_MEM,
+ CSM_DP_MMAP_TYPE_RING,
+ CSM_DP_MMAP_TYPE_LAST,
+};
+
+enum csm_dp_rx_type {
+ CSM_DP_RX_TYPE_FAPI,
+ CSM_DP_RX_TYPE_LAST,
+};
+
+#define CSM_DP_BUFFER_FENCE_SIG 0xDEADFACE
+#define CSM_DP_BUFFER_SIG 0xDAC0FFEE
+
+/*
+ * A buffer control is an area with size of L1_CACHE_BYTES (64 bytes for arm64).
+ * It is placed at the beginning of a buffer.
+ * csm_dp_buf_cntrl is placed at the control area. The last 4 bytes of the area is a fence defined
+ * as CSM_DP_BUFFER_FENCE_SIG.
+ * The size of csm_dp_buf_cntrl should be less than L1_CACHE_BYTES.
+ * User data is placed after the control area of L1_CACHE_BYTES size.
+ */
+#define CSM_DP_L1_CACHE_BYTES 64 /*
+ * CSM_DP_L1_CACHE_BYTES is the same as
+ * L1_CACHE_BYTES.
+ * csm_dp_ioctl.h is included in
+ * the applications,
+ * The symbol L1_CACHE_BYTES is defined in the
+ * kernel, not be used here. Therefore,
+ * it is redefined.
+ */
+/*
+ * xmit_status definition
+ * If xmit errors, defined as -(error code)
+ */
+#define CSM_DP_XMIT_IN_PROGRESS (1)
+#define CSM_DP_XMIT_OK 0
+
+/*
+ * maximum mtu size for CSM DP application, including csm_dp header
+ * Note, need to make sure both sides in sync between Host and Q6
+ */
+#define CSM_DP_MAX_DL_MSG_LEN ((2 * 1024 * 1024) - CSM_DP_L1_CACHE_BYTES)
+#define CSM_DP_MAX_UL_MSG_LEN CSM_DP_MAX_DL_MSG_LEN
+
+#define CSM_DP_DEFAULT_UL_BUF_SIZE (512 * 1024)
+#define CSM_DP_DEFAULT_UL_BUF_CNT 2500
+
+#define CSM_DP_INVALID_BUF_INDEX ((uint32_t)-1)
+
+struct csm_dp_buf_cntrl {
+ uint32_t signature;
+ uint32_t state;
+ int32_t xmit_status;
+ uint16_t mem_type; /* enum csm_dp_mem_type */
+ uint32_t buf_index;
+ struct csm_dp_buf_cntrl *next; /* used by kernel only */
+ uint32_t next_buf_index; /* used in Rx, kernel writes, user reads */
+ uint32_t len; /* used in Rx, kernel writes, user reads */
+ struct csm_dp_buf_cntrl *next_packet; /* used in Rx only */
+ uint16_t buf_count; /* used in Rx only */
+ unsigned char spare[CSM_DP_L1_CACHE_BYTES
+ - sizeof(uint32_t) /* signature */
+ - sizeof(uint32_t) /* state */
+ - sizeof(int32_t) /* xmit_status */
+ - sizeof(uint16_t) /* mem_type */
+ - sizeof(uint32_t) /* buf_index */
+ - sizeof(struct csm_dp_buf_cntrl *) /* next */
+ - sizeof(uint32_t) /* next_buf_index */
+ - sizeof(uint32_t) /* len */
+ - sizeof(struct csm_dp_buf_cntrl *) /* next_packet */
+ - sizeof(uint16_t) /* buf_count */
+ - sizeof(uint32_t)];/* fence */
+ uint32_t fence;
+} __attribute__((packed));
+
+enum csm_dp_buf_state {
+ CSM_DP_BUF_STATE_KERNEL_FREE,
+ CSM_DP_BUF_STATE_KERNEL_ALLOC_RECV_DMA,
+ CSM_DP_BUF_STATE_KERNEL_RECVCMP_MSGQ_TO_APP,
+ CSM_DP_BUF_STATE_KERNEL_XMIT_DMA,
+ CSM_DP_BUF_STATE_KERNEL_XMIT_DMA_COMP,
+ CSM_DP_BUF_STATE_USER_FREE,
+ CSM_DP_BUF_STATE_USER_ALLOC,
+ CSM_DP_BUF_STATE_USER_RECV,
+ CSM_DP_BUF_STATE_LAST,
+};
+
+enum csm_dp_channel {
+ CSM_DP_CH_CONTROL,
+ CSM_DP_CH_DATA,
+};
+
+struct csm_dp_ring_element {
+ uint64_t element_ctrl; /* 1 entry not valid, 0 valid */
+ /* Other bits for control flags: tbd */
+
+ unsigned long element_data;
+ /*
+ * If the ring is used for
+ * csm dp buffer management,
+ * ring data is pointing to
+ * user data
+ */
+};
+
+struct csm_dp_mmap_cfg {
+ __u64 length; /* length parameter for mmap */
+ __u32 cookie; /* last parameter for mmap */
+};
+
+struct csm_dp_ring_cfg {
+ struct csm_dp_mmap_cfg mmap; /* mmap parameters */
+ __u32 size; /* ring size */
+ __u32 prod_head_off; /* page offset of prod_head */
+ __u32 prod_tail_off; /* page offset of prod_tail */
+ __u32 cons_head_off; /* page offset of cons_head */
+ __u32 cons_tail_off; /* page offset of cons_tail */
+ __u32 ringbuf_off; /* page offset of ring buffer */
+};
+
+struct csm_dp_mem_cfg {
+ struct csm_dp_mmap_cfg mmap; /* mmap parameters */
+ __u32 buf_sz; /* size of buffer for user data */
+ __u32 buf_cnt; /* number of buffer */
+ __u32 buf_overhead_sz; /*
+ * size of buffer overhead,
+ * on top of buf_sz.
+ */
+ __u32 cluster_size; /* cluster size in bytes.
+ * number of buffers in a cluster:
+ * cluster_size /(buf_overhead_sz +
+ * buf_sz)
+ * A buffer starts at beginning of
+ * a cluster. Spared space with
+ * size less than (buf_overhead_sz
+ * + buf_sz) at end of
+ * a cluster is not used.
+ */
+ __u32 num_cluster; /* number of cluster */
+ __u32 buf_per_cluster; /* number of buffers per cluster */
+};
+
+struct csm_dp_mempool_cfg {
+ enum csm_dp_mem_type type;
+ struct csm_dp_mem_cfg mem;
+ struct csm_dp_ring_cfg ring;
+};
+
+struct csm_dp_ioctl_mempool_alloc {
+ __u32 type; /* type defined in enum csm_dp_mem_type */
+ __u32 buf_sz; /* size of buffer */
+ __u32 buf_num; /* number of buffer */
+ struct csm_dp_mempool_cfg *cfg; /* for kernel to return config info */
+};
+
+struct csm_dp_ioctl_getcfg {
+ __u32 type;
+ void *cfg;
+};
+
+struct csm_dp_ioctl_tx {
+ enum csm_dp_channel ch;
+ struct iovec iov;
+ __u32 flags; /* CSM_DP_IOCTL_TX_FLAG_xxx */
+};
+
+struct csm_dp_ioctl_getstats {
+ enum csm_dp_channel ch; /* IN param, set by caller */
+ __u64 tx_cnt;
+ __u64 tx_acked;
+ __u64 rx_cnt;
+ __u64 reserved[10]; /* for future use */
+};
+
+static inline int csm_dp_mem_type_is_valid(enum csm_dp_mem_type type)
+{
+ return (type >= 0 && type < CSM_DP_MEM_TYPE_LAST);
+}
+
+static inline const char *csm_dp_mem_type_to_str(enum csm_dp_mem_type type)
+{
+ switch (type) {
+ case CSM_DP_MEM_TYPE_DL_CONTROL: return "DL_CTRL";
+ case CSM_DP_MEM_TYPE_DL_DATA: return "DL_DATA";
+ case CSM_DP_MEM_TYPE_UL_CONTROL: return "UL_CTRL";
+ case CSM_DP_MEM_TYPE_UL_DATA: return "UL_DATA";
+ default: return "unknown";
+ }
+}
+
+static inline int csm_dp_mmap_type_is_valid(enum csm_dp_mmap_type type)
+{
+ return (type >= 0 && type < CSM_DP_MMAP_TYPE_LAST);
+}
+
+static inline const char *csm_dp_mmap_type_to_str(enum csm_dp_mmap_type type)
+{
+ switch (type) {
+ case CSM_DP_MMAP_TYPE_MEM: return "Memory";
+ case CSM_DP_MMAP_TYPE_RING: return "Ring";
+ default: return "unknown";
+ }
+}
+
+static inline int csm_dp_rx_type_is_valid(enum csm_dp_rx_type type)
+{
+ return (type >= 0 && type < CSM_DP_RX_TYPE_LAST);
+}
+
+static inline const char *csm_dp_rx_type_to_str(enum csm_dp_rx_type type)
+{
+ switch (type) {
+ case CSM_DP_RX_TYPE_FAPI: return "FAPI";
+ default: return "unknown";
+ }
+}
+
+static inline const char *csm_dp_buf_state_to_str(enum csm_dp_buf_state state)
+{
+ switch (state) {
+ case CSM_DP_BUF_STATE_KERNEL_FREE:
+ return "KERNEL FREE";
+ case CSM_DP_BUF_STATE_KERNEL_ALLOC_RECV_DMA:
+ return "KERNEL ALLOC RECV DMA";
+ case CSM_DP_BUF_STATE_KERNEL_RECVCMP_MSGQ_TO_APP:
+ return "KERNEL RECV CMP MSGQ TO APP";
+ case CSM_DP_BUF_STATE_KERNEL_XMIT_DMA:
+ return "KERNEL XMIT DMA";
+ case CSM_DP_BUF_STATE_KERNEL_XMIT_DMA_COMP:
+ return "KERNEL XMIT DMA COMP";
+ case CSM_DP_BUF_STATE_USER_FREE:
+ return "USER FREE";
+ case CSM_DP_BUF_STATE_USER_ALLOC:
+ return "USER ALLOC";
+ case CSM_DP_BUF_STATE_USER_RECV:
+ return "USER RECV";
+ case CSM_DP_BUF_STATE_LAST:
+ default:
+ return "unknown";
+ };
+}
+
+static inline bool csm_dp_mem_type_is_ul(enum csm_dp_mem_type type)
+{
+ return type == CSM_DP_MEM_TYPE_UL_CONTROL || type == CSM_DP_MEM_TYPE_UL_DATA;
+}
+
+static inline bool csm_dp_mem_type_is_dl(enum csm_dp_mem_type type)
+{
+ return type == CSM_DP_MEM_TYPE_DL_CONTROL || type == CSM_DP_MEM_TYPE_DL_DATA;
+}
+
+#endif /* __QCOM_CSM_DP_IOCTL_H__ */
--
2.34.1
Powered by blists - more mailing lists