| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20210316124237.3469-1-jarvis.w.jiang@gmail.com>
Date: Tue, 16 Mar 2021 05:42:37 -0700
From: Jarvis Jiang <jarvis.w.jiang@...il.com>
To: davem@...emloft.net, rppt@...ux.ibm.com, akpm@...ux-foundation.org
Cc: linux-kernel@...r.kernel.org, netdev@...r.kernel.org,
linux-mm@...ck.org, cchen50@...ovo.com, mpearson@...ovo.com,
Jarvis Jiang <jarvis.w.jiang@...il.com>
Subject: [PATCH] Add MHI bus support and driver for T99W175 5G modem
T99W175 using MBIM or RmNet over PCIe interface with
MHI protocol support.
Ported from IPQ8072 platform, including MHI, MBIM, RmNet
Supporting below PCI devices:
PCI_DEVICE(0x17cb, 0x0300)
PCI_DEVICE(0x17cb, 0x0301)
PCI_DEVICE(0x17cb, 0x0302)
PCI_DEVICE(0x17cb, 0x0303)
PCI_DEVICE(0x17cb, 0x0304)
PCI_DEVICE(0x17cb, 0x0305)
PCI_DEVICE(0x17cb, 0x0306)
PCI_DEVICE(0x105b, 0xe0ab)
PCI_DEVICE(0x105b, 0xe0b0)
PCI_DEVICE(0x105b, 0xe0b1)
PCI_DEVICE(0x105b, 0xe0b3)
PCI_DEVICE(0x1269, 0x00b3)
PCI_DEVICE(0x03f0, 0x0a6c)
Signed-off-by: Jarvis Jiang <jarvis.w.jiang@...il.com>
---
MAINTAINERS | 16 +
drivers/bus/Kconfig | 1 +
drivers/bus/Makefile | 3 +
drivers/bus/mhi/Kconfig | 27 +
drivers/bus/mhi/Makefile | 9 +
drivers/bus/mhi/controllers/Kconfig | 13 +
drivers/bus/mhi/controllers/Makefile | 2 +
drivers/bus/mhi/controllers/mhi_arch_qti.c | 275 ++
drivers/bus/mhi/controllers/mhi_qti.c | 970 +++++++
drivers/bus/mhi/controllers/mhi_qti.h | 44 +
drivers/bus/mhi/core/Makefile | 2 +
drivers/bus/mhi/core/mhi_boot.c | 590 +++++
drivers/bus/mhi/core/mhi_dtr.c | 223 ++
drivers/bus/mhi/core/mhi_init.c | 1901 ++++++++++++++
drivers/bus/mhi/core/mhi_internal.h | 826 ++++++
drivers/bus/mhi/core/mhi_main.c | 2261 +++++++++++++++++
drivers/bus/mhi/core/mhi_pm.c | 1158 +++++++++
drivers/bus/mhi/devices/Kconfig | 43 +
drivers/bus/mhi/devices/Makefile | 3 +
drivers/bus/mhi/devices/mhi_netdev.c | 1830 +++++++++++++
drivers/bus/mhi/devices/mhi_satellite.c | 1155 +++++++++
drivers/bus/mhi/devices/mhi_uci.c | 802 ++++++
drivers/net/ethernet/qualcomm/rmnet/Makefile | 2 +-
.../ethernet/qualcomm/rmnet/rmnet_config.c | 131 +-
.../ethernet/qualcomm/rmnet/rmnet_config.h | 110 +-
.../qualcomm/rmnet/rmnet_descriptor.c | 1225 +++++++++
.../qualcomm/rmnet/rmnet_descriptor.h | 152 ++
.../ethernet/qualcomm/rmnet/rmnet_handlers.c | 321 ++-
.../ethernet/qualcomm/rmnet/rmnet_handlers.h | 27 +-
.../net/ethernet/qualcomm/rmnet/rmnet_map.h | 238 +-
.../qualcomm/rmnet/rmnet_map_command.c | 304 ++-
.../ethernet/qualcomm/rmnet/rmnet_map_data.c | 1029 +++++++-
.../ethernet/qualcomm/rmnet/rmnet_private.h | 19 +-
.../net/ethernet/qualcomm/rmnet/rmnet_trace.h | 250 ++
.../net/ethernet/qualcomm/rmnet/rmnet_vnd.c | 101 +-
.../net/ethernet/qualcomm/rmnet/rmnet_vnd.h | 16 +-
include/linux/ipc_logging.h | 291 +++
include/linux/mhi.h | 743 ++++++
include/linux/mod_devicetable.h | 22 +-
include/linux/msm-bus.h | 214 ++
include/linux/msm_pcie.h | 173 ++
include/linux/netdevice.h | 18 +-
include/uapi/linux/if_link.h | 4 +
include/uapi/linux/msm_rmnet.h | 170 ++
mm/memblock.c | 2 +
net/core/dev.c | 192 +-
46 files changed, 17700 insertions(+), 208 deletions(-)
create mode 100644 drivers/bus/mhi/Kconfig
create mode 100644 drivers/bus/mhi/Makefile
create mode 100644 drivers/bus/mhi/controllers/Kconfig
create mode 100644 drivers/bus/mhi/controllers/Makefile
create mode 100644 drivers/bus/mhi/controllers/mhi_arch_qti.c
create mode 100644 drivers/bus/mhi/controllers/mhi_qti.c
create mode 100644 drivers/bus/mhi/controllers/mhi_qti.h
create mode 100644 drivers/bus/mhi/core/Makefile
create mode 100644 drivers/bus/mhi/core/mhi_boot.c
create mode 100644 drivers/bus/mhi/core/mhi_dtr.c
create mode 100644 drivers/bus/mhi/core/mhi_init.c
create mode 100644 drivers/bus/mhi/core/mhi_internal.h
create mode 100644 drivers/bus/mhi/core/mhi_main.c
create mode 100644 drivers/bus/mhi/core/mhi_pm.c
create mode 100644 drivers/bus/mhi/devices/Kconfig
create mode 100644 drivers/bus/mhi/devices/Makefile
create mode 100644 drivers/bus/mhi/devices/mhi_netdev.c
create mode 100644 drivers/bus/mhi/devices/mhi_satellite.c
create mode 100644 drivers/bus/mhi/devices/mhi_uci.c
create mode 100644 drivers/net/ethernet/qualcomm/rmnet/rmnet_descriptor.c
create mode 100644 drivers/net/ethernet/qualcomm/rmnet/rmnet_descriptor.h
create mode 100644 drivers/net/ethernet/qualcomm/rmnet/rmnet_trace.h
create mode 100644 include/linux/ipc_logging.h
create mode 100644 include/linux/mhi.h
create mode 100644 include/linux/msm-bus.h
create mode 100644 include/linux/msm_pcie.h
create mode 100644 include/uapi/linux/msm_rmnet.h
diff --git a/MAINTAINERS b/MAINTAINERS
index 9d3a5c54a41d..79f61526a4c0 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -6413,6 +6413,22 @@ S: Odd Fixes
L: linux-block@...r.kernel.org
F: drivers/block/floppy.c
+FOXCONN 5G MODEM DRIVER
+M: Jarvis Jiang <jarvis.w.jiang@...il.com>
+S: Orphan
+F: drivers/bus/mhi/
+F: drivers/net/ethernet/qualcomm/rmnet/
+F: include/linux/ipc_logging.h
+F: include/linux/mhi.h
+F: include/linux/mod_devicetable.h
+F: include/linux/msm-bus.h
+F: include/linux/msm_pcie.h
+F: include/linux/netdevice.h
+F: include/uapi/linux/if_link.h
+F: include/uapi/linux/msm_rmnet.h
+F: mm/memblock.c
+F: net/core/dev.c
+
FPGA MANAGER FRAMEWORK
M: Moritz Fischer <mdf@...nel.org>
L: linux-fpga@...r.kernel.org
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 6b331061d34b..53a5ae6fd284 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -192,5 +192,6 @@ config DA8XX_MSTPRI
peripherals.
source "drivers/bus/fsl-mc/Kconfig"
+source "drivers/bus/mhi/Kconfig"
endmenu
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index 16b43d3468c6..b5b3e219ba8a 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -33,3 +33,6 @@ obj-$(CONFIG_UNIPHIER_SYSTEM_BUS) += uniphier-system-bus.o
obj-$(CONFIG_VEXPRESS_CONFIG) += vexpress-config.o
obj-$(CONFIG_DA8XX_MSTPRI) += da8xx-mstpri.o
+
+# MHI
+obj-$(CONFIG_MHI_BUS) += mhi/
diff --git a/drivers/bus/mhi/Kconfig b/drivers/bus/mhi/Kconfig
new file mode 100644
index 000000000000..0beaa862e054
--- /dev/null
+++ b/drivers/bus/mhi/Kconfig
@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# MHI bus
+#
+# Copyright (c) 2018-2020, The Linux Foundation. All rights reserved.
+#
+
+config MHI_BUS
+ tristate "Modem Host Interface (MHI) bus"
+ help
+ Bus driver for MHI protocol. Modem Host Interface (MHI) is a
+ communication protocol used by the host processors to control
+ and communicate with modem devices over a high speed peripheral
+ bus or shared memory.
+
+config MHI_DEBUG
+ bool "MHI debug support"
+ depends on MHI_BUS
+ help
+ Say yes here to enable debugging support in the MHI transport
+ and individual MHI client drivers. This option will impact
+ throughput as individual MHI packets and state transitions
+ will be logged.
+
+source "drivers/bus/mhi/controllers/Kconfig"
+source "drivers/bus/mhi/devices/Kconfig"
+
diff --git a/drivers/bus/mhi/Makefile b/drivers/bus/mhi/Makefile
new file mode 100644
index 000000000000..eb9a6f62a65e
--- /dev/null
+++ b/drivers/bus/mhi/Makefile
@@ -0,0 +1,9 @@
+# Makefile for the MHI stack
+#
+
+# core layer
+obj-y += core/
+
+obj-y += controllers/
+obj-y += devices/
+
diff --git a/drivers/bus/mhi/controllers/Kconfig b/drivers/bus/mhi/controllers/Kconfig
new file mode 100644
index 000000000000..e18b38b25e8e
--- /dev/null
+++ b/drivers/bus/mhi/controllers/Kconfig
@@ -0,0 +1,13 @@
+menu "MHI controllers"
+
+config MHI_QTI
+ tristate "MHI QTI"
+ depends on MHI_BUS
+ help
+ If you say yes to this option, MHI bus support for QTI modem chipsets
+ will be enabled. QTI PCIe based modems uses MHI as the communication
+ protocol. MHI control driver is the bus master for such modems. As the
+ bus master driver, it oversees power management operations such as
+ suspend, resume, powering on and off the device.
+
+endmenu
diff --git a/drivers/bus/mhi/controllers/Makefile b/drivers/bus/mhi/controllers/Makefile
new file mode 100644
index 000000000000..7a4954861c6a
--- /dev/null
+++ b/drivers/bus/mhi/controllers/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_MHI_QTI) += mhi_qti_m.o
+mhi_qti_m-y += mhi_qti.o mhi_arch_qti.o
diff --git a/drivers/bus/mhi/controllers/mhi_arch_qti.c b/drivers/bus/mhi/controllers/mhi_arch_qti.c
new file mode 100644
index 000000000000..d27f8b3abbcf
--- /dev/null
+++ b/drivers/bus/mhi/controllers/mhi_arch_qti.c
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.*/
+
+#include <linux/async.h>
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/list.h>
+#include <linux/of.h>
+#include <linux/memblock.h>
+#include <linux/module.h>
+#include <linux/msm-bus.h>
+#include <linux/msm_pcie.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/mhi.h>
+#include <linux/ipc_logging.h>
+#include "mhi_qti.h"
+
+struct arch_info {
+ struct mhi_dev *mhi_dev;
+ struct msm_bus_scale_pdata *msm_bus_pdata;
+ u32 bus_client;
+ struct pci_saved_state *pcie_state;
+ struct pci_saved_state *ref_pcie_state;
+ struct dma_iommu_mapping *mapping;
+};
+
+struct mhi_bl_info {
+ struct mhi_device *mhi_device;
+ async_cookie_t cookie;
+ void *ipc_log;
+};
+
+/* ipc log markings */
+#define DLOG "Dev->Host: "
+#define HLOG "Host: "
+
+#ifdef CONFIG_MHI_DEBUG
+
+#define MHI_IPC_LOG_PAGES (100)
+enum MHI_DEBUG_LEVEL mhi_ipc_log_lvl = MHI_MSG_LVL_VERBOSE;
+
+#else
+
+#define MHI_IPC_LOG_PAGES (10)
+enum MHI_DEBUG_LEVEL mhi_ipc_log_lvl = MHI_MSG_LVL_ERROR;
+
+#endif
+
+static int mhi_arch_set_bus_request(struct mhi_controller *mhi_cntrl, int index)
+{
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+ struct arch_info *arch_info = mhi_dev->arch_info;
+
+ MHI_LOG("Setting bus request to index %d\n", index);
+
+ if (arch_info->bus_client)
+ return msm_bus_scale_client_update_request(
+ arch_info->bus_client, index);
+
+ /* default return success */
+ return 0;
+}
+
+static void mhi_bl_dl_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result)
+{
+ struct mhi_bl_info *mhi_bl_info = mhi_device_get_devdata(mhi_dev);
+ char *buf = mhi_result->buf_addr;
+
+ /* force a null at last character */
+ buf[mhi_result->bytes_xferd - 1] = 0;
+
+ ipc_log_string(mhi_bl_info->ipc_log, "%s %s", DLOG, buf);
+}
+
+static void mhi_bl_dummy_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result)
+{
+}
+
+static void mhi_bl_remove(struct mhi_device *mhi_dev)
+{
+ struct mhi_bl_info *mhi_bl_info = mhi_device_get_devdata(mhi_dev);
+
+ ipc_log_string(mhi_bl_info->ipc_log, HLOG "Received Remove notif.\n");
+
+ /* wait for boot monitor to exit */
+ async_synchronize_cookie(mhi_bl_info->cookie + 1);
+}
+
+static void mhi_bl_boot_monitor(void *data, async_cookie_t cookie)
+{
+ struct mhi_bl_info *mhi_bl_info = data;
+ struct mhi_device *mhi_device = mhi_bl_info->mhi_device;
+ struct mhi_controller *mhi_cntrl = mhi_device->mhi_cntrl;
+ /* 15 sec timeout for booting device */
+ const u32 timeout = msecs_to_jiffies(15000);
+
+ /* wait for device to enter boot stage */
+ wait_event_timeout(mhi_cntrl->state_event, mhi_cntrl->ee == MHI_EE_AMSS
+ || mhi_cntrl->ee == MHI_EE_DISABLE_TRANSITION,
+ timeout);
+
+ if (mhi_cntrl->ee == MHI_EE_AMSS) {
+ ipc_log_string(mhi_bl_info->ipc_log, HLOG
+ "Device successfully booted to mission mode\n");
+
+ mhi_unprepare_from_transfer(mhi_device);
+ } else {
+ ipc_log_string(mhi_bl_info->ipc_log, HLOG
+ "Device failed to boot to mission mode, ee = %s\n",
+ TO_MHI_EXEC_STR(mhi_cntrl->ee));
+ }
+}
+
+static int mhi_bl_probe(struct mhi_device *mhi_dev,
+ const struct mhi_device_id *id)
+{
+ char node_name[32];
+ struct mhi_bl_info *mhi_bl_info;
+
+ mhi_bl_info = devm_kzalloc(&mhi_dev->dev, sizeof(*mhi_bl_info),
+ GFP_KERNEL);
+ if (!mhi_bl_info)
+ return -ENOMEM;
+
+ snprintf(node_name, sizeof(node_name), "mhi_bl_%04x_%02u.%02u.%02u",
+ mhi_dev->dev_id, mhi_dev->domain, mhi_dev->bus, mhi_dev->slot);
+
+ mhi_bl_info->ipc_log = ipc_log_context_create(MHI_IPC_LOG_PAGES,
+ node_name, 0);
+ if (!mhi_bl_info->ipc_log)
+ return -EINVAL;
+
+ mhi_bl_info->mhi_device = mhi_dev;
+ mhi_device_set_devdata(mhi_dev, mhi_bl_info);
+
+ ipc_log_string(mhi_bl_info->ipc_log, HLOG
+ "Entered SBL, Session ID:0x%x\n",
+ mhi_dev->mhi_cntrl->session_id);
+
+ /* start a thread to monitor entering mission mode */
+ mhi_bl_info->cookie = async_schedule(mhi_bl_boot_monitor, mhi_bl_info);
+
+ return 0;
+}
+
+static const struct mhi_device_id mhi_bl_match_table[] = {
+ { .chan = "BL" },
+ { },
+};
+
+static struct mhi_driver mhi_bl_driver = {
+ .id_table = mhi_bl_match_table,
+ .remove = mhi_bl_remove,
+ .probe = mhi_bl_probe,
+ .ul_xfer_cb = mhi_bl_dummy_cb,
+ .dl_xfer_cb = mhi_bl_dl_cb,
+ .driver = {
+ .name = "MHI_BL",
+ .owner = THIS_MODULE,
+ },
+};
+
+int mhi_arch_pcie_init(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+ struct arch_info *arch_info = mhi_dev->arch_info;
+ char node[32];
+
+ if (!arch_info) {
+ arch_info = devm_kzalloc(&mhi_dev->pci_dev->dev,
+ sizeof(*arch_info), GFP_KERNEL);
+ if (!arch_info)
+ return -ENOMEM;
+
+ mhi_dev->arch_info = arch_info;
+
+ snprintf(node, sizeof(node), "mhi_%04x_%02u.%02u.%02u",
+ mhi_cntrl->dev_id, mhi_cntrl->domain, mhi_cntrl->bus,
+ mhi_cntrl->slot);
+ mhi_cntrl->log_buf = ipc_log_context_create(MHI_IPC_LOG_PAGES,
+ node, 0);
+ mhi_cntrl->log_lvl = mhi_ipc_log_lvl;
+
+ /* save reference state for pcie config space */
+ arch_info->ref_pcie_state =
+ pci_store_saved_state(mhi_dev->pci_dev);
+
+ mhi_driver_register(&mhi_bl_driver);
+ }
+
+ return mhi_arch_set_bus_request(mhi_cntrl, 1);
+}
+
+void mhi_arch_pcie_deinit(struct mhi_controller *mhi_cntrl)
+{
+ mhi_arch_set_bus_request(mhi_cntrl, 0);
+}
+
+int mhi_arch_link_off(struct mhi_controller *mhi_cntrl, bool graceful)
+{
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+ struct arch_info *arch_info = mhi_dev->arch_info;
+ struct pci_dev *pci_dev = mhi_dev->pci_dev;
+ int ret;
+
+ MHI_LOG("Entered\n");
+
+ if (graceful) {
+ pci_clear_master(pci_dev);
+ ret = pci_save_state(mhi_dev->pci_dev);
+ if (ret) {
+ MHI_ERR("Failed with pci_save_state, ret:%d\n", ret);
+ return ret;
+ }
+
+ arch_info->pcie_state = pci_store_saved_state(pci_dev);
+ pci_disable_device(pci_dev);
+ }
+
+ /*
+ * We will always attempt to put link into D3hot, however
+ * link down may have happened due to error fatal, so
+ * ignoring the return code
+ */
+ pci_set_power_state(pci_dev, PCI_D3hot);
+
+ /* release the resources */
+ mhi_arch_set_bus_request(mhi_cntrl, 0);
+
+ MHI_LOG("Exited\n");
+
+ return 0;
+}
+
+int mhi_arch_link_on(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+ struct arch_info *arch_info = mhi_dev->arch_info;
+ struct pci_dev *pci_dev = mhi_dev->pci_dev;
+ int ret;
+
+ MHI_LOG("Entered\n");
+
+ /* request resources and establish link trainning */
+ ret = mhi_arch_set_bus_request(mhi_cntrl, 1);
+ if (ret)
+ MHI_LOG("Could not set bus frequency, ret:%d\n", ret);
+
+ ret = pci_set_power_state(pci_dev, PCI_D0);
+ if (ret) {
+ MHI_ERR("Failed to set PCI_D0 state, ret:%d\n", ret);
+ return ret;
+ }
+
+ ret = pci_enable_device(pci_dev);
+ if (ret) {
+ MHI_ERR("Failed to enable device, ret:%d\n", ret);
+ return ret;
+ }
+
+ ret = pci_load_and_free_saved_state(pci_dev, &arch_info->pcie_state);
+ if (ret)
+ MHI_LOG("Failed to load saved cfg state\n");
+
+ pci_restore_state(pci_dev);
+ pci_set_master(pci_dev);
+
+ MHI_LOG("Exited\n");
+
+ return 0;
+}
diff --git a/drivers/bus/mhi/controllers/mhi_qti.c b/drivers/bus/mhi/controllers/mhi_qti.c
new file mode 100644
index 000000000000..8333f64ea7c6
--- /dev/null
+++ b/drivers/bus/mhi/controllers/mhi_qti.c
@@ -0,0 +1,970 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.*/
+
+#ifdef CONFIG_ARM
+#include <asm/arch_timer.h>
+#endif
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/list.h>
+#include <linux/of.h>
+#include <linux/memblock.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/mhi.h>
+#include <linux/msi.h>
+#include <linux/interrupt.h>
+#include <linux/notifier.h>
+#include <linux/io.h>
+#include <linux/gpio/consumer.h>
+#include "mhi_qti.h"
+#include "../core/mhi_internal.h"
+
+/* #define MHI_KMOD_BUILD */
+
+#define WDOG_TIMEOUT 30
+#define MHI_PANIC_TIMER_STEP 1000
+
+static int mhi_panic_timeout; /* volatile? */
+static int mhi_pci_probe_result;
+
+void __iomem *wdt;
+
+static struct kobject *mhi_kobj;
+
+struct notifier_block *global_mhi_panic_notifier;
+
+static ssize_t sysfs_show(struct kobject *kobj, struct kobj_attribute *attr,
+ char *buf);
+static ssize_t sysfs_store(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *buf, size_t count);
+
+static struct kobj_attribute mhi_attr =
+ __ATTR(mhi_panic_timeout, 0660, sysfs_show, sysfs_store);
+
+static ssize_t sysfs_show(struct kobject *kobj, struct kobj_attribute *attr,
+ char *buf)
+{
+ return snprintf(buf, PAGE_SIZE, "%d\n", mhi_panic_timeout);
+}
+
+static ssize_t sysfs_store(struct kobject *kobj, struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ if (sscanf(buf, "%du", &mhi_panic_timeout) != 1) {
+ pr_err("failed to read timeout value from string\n");
+ return -EINVAL;
+ }
+ return count;
+}
+
+struct firmware_info {
+ unsigned int dev_id;
+ const char *fw_image;
+ const char *edl_image;
+};
+
+static const struct firmware_info firmware_table[] = {
+ { .dev_id = 0x306, .fw_image = "sdx55m/sbl1.mbn" },
+ { .dev_id = 0x305, .fw_image = "sdx50m/sbl1.mbn" },
+ { .dev_id = 0x304, .fw_image = "sbl.mbn", .edl_image = "edl.mbn" },
+ /* default, set to debug.mbn */
+ { .fw_image = "debug.mbn" },
+};
+
+static int debug_mode;
+module_param_named(debug_mode, debug_mode, int, 0644);
+
+int mhi_debugfs_trigger_m0(void *data, u64 val)
+{
+ struct mhi_controller *mhi_cntrl = data;
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+
+ MHI_LOG("Trigger M3 Exit\n");
+ pm_runtime_get(&mhi_dev->pci_dev->dev);
+ pm_runtime_put(&mhi_dev->pci_dev->dev);
+
+ return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(debugfs_trigger_m0_fops, NULL,
+ mhi_debugfs_trigger_m0, "%llu\n");
+
+int mhi_debugfs_trigger_m3(void *data, u64 val)
+{
+ struct mhi_controller *mhi_cntrl = data;
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+
+ MHI_LOG("Trigger M3 Entry\n");
+ pm_runtime_mark_last_busy(&mhi_dev->pci_dev->dev);
+ pm_request_autosuspend(&mhi_dev->pci_dev->dev);
+
+ return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(debugfs_trigger_m3_fops, NULL,
+ mhi_debugfs_trigger_m3, "%llu\n");
+
+void mhi_deinit_pci_dev(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+ struct pci_dev *pci_dev = mhi_dev->pci_dev;
+
+ pm_runtime_mark_last_busy(&pci_dev->dev);
+ pm_runtime_dont_use_autosuspend(&pci_dev->dev);
+ pm_runtime_disable(&pci_dev->dev);
+ pci_free_irq_vectors(pci_dev);
+ kfree(mhi_cntrl->irq);
+ mhi_cntrl->irq = NULL;
+ iounmap(mhi_cntrl->regs);
+ iounmap(wdt);
+ mhi_cntrl->regs = NULL;
+ pci_clear_master(pci_dev);
+ pci_release_region(pci_dev, mhi_dev->resn);
+ pci_disable_device(pci_dev);
+}
+
+static int mhi_init_pci_dev(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+ struct pci_dev *pci_dev = mhi_dev->pci_dev;
+ int ret;
+ resource_size_t len;
+ int i;
+
+ mhi_dev->resn = MHI_PCI_BAR_NUM;
+ ret = pci_assign_resource(pci_dev, mhi_dev->resn);
+ if (ret) {
+ MHI_ERR("Error assign pci resources, ret:%d\n", ret);
+ return ret;
+ }
+
+ ret = pci_enable_device(pci_dev);
+ if (ret) {
+ MHI_ERR("Error enabling device, ret:%d\n", ret);
+ goto error_enable_device;
+ }
+
+ ret = pci_request_region(pci_dev, mhi_dev->resn, "mhi");
+ if (ret) {
+ MHI_ERR("Error pci_request_region, ret:%d\n", ret);
+ goto error_request_region;
+ }
+
+ pci_set_master(pci_dev);
+
+ mhi_cntrl->base_addr = pci_resource_start(pci_dev, mhi_dev->resn);
+ len = pci_resource_len(pci_dev, mhi_dev->resn);
+ mhi_cntrl->regs = ioremap_nocache(mhi_cntrl->base_addr, len);
+ if (!mhi_cntrl->regs) {
+ MHI_ERR("Error ioremap region\n");
+ goto error_ioremap;
+ }
+#ifdef CONFIG_X86
+ ret = pci_alloc_irq_vectors(pci_dev, 1,
+ mhi_cntrl->msi_required, PCI_IRQ_ALL_TYPES);
+#else
+ ret = pci_alloc_irq_vectors(pci_dev, mhi_cntrl->msi_required,
+ mhi_cntrl->msi_required, PCI_IRQ_NOMSIX);
+#endif
+ if (IS_ERR_VALUE((ulong)ret)) {
+ MHI_ERR("Failed to enable MSI, ret:%d\n", ret);
+ goto error_req_msi;
+ }
+
+ mhi_cntrl->msi_allocated = (ret < mhi_cntrl->msi_required) ? 1 : ret;
+ mhi_cntrl->irq = kmalloc_array(mhi_cntrl->msi_allocated,
+ sizeof(*mhi_cntrl->irq), GFP_KERNEL);
+ if (!mhi_cntrl->irq) {
+ ret = -ENOMEM;
+ goto error_alloc_msi_vec;
+ }
+
+ for (i = 0; i < mhi_cntrl->msi_allocated; i++) {
+ mhi_cntrl->irq[i] = pci_irq_vector(pci_dev, i);
+ if (mhi_cntrl->irq[i] < 0) {
+ ret = mhi_cntrl->irq[i];
+ goto error_get_irq_vec;
+ }
+ }
+
+ dev_set_drvdata(&pci_dev->dev, mhi_cntrl);
+
+ /* configure runtime pm */
+ pm_runtime_set_autosuspend_delay(&pci_dev->dev, MHI_RPM_SUSPEND_TMR_MS);
+ pm_runtime_use_autosuspend(&pci_dev->dev);
+ pm_suspend_ignore_children(&pci_dev->dev, true);
+
+ /*
+ * pci framework will increment usage count (twice) before
+ * calling local device driver probe function.
+ * 1st pci.c pci_pm_init() calls pm_runtime_forbid
+ * 2nd pci-driver.c local_pci_probe calls pm_runtime_get_sync
+ * Framework expect pci device driver to call
+ * pm_runtime_put_noidle to decrement usage count after
+ * successful probe and call pm_runtime_allow to enable
+ * runtime suspend.
+ */
+ pm_runtime_mark_last_busy(&pci_dev->dev);
+ pm_runtime_put_noidle(&pci_dev->dev);
+
+ return 0;
+
+error_get_irq_vec:
+ kfree(mhi_cntrl->irq);
+ mhi_cntrl->irq = NULL;
+
+error_alloc_msi_vec:
+ pci_free_irq_vectors(pci_dev);
+
+error_req_msi:
+ iounmap(mhi_cntrl->regs);
+
+error_ioremap:
+ pci_clear_master(pci_dev);
+
+error_request_region:
+ pci_disable_device(pci_dev);
+
+error_enable_device:
+ pci_release_region(pci_dev, mhi_dev->resn);
+
+ return ret;
+}
+
+static int mhi_runtime_suspend(struct device *dev)
+{
+ int ret = 0;
+ struct mhi_controller *mhi_cntrl = dev_get_drvdata(dev);
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+
+ MHI_LOG("Enter\n");
+
+ mutex_lock(&mhi_cntrl->pm_mutex);
+
+ if (!mhi_dev->powered_on) {
+ MHI_LOG("Not fully powered, return success\n");
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+ return 0;
+ }
+
+ ret = mhi_pm_suspend(mhi_cntrl);
+ if (ret) {
+ MHI_LOG("Abort due to ret:%d\n", ret);
+ goto exit_runtime_suspend;
+ }
+
+ ret = mhi_arch_link_off(mhi_cntrl, true);
+ if (ret)
+ MHI_ERR("Failed to Turn off link ret:%d\n", ret);
+
+exit_runtime_suspend:
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+ MHI_LOG("Exited with ret:%d\n", ret);
+
+ return ret;
+}
+
+static int mhi_runtime_idle(struct device *dev)
+{
+ struct mhi_controller *mhi_cntrl = dev_get_drvdata(dev);
+
+ MHI_LOG("Entered returning -EBUSY\n");
+
+ /*
+ * RPM framework during runtime resume always calls
+ * rpm_idle to see if device ready to suspend.
+ * If dev.power usage_count count is 0, rpm fw will call
+ * rpm_idle cb to see if device is ready to suspend.
+ * if cb return 0, or cb not defined the framework will
+ * assume device driver is ready to suspend;
+ * therefore, fw will schedule runtime suspend.
+ * In MHI power management, MHI host shall go to
+ * runtime suspend only after entering MHI State M2, even if
+ * usage count is 0. Return -EBUSY to disable automatic suspend.
+ */
+ return -EBUSY;
+}
+
+static int mhi_runtime_resume(struct device *dev)
+{
+ int ret = 0;
+ struct mhi_controller *mhi_cntrl = dev_get_drvdata(dev);
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+
+ MHI_LOG("Enter\n");
+
+ mutex_lock(&mhi_cntrl->pm_mutex);
+
+ if (!mhi_dev->powered_on) {
+ MHI_LOG("Not fully powered, return success\n");
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+ return 0;
+ }
+
+ /* turn on link */
+ ret = mhi_arch_link_on(mhi_cntrl);
+ if (ret)
+ goto rpm_resume_exit;
+
+ /* enter M0 state */
+ ret = mhi_pm_resume(mhi_cntrl);
+
+rpm_resume_exit:
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+ MHI_LOG("Exited with :%d\n", ret);
+
+ return ret;
+}
+
+static int mhi_system_resume(struct device *dev)
+{
+ int ret = 0;
+ struct mhi_controller *mhi_cntrl = dev_get_drvdata(dev);
+
+ ret = mhi_runtime_resume(dev);
+ if (ret) {
+ MHI_ERR("Failed to resume link\n");
+ } else {
+ pm_runtime_set_active(dev);
+ pm_runtime_enable(dev);
+ }
+
+ return ret;
+}
+
+int mhi_system_suspend(struct device *dev)
+{
+ struct mhi_controller *mhi_cntrl = dev_get_drvdata(dev);
+ int ret;
+
+ MHI_LOG("Entered\n");
+
+ /* if rpm status still active then force suspend */
+ if (!pm_runtime_status_suspended(dev)) {
+ ret = mhi_runtime_suspend(dev);
+ if (ret) {
+ MHI_LOG("suspend failed ret:%d\n", ret);
+ return ret;
+ }
+ }
+
+ pm_runtime_set_suspended(dev);
+ pm_runtime_disable(dev);
+
+ MHI_LOG("Exit\n");
+ return 0;
+}
+
+/* checks if link is down */
+static int mhi_link_status(struct mhi_controller *mhi_cntrl, void *priv)
+{
+ struct mhi_dev *mhi_dev = priv;
+ u16 dev_id;
+ int ret;
+
+ /* try reading device id, if dev id don't match, link is down */
+ ret = pci_read_config_word(mhi_dev->pci_dev, PCI_DEVICE_ID, &dev_id);
+
+ return (ret || dev_id != mhi_cntrl->dev_id) ? -EIO : 0;
+}
+
+/* disable PCIe L1 */
+static int mhi_lpm_disable(struct mhi_controller *mhi_cntrl, void *priv)
+{
+ struct mhi_dev *mhi_dev = priv;
+ struct pci_dev *pci_dev = mhi_dev->pci_dev;
+ int lnkctl = pci_dev->pcie_cap + PCI_EXP_LNKCTL;
+ u8 val;
+ int ret;
+
+ ret = pci_read_config_byte(pci_dev, lnkctl, &val);
+ if (ret) {
+ MHI_ERR("Error reading LNKCTL, ret:%d\n", ret);
+ return ret;
+ }
+
+ /* L1 is not supported or already disabled */
+ if (!(val & PCI_EXP_LNKCTL_ASPM_L1))
+ return 0;
+
+ val &= ~PCI_EXP_LNKCTL_ASPM_L1;
+ ret = pci_write_config_byte(pci_dev, lnkctl, val);
+ if (ret) {
+ MHI_ERR("Error writing LNKCTL to disable LPM, ret:%d\n", ret);
+ return ret;
+ }
+
+ mhi_dev->lpm_disabled = true;
+
+ return ret;
+}
+
+/* enable PCIe L1 */
+static int mhi_lpm_enable(struct mhi_controller *mhi_cntrl, void *priv)
+{
+ struct mhi_dev *mhi_dev = priv;
+ struct pci_dev *pci_dev = mhi_dev->pci_dev;
+ int lnkctl = pci_dev->pcie_cap + PCI_EXP_LNKCTL;
+ u8 val;
+ int ret;
+
+ /* L1 is not supported or already disabled */
+ if (!mhi_dev->lpm_disabled)
+ return 0;
+
+ ret = pci_read_config_byte(pci_dev, lnkctl, &val);
+ if (ret) {
+ MHI_ERR("Error reading LNKCTL, ret:%d\n", ret);
+ return ret;
+ }
+
+ val |= PCI_EXP_LNKCTL_ASPM_L1;
+ ret = pci_write_config_byte(pci_dev, lnkctl, val);
+ if (ret) {
+ MHI_ERR("Error writing LNKCTL to enable LPM, ret:%d\n", ret);
+ return ret;
+ }
+
+ mhi_dev->lpm_disabled = false;
+
+ return ret;
+}
+
+static int mhi_power_up(struct mhi_controller *mhi_cntrl)
+{
+ enum mhi_dev_state dev_state = mhi_get_mhi_state(mhi_cntrl);
+ const u32 delayus = 10;
+ int itr = DIV_ROUND_UP(mhi_cntrl->timeout_ms * 1000, delayus);
+ int ret;
+
+ /*
+ * It's possible device did not go thru a cold reset before
+ * power up and still in error state. If device in error state,
+ * we need to trigger a soft reset before continue with power
+ * up
+ */
+ if (dev_state == MHI_STATE_SYS_ERR) {
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET);
+ while (itr--) {
+ dev_state = mhi_get_mhi_state(mhi_cntrl);
+ if (dev_state != MHI_STATE_SYS_ERR)
+ break;
+ usleep_range(delayus, delayus << 1);
+ }
+ /* device still in error state, abort power up */
+ if (dev_state == MHI_STATE_SYS_ERR)
+ return -EIO;
+ }
+
+ ret = mhi_async_power_up(mhi_cntrl);
+
+ /* power up create the dentry */
+ if (mhi_cntrl->dentry) {
+ debugfs_create_file("m0", 0444, mhi_cntrl->dentry, mhi_cntrl,
+ &debugfs_trigger_m0_fops);
+ debugfs_create_file("m3", 0444, mhi_cntrl->dentry, mhi_cntrl,
+ &debugfs_trigger_m3_fops);
+ }
+
+ return ret;
+}
+
+static int mhi_runtime_get(struct mhi_controller *mhi_cntrl, void *priv)
+{
+ struct mhi_dev *mhi_dev = priv;
+ struct device *dev = &mhi_dev->pci_dev->dev;
+
+ return pm_runtime_get(dev);
+}
+
+static void mhi_runtime_put(struct mhi_controller *mhi_cntrl, void *priv)
+{
+ struct mhi_dev *mhi_dev = priv;
+ struct device *dev = &mhi_dev->pci_dev->dev;
+
+ pm_runtime_put_noidle(dev);
+}
+
+static void mhi_status_cb(struct mhi_controller *mhi_cntrl,
+ void *priv, enum MHI_CB reason)
+{
+ struct mhi_dev *mhi_dev = priv;
+ struct device *dev = &mhi_dev->pci_dev->dev;
+
+ if (reason == MHI_CB_IDLE) {
+ MHI_LOG("Schedule runtime suspend 1\n");
+ pm_runtime_mark_last_busy(dev);
+ pm_request_autosuspend(dev);
+ }
+}
+
+/* capture host SoC XO time in ticks */
+static u64 mhi_time_get(struct mhi_controller *mhi_cntrl, void *priv)
+{
+#ifdef CONFIG_X86
+ u64 now;
+
+ now = rdtsc_ordered();
+
+ return now;
+#else
+ return arch_counter_get_cntvct();
+#endif
+}
+
+static ssize_t timeout_ms_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct mhi_device *mhi_dev = to_mhi_device(dev);
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+
+ /* buffer provided by sysfs has a minimum size of PAGE_SIZE */
+ return snprintf(buf, PAGE_SIZE, "%u\n", mhi_cntrl->timeout_ms);
+}
+
+static ssize_t timeout_ms_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct mhi_device *mhi_dev = to_mhi_device(dev);
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ u32 timeout_ms;
+
+ if (kstrtou32(buf, 0, &timeout_ms) < 0)
+ return -EINVAL;
+
+ mhi_cntrl->timeout_ms = timeout_ms;
+
+ return count;
+}
+
+static DEVICE_ATTR_RW(timeout_ms);
+
+static ssize_t power_up_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ struct mhi_device *mhi_dev = to_mhi_device(dev);
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+
+ ret = mhi_power_up(mhi_cntrl);
+ if (ret)
+ return ret;
+
+ return count;
+}
+
+static DEVICE_ATTR_WO(power_up);
+
+static struct attribute *mhi_attrs[] = {
+ &dev_attr_timeout_ms.attr,
+ &dev_attr_power_up.attr,
+ NULL
+};
+
+static const struct attribute_group mhi_group = {
+ .attrs = mhi_attrs,
+};
+
+static struct mhi_controller *mhi_register_controller(struct pci_dev *pci_dev)
+{
+ struct mhi_controller *mhi_cntrl;
+ struct mhi_dev *mhi_dev;
+ struct device_node *of_node = pci_dev->dev.of_node;
+ const struct firmware_info *firmware_info;
+ bool use_bb;
+ u64 addr_win[2];
+ int ret, i;
+
+#ifdef CONFIG_OF
+ if (!of_node)
+ return ERR_PTR(-ENODEV);
+#endif
+
+ mhi_cntrl = mhi_alloc_controller(sizeof(*mhi_dev));
+ if (!mhi_cntrl)
+ return ERR_PTR(-ENOMEM);
+
+ mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+
+ mhi_cntrl->domain = pci_domain_nr(pci_dev->bus);
+ mhi_cntrl->dev_id = pci_dev->device;
+ mhi_cntrl->bus = pci_dev->bus->number;
+ mhi_cntrl->slot = PCI_SLOT(pci_dev->devfn);
+
+#ifdef CONFIG_OF
+ use_bb = of_property_read_bool(of_node, "mhi,use-bb");
+#else
+ use_bb = false;
+#endif
+
+ /*
+ * if s1 translation enabled or using bounce buffer pull iova addr
+ * from dt
+ */
+ if (use_bb || (mhi_dev->smmu_cfg & MHI_SMMU_ATTACH &&
+ !(mhi_dev->smmu_cfg & MHI_SMMU_S1_BYPASS))) {
+#ifdef CONFIG_OF
+ ret = of_property_count_elems_of_size(of_node, "qti,addr-win",
+ sizeof(addr_win));
+ if (ret != 1)
+ goto error_register;
+ ret = of_property_read_u64_array(of_node, "qti,addr-win",
+ addr_win, 2);
+ if (ret)
+ goto error_register;
+#else
+ goto error_register;
+#endif
+ } else {
+#ifdef MHI_KMOD_BUILD
+ /* FIXME: Modify values for different platform */
+ addr_win[0] = 0x1000;
+ addr_win[1] = 0x17c800000;
+#else
+ addr_win[0] = memblock_start_of_DRAM();
+ addr_win[1] = memblock_end_of_DRAM();
+#endif
+ }
+
+ mhi_dev->iova_start = addr_win[0];
+ mhi_dev->iova_stop = addr_win[1];
+
+ /*
+ * If S1 is enabled, set MHI_CTRL start address to 0 so we can use low
+ * level mapping api to map buffers outside of smmu domain
+ */
+ if (mhi_dev->smmu_cfg & MHI_SMMU_ATTACH &&
+ !(mhi_dev->smmu_cfg & MHI_SMMU_S1_BYPASS))
+ mhi_cntrl->iova_start = 0;
+ else
+ mhi_cntrl->iova_start = addr_win[0];
+
+ mhi_cntrl->iova_stop = mhi_dev->iova_stop;
+ mhi_cntrl->of_node = of_node;
+
+ mhi_dev->pci_dev = pci_dev;
+
+ /* setup power management apis */
+ mhi_cntrl->status_cb = mhi_status_cb;
+ mhi_cntrl->runtime_get = mhi_runtime_get;
+ mhi_cntrl->runtime_put = mhi_runtime_put;
+ mhi_cntrl->link_status = mhi_link_status;
+
+ mhi_cntrl->lpm_disable = mhi_lpm_disable;
+ mhi_cntrl->lpm_enable = mhi_lpm_enable;
+ mhi_cntrl->time_get = mhi_time_get;
+
+ ret = of_register_mhi_controller(mhi_cntrl);
+ if (ret)
+ goto error_register;
+
+ for (i = 0; i < ARRAY_SIZE(firmware_table); i++) {
+ firmware_info = firmware_table + i;
+
+ /* debug mode always use default */
+ if (!debug_mode && mhi_cntrl->dev_id == firmware_info->dev_id)
+ break;
+ }
+
+ mhi_cntrl->fw_image = firmware_info->fw_image;
+ mhi_cntrl->edl_image = firmware_info->edl_image;
+
+ if (sysfs_create_group(&mhi_cntrl->mhi_dev->dev.kobj, &mhi_group))
+ MHI_ERR("Error while creating the sysfs group\n");
+
+ return mhi_cntrl;
+
+error_register:
+ mhi_free_controller(mhi_cntrl);
+
+ return ERR_PTR(-EINVAL);
+}
+
+static int mhi_panic_handler(struct notifier_block *this,
+ unsigned long event, void *ptr)
+{
+ int mdmreboot = 0, i;
+ struct gpio_desc *ap2mdm, *mdm2ap;
+ struct mhi_controller *mhi_cntrl = container_of(this,
+ struct mhi_controller,
+ mhi_panic_notifier);
+
+ ap2mdm = devm_gpiod_get_optional(mhi_cntrl->dev, "ap2mdm",
+ GPIOD_OUT_LOW);
+ if (IS_ERR(ap2mdm))
+ return PTR_ERR(ap2mdm);
+
+ mdm2ap = devm_gpiod_get_optional(mhi_cntrl->dev, "mdm2ap", GPIOD_IN);
+ if (IS_ERR(mdm2ap))
+ return PTR_ERR(mdm2ap);
+
+ /*
+ * ap2mdm_status is set to 0 to indicate the SDX
+ * that IPQ has crashed. Now the SDX has to take
+ * dump.
+ */
+ gpiod_set_value(ap2mdm, 0);
+
+ if (!mhi_panic_timeout)
+ return NOTIFY_DONE;
+
+ if (mhi_panic_timeout > WDOG_TIMEOUT)
+ writel(0, wdt);
+
+ for (i = 0; i < mhi_panic_timeout; i++) {
+ /*
+ * Waiting for the mdm2ap status to be 0
+ * which indicates that SDX is rebooting and entering
+ * the crashdump path.
+ */
+ if (!mdmreboot && gpiod_get_value(mdm2ap)) {
+ MHI_LOG("MDM is rebooting to the crashdump path\n");
+ mdmreboot = 1;
+ }
+
+ /*
+ * Waiting for the mdm2ap status to be 1
+ * which indicates that SDX has completed crashdump
+ * collection and booted successfully.
+ */
+ if (mdmreboot && !(gpiod_get_value(mdm2ap))) {
+ MHI_LOG("MDM booted successfully after crashdump\n");
+ break;
+ }
+
+ mdelay(MHI_PANIC_TIMER_STEP);
+ }
+
+ if (mhi_panic_timeout > WDOG_TIMEOUT)
+ writel(1, wdt);
+
+ return NOTIFY_DONE;
+}
+
+void mhi_wdt_panic_handler(void)
+{
+ mhi_panic_handler(global_mhi_panic_notifier, 0, NULL);
+}
+EXPORT_SYMBOL(mhi_wdt_panic_handler);
+
+int __meminit mhi_pci_probe(struct pci_dev *pci_dev,
+ const struct pci_device_id *device_id)
+{
+ struct mhi_controller *mhi_cntrl;
+ u32 domain = pci_domain_nr(pci_dev->bus);
+ u32 bus = pci_dev->bus->number;
+ u32 dev_id = pci_dev->device;
+ u32 slot = PCI_SLOT(pci_dev->devfn);
+ struct mhi_dev *mhi_dev;
+ int ret;
+ bool use_panic_notifier;
+
+ /* see if we already registered */
+ mhi_cntrl = mhi_bdf_to_controller(domain, bus, slot, dev_id);
+ if (!mhi_cntrl)
+ mhi_cntrl = mhi_register_controller(pci_dev);
+
+ if (IS_ERR(mhi_cntrl))
+ return PTR_ERR(mhi_cntrl);
+
+ mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+ mhi_dev->powered_on = true;
+
+ ret = mhi_arch_pcie_init(mhi_cntrl);
+ if (ret)
+ return ret;
+
+ mhi_cntrl->dev = &pci_dev->dev;
+ ret = mhi_init_pci_dev(mhi_cntrl);
+ if (ret)
+ goto error_init_pci;
+
+ /* start power up sequence */
+ if (!debug_mode) {
+ ret = mhi_power_up(mhi_cntrl);
+ if (ret)
+ goto error_power_up;
+ }
+
+ pm_runtime_mark_last_busy(&pci_dev->dev);
+ pm_runtime_allow(&pci_dev->dev);
+
+#ifdef CONFIG_OF
+ use_panic_notifier =
+ of_property_read_bool(mhi_cntrl->of_node, "mhi,use-panic-notifer");
+#else
+ use_panic_notifier = true;
+#endif
+
+ if (!use_panic_notifier)
+ goto good_out;
+
+ mhi_cntrl->mhi_panic_notifier.notifier_call = mhi_panic_handler;
+ global_mhi_panic_notifier = &(mhi_cntrl->mhi_panic_notifier);
+
+ ret = atomic_notifier_chain_register(&panic_notifier_list,
+ &mhi_cntrl->mhi_panic_notifier);
+ MHI_LOG("MHI panic notifier registered\n");
+
+ wdt = ioremap(0x0B017008, 4);
+
+ /* Creating a directory in /sys/kernel/ */
+ mhi_kobj = kobject_create_and_add("mhi", kernel_kobj);
+
+ if (mhi_kobj) {
+ /* Creating sysfs file for mhi_panic_timeout */
+ if (sysfs_create_file(mhi_kobj, &mhi_attr.attr)) {
+ MHI_ERR
+ ("Cannot create sysfs file mhi_panic_timeout\n");
+ kobject_put(mhi_kobj);
+ }
+ } else {
+ MHI_ERR("Unable to create mhi sysfs entry\n");
+ }
+
+good_out:
+ MHI_LOG("Return successful\n");
+
+ return 0;
+
+error_power_up:
+ mhi_deinit_pci_dev(mhi_cntrl);
+
+error_init_pci:
+ mhi_arch_pcie_deinit(mhi_cntrl);
+ mhi_unregister_mhi_controller(mhi_cntrl);
+ mhi_pci_probe_result = ret;
+
+ return ret;
+}
+
+void mhi_pci_device_removed(struct pci_dev *pci_dev)
+{
+ struct mhi_controller *mhi_cntrl;
+ u32 domain = pci_domain_nr(pci_dev->bus);
+ u32 bus = pci_dev->bus->number;
+ u32 dev_id = pci_dev->device;
+ u32 slot = PCI_SLOT(pci_dev->devfn);
+
+ if (mhi_kobj) {
+ sysfs_remove_file(mhi_kobj, &mhi_attr.attr);
+ kobject_del(mhi_kobj);
+ }
+
+ mhi_cntrl = mhi_bdf_to_controller(domain, bus, slot, dev_id);
+
+ if (mhi_cntrl) {
+
+ struct mhi_dev *mhi_dev = mhi_controller_get_devdata(mhi_cntrl);
+
+ pm_stay_awake(&mhi_cntrl->mhi_dev->dev);
+
+ /* if link is in drv suspend, wake it up */
+ pm_runtime_get_sync(&pci_dev->dev);
+
+ mutex_lock(&mhi_cntrl->pm_mutex);
+ if (!mhi_dev->powered_on) {
+ MHI_LOG("Not in active state\n");
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+ pm_runtime_put_noidle(&pci_dev->dev);
+ return;
+ }
+ mhi_dev->powered_on = false;
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+
+ pm_runtime_put_noidle(&pci_dev->dev);
+
+ MHI_LOG("Triggering shutdown process\n");
+ mhi_power_down(mhi_cntrl, false);
+
+ /* turn the link off */
+ mhi_deinit_pci_dev(mhi_cntrl);
+ mhi_arch_link_off(mhi_cntrl, false);
+
+ mhi_arch_pcie_deinit(mhi_cntrl);
+
+ pm_relax(&mhi_cntrl->mhi_dev->dev);
+
+ mhi_unregister_mhi_controller(mhi_cntrl);
+
+ mhi_free_controller(mhi_cntrl);
+ }
+}
+
+int get_mhi_pci_status(void)
+{
+ return mhi_pci_probe_result;
+}
+EXPORT_SYMBOL(get_mhi_pci_status);
+
+static const struct dev_pm_ops pm_ops = {
+ SET_RUNTIME_PM_OPS(mhi_runtime_suspend,
+ mhi_runtime_resume,
+ mhi_runtime_idle)
+ SET_SYSTEM_SLEEP_PM_OPS(mhi_system_suspend, mhi_system_resume)
+};
+
+static struct pci_device_id mhi_pcie_device_id[] = {
+ { PCI_DEVICE(MHI_PCIE_VENDOR_ID, 0x0300) },
+ { PCI_DEVICE(MHI_PCIE_VENDOR_ID, 0x0301) },
+ { PCI_DEVICE(MHI_PCIE_VENDOR_ID, 0x0302) },
+ { PCI_DEVICE(MHI_PCIE_VENDOR_ID, 0x0303) },
+ { PCI_DEVICE(MHI_PCIE_VENDOR_ID, 0x0304) },
+ { PCI_DEVICE(MHI_PCIE_VENDOR_ID, 0x0305) },
+ { PCI_DEVICE(MHI_PCIE_VENDOR_ID, 0x0306) },
+ /* SDX55 module */
+ { PCI_DEVICE(0x105b, 0xe0ab) },
+ { PCI_DEVICE(0x105b, 0xe0b0) },
+ { PCI_DEVICE(0x105b, 0xe0b1) },
+ { PCI_DEVICE(0x105b, 0xe0b3) },
+ { PCI_DEVICE(0x1269, 0x00b3) }, /* Thales */
+ { PCI_DEVICE(0x03f0, 0x0a6c) },
+ { PCI_DEVICE(MHI_PCIE_VENDOR_ID, MHI_PCIE_DEBUG_ID) },
+ { 0 },
+};
+
+static struct pci_driver mhi_pcie_driver = {
+ .name = "mhi",
+ .id_table = mhi_pcie_device_id,
+ .probe = mhi_pci_probe,
+ .remove = mhi_pci_device_removed,
+ .driver = {
+ .pm = &pm_ops
+ }
+};
+
+static int __init mhi_pcie_init(void)
+{
+ int ret;
+
+ ret = pci_register_driver(&mhi_pcie_driver);
+ ret |= mhi_pci_probe_result;
+ if (ret) {
+ pr_err("failed to register mhi pcie driver: %d\n",
+ ret);
+ pci_unregister_driver(&mhi_pcie_driver);
+ }
+
+ return ret;
+}
+
+module_init(mhi_pcie_init);
+
+static void __exit mhi_pcie_exit(void)
+{
+ pci_unregister_driver(&mhi_pcie_driver);
+}
+
+module_exit(mhi_pcie_exit);
+
+MODULE_AUTHOR("Qualcomm Corporation");
+MODULE_DESCRIPTION("Qualcomm Modem Host Interface Bus Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/bus/mhi/controllers/mhi_qti.h b/drivers/bus/mhi/controllers/mhi_qti.h
new file mode 100644
index 000000000000..34e6e4ad9f31
--- /dev/null
+++ b/drivers/bus/mhi/controllers/mhi_qti.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.*/
+
+#ifndef _MHI_QTI_
+#define _MHI_QTI_
+
+/* iova cfg bitmask */
+#define MHI_SMMU_ATTACH BIT(0)
+#define MHI_SMMU_S1_BYPASS BIT(1)
+#define MHI_SMMU_FAST BIT(2)
+#define MHI_SMMU_ATOMIC BIT(3)
+#define MHI_SMMU_FORCE_COHERENT BIT(4)
+
+#define MHI_PCIE_VENDOR_ID (0x17cb)
+#define MHI_PCIE_DEBUG_ID (0xffff)
+
+/* runtime suspend timer */
+#define MHI_RPM_SUSPEND_TMR_MS (250)
+#define MHI_PCI_BAR_NUM (0)
+
+struct mhi_dev {
+ struct pci_dev *pci_dev;
+ u32 smmu_cfg;
+ int resn;
+ void *arch_info;
+ bool powered_on;
+ dma_addr_t iova_start;
+ dma_addr_t iova_stop;
+ bool lpm_disabled;
+};
+
+void mhi_deinit_pci_dev(struct mhi_controller *mhi_cntrl);
+int mhi_pci_probe(struct pci_dev *pci_dev,
+ const struct pci_device_id *device_id);
+
+void mhi_pci_device_removed(struct pci_dev *pci_dev);
+int mhi_arch_pcie_init(struct mhi_controller *mhi_cntrl);
+void mhi_arch_pcie_deinit(struct mhi_controller *mhi_cntrl);
+int mhi_arch_iommu_init(struct mhi_controller *mhi_cntrl);
+void mhi_arch_iommu_deinit(struct mhi_controller *mhi_cntrl);
+int mhi_arch_link_off(struct mhi_controller *mhi_cntrl, bool graceful);
+int mhi_arch_link_on(struct mhi_controller *mhi_cntrl);
+
+#endif /* _MHI_QTI_ */
diff --git a/drivers/bus/mhi/core/Makefile b/drivers/bus/mhi/core/Makefile
new file mode 100644
index 000000000000..b1c81f2c9ac0
--- /dev/null
+++ b/drivers/bus/mhi/core/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_MHI_BUS) += mhi.o
+mhi-y := mhi_init.o mhi_main.o mhi_pm.o mhi_boot.o mhi_dtr.o
diff --git a/drivers/bus/mhi/core/mhi_boot.c b/drivers/bus/mhi/core/mhi_boot.c
new file mode 100644
index 000000000000..c669e9ce73c1
--- /dev/null
+++ b/drivers/bus/mhi/core/mhi_boot.c
@@ -0,0 +1,590 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. */
+
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/dma-mapping.h>
+#include <linux/firmware.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/of.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/wait.h>
+#include <linux/mhi.h>
+#include "mhi_internal.h"
+
+/* setup rddm vector table for rddm transfer */
+static void mhi_rddm_prepare(struct mhi_controller *mhi_cntrl,
+ struct image_info *img_info)
+{
+ struct mhi_buf *mhi_buf = img_info->mhi_buf;
+ struct bhi_vec_entry *bhi_vec = img_info->bhi_vec;
+ int i = 0;
+
+ for (i = 0; i < img_info->entries - 1; i++, mhi_buf++, bhi_vec++) {
+ MHI_VERB("Setting vector:%pad size:%zu\n",
+ &mhi_buf->dma_addr, mhi_buf->len);
+ bhi_vec->dma_addr = mhi_buf->dma_addr;
+ bhi_vec->size = mhi_buf->len;
+ }
+}
+
+/* collect rddm during kernel panic */
+static int __mhi_download_rddm_in_panic(struct mhi_controller *mhi_cntrl)
+{
+ int ret;
+ struct mhi_buf *mhi_buf;
+ u32 sequence_id;
+ u32 rx_status;
+ enum mhi_ee ee;
+ struct image_info *rddm_image = mhi_cntrl->rddm_image;
+ const u32 delayus = 2000;
+ u32 retry = (mhi_cntrl->timeout_ms * 1000) / delayus;
+ const u32 rddm_timeout_us = 200000;
+ int rddm_retry = rddm_timeout_us / delayus; /* time to enter rddm */
+ void __iomem *base = mhi_cntrl->bhie;
+
+ MHI_LOG("Entered with pm_state:%s dev_state:%s ee:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ TO_MHI_EXEC_STR(mhi_cntrl->ee));
+
+ /*
+ * This should only be executing during a kernel panic, we expect all
+ * other cores to shutdown while we're collecting rddm buffer. After
+ * returning from this function, we expect device to reset.
+ *
+ * Normaly, we would read/write pm_state only after grabbing
+ * pm_lock, since we're in a panic, skipping it. Also there is no
+ * gurantee this state change would take effect since
+ * we're setting it w/o grabbing pmlock, it's best effort
+ */
+ mhi_cntrl->pm_state = MHI_PM_LD_ERR_FATAL_DETECT;
+ /* update should take the effect immediately */
+ smp_wmb();
+
+ /* setup the RX vector table */
+ mhi_rddm_prepare(mhi_cntrl, rddm_image);
+ mhi_buf = &rddm_image->mhi_buf[rddm_image->entries - 1];
+
+ MHI_LOG("Starting BHIe programming for RDDM\n");
+
+ mhi_write_reg(mhi_cntrl, base, BHIE_RXVECADDR_HIGH_OFFS,
+ upper_32_bits(mhi_buf->dma_addr));
+
+ mhi_write_reg(mhi_cntrl, base, BHIE_RXVECADDR_LOW_OFFS,
+ lower_32_bits(mhi_buf->dma_addr));
+
+ mhi_write_reg(mhi_cntrl, base, BHIE_RXVECSIZE_OFFS, mhi_buf->len);
+ sequence_id = prandom_u32() & BHIE_RXVECSTATUS_SEQNUM_BMSK;
+
+ if (unlikely(!sequence_id))
+ sequence_id = 1;
+
+ mhi_write_reg_field(mhi_cntrl, base, BHIE_RXVECDB_OFFS,
+ BHIE_RXVECDB_SEQNUM_BMSK, BHIE_RXVECDB_SEQNUM_SHFT,
+ sequence_id);
+
+ MHI_LOG("Trigger device into RDDM mode\n");
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_SYS_ERR);
+
+ MHI_LOG("Waiting for device to enter RDDM\n");
+ while (rddm_retry--) {
+ ee = mhi_get_exec_env(mhi_cntrl);
+ if (ee == MHI_EE_RDDM)
+ break;
+
+ udelay(delayus);
+ }
+
+ if (rddm_retry <= 0) {
+ /* This is a hardware reset, will force device to enter rddm */
+ MHI_LOG("Did not enter RDDM, req. reset to force RDDM\n");
+ mhi_write_reg(mhi_cntrl, mhi_cntrl->regs,
+ MHI_SOC_RESET_REQ_OFFSET, MHI_SOC_RESET_REQ);
+ udelay(delayus);
+ }
+
+ ee = mhi_get_exec_env(mhi_cntrl);
+ MHI_LOG("Waiting for image download completion, current EE:%s\n",
+ TO_MHI_EXEC_STR(ee));
+ while (retry--) {
+ ret = mhi_read_reg_field(mhi_cntrl, base, BHIE_RXVECSTATUS_OFFS,
+ BHIE_RXVECSTATUS_STATUS_BMSK,
+ BHIE_RXVECSTATUS_STATUS_SHFT,
+ &rx_status);
+ if (ret)
+ return -EIO;
+
+ if (rx_status == BHIE_RXVECSTATUS_STATUS_XFER_COMPL) {
+ MHI_LOG("RDDM successfully collected\n");
+ return 0;
+ }
+
+ udelay(delayus);
+ }
+
+ ee = mhi_get_exec_env(mhi_cntrl);
+ ret = mhi_read_reg(mhi_cntrl, base, BHIE_RXVECSTATUS_OFFS, &rx_status);
+
+ MHI_ERR("Did not complete RDDM transfer\n");
+ MHI_ERR("Current EE:%s\n", TO_MHI_EXEC_STR(ee));
+ MHI_ERR("RXVEC_STATUS:0x%x, ret:%d\n", rx_status, ret);
+
+ return -EIO;
+}
+
+/* download ramdump image from device */
+int mhi_download_rddm_img(struct mhi_controller *mhi_cntrl, bool in_panic)
+{
+ void __iomem *base = mhi_cntrl->bhie;
+ rwlock_t *pm_lock = &mhi_cntrl->pm_lock;
+ struct image_info *rddm_image = mhi_cntrl->rddm_image;
+ struct mhi_buf *mhi_buf;
+ int ret;
+ u32 rx_status;
+ u32 sequence_id;
+
+ if (!rddm_image)
+ return -ENOMEM;
+
+ if (in_panic)
+ return __mhi_download_rddm_in_panic(mhi_cntrl);
+
+ MHI_LOG("Waiting for device to enter RDDM state from EE:%s\n",
+ TO_MHI_EXEC_STR(mhi_cntrl->ee));
+
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->ee == MHI_EE_RDDM ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ MHI_ERR("MHI is not in valid state, pm_state:%s ee:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_EXEC_STR(mhi_cntrl->ee));
+ return -EIO;
+ }
+
+ mhi_rddm_prepare(mhi_cntrl, mhi_cntrl->rddm_image);
+
+ /* vector table is the last entry */
+ mhi_buf = &rddm_image->mhi_buf[rddm_image->entries - 1];
+
+ read_lock_bh(pm_lock);
+ if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
+ read_unlock_bh(pm_lock);
+ return -EIO;
+ }
+
+ MHI_LOG("Starting BHIe Programming for RDDM\n");
+
+ mhi_write_reg(mhi_cntrl, base, BHIE_RXVECADDR_HIGH_OFFS,
+ upper_32_bits(mhi_buf->dma_addr));
+
+ mhi_write_reg(mhi_cntrl, base, BHIE_RXVECADDR_LOW_OFFS,
+ lower_32_bits(mhi_buf->dma_addr));
+
+ mhi_write_reg(mhi_cntrl, base, BHIE_RXVECSIZE_OFFS, mhi_buf->len);
+
+ sequence_id = prandom_u32() & BHIE_RXVECSTATUS_SEQNUM_BMSK;
+ mhi_write_reg_field(mhi_cntrl, base, BHIE_RXVECDB_OFFS,
+ BHIE_RXVECDB_SEQNUM_BMSK, BHIE_RXVECDB_SEQNUM_SHFT,
+ sequence_id);
+ read_unlock_bh(pm_lock);
+
+ MHI_LOG("Upper:0x%x Lower:0x%x len:0x%zx sequence:%u\n",
+ upper_32_bits(mhi_buf->dma_addr),
+ lower_32_bits(mhi_buf->dma_addr), mhi_buf->len, sequence_id);
+ MHI_LOG("Waiting for image download completion\n");
+
+ /* waiting for image download completion */
+ wait_event_timeout(mhi_cntrl->state_event,
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) ||
+ mhi_read_reg_field(mhi_cntrl, base,
+ BHIE_RXVECSTATUS_OFFS,
+ BHIE_RXVECSTATUS_STATUS_BMSK,
+ BHIE_RXVECSTATUS_STATUS_SHFT,
+ &rx_status) || rx_status,
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+ return -EIO;
+
+ return (rx_status == BHIE_RXVECSTATUS_STATUS_XFER_COMPL) ? 0 : -EIO;
+}
+EXPORT_SYMBOL(mhi_download_rddm_img);
+
+static int mhi_fw_load_amss(struct mhi_controller *mhi_cntrl,
+ const struct mhi_buf *mhi_buf)
+{
+ void __iomem *base = mhi_cntrl->bhie;
+ rwlock_t *pm_lock = &mhi_cntrl->pm_lock;
+ u32 tx_status;
+
+ read_lock_bh(pm_lock);
+ if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
+ read_unlock_bh(pm_lock);
+ return -EIO;
+ }
+
+ MHI_LOG("Starting BHIe Programming\n");
+
+ mhi_write_reg(mhi_cntrl, base, BHIE_TXVECADDR_HIGH_OFFS,
+ upper_32_bits(mhi_buf->dma_addr));
+
+ mhi_write_reg(mhi_cntrl, base, BHIE_TXVECADDR_LOW_OFFS,
+ lower_32_bits(mhi_buf->dma_addr));
+
+ mhi_write_reg(mhi_cntrl, base, BHIE_TXVECSIZE_OFFS, mhi_buf->len);
+
+ mhi_cntrl->sequence_id = prandom_u32() & BHIE_TXVECSTATUS_SEQNUM_BMSK;
+ mhi_write_reg_field(mhi_cntrl, base, BHIE_TXVECDB_OFFS,
+ BHIE_TXVECDB_SEQNUM_BMSK, BHIE_TXVECDB_SEQNUM_SHFT,
+ mhi_cntrl->sequence_id);
+ read_unlock_bh(pm_lock);
+
+ MHI_LOG("Upper:0x%x Lower:0x%x len:0x%zx sequence:%u\n",
+ upper_32_bits(mhi_buf->dma_addr),
+ lower_32_bits(mhi_buf->dma_addr),
+ mhi_buf->len, mhi_cntrl->sequence_id);
+ MHI_LOG("Waiting for image transfer completion\n");
+
+ /* waiting for image download completion */
+ wait_event_timeout(mhi_cntrl->state_event,
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) ||
+ mhi_read_reg_field(mhi_cntrl, base,
+ BHIE_TXVECSTATUS_OFFS,
+ BHIE_TXVECSTATUS_STATUS_BMSK,
+ BHIE_TXVECSTATUS_STATUS_SHFT,
+ &tx_status) || tx_status,
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+ return -EIO;
+
+ return (tx_status == BHIE_TXVECSTATUS_STATUS_XFER_COMPL) ? 0 : -EIO;
+}
+
+static int mhi_fw_load_sbl(struct mhi_controller *mhi_cntrl,
+ dma_addr_t dma_addr, size_t size)
+{
+ u32 tx_status, val;
+ int i, ret;
+ void __iomem *base = mhi_cntrl->bhi;
+ rwlock_t *pm_lock = &mhi_cntrl->pm_lock;
+ struct {
+ char *name;
+ u32 offset;
+ } error_reg[] = {
+ { "ERROR_CODE", BHI_ERRCODE },
+ { "ERROR_DBG1", BHI_ERRDBG1 },
+ { "ERROR_DBG2", BHI_ERRDBG2 },
+ { "ERROR_DBG3", BHI_ERRDBG3 },
+ { NULL },
+ };
+
+ MHI_LOG("Starting BHI programming\n");
+
+ /* program start sbl download via bhi protocol */
+ read_lock_bh(pm_lock);
+ if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
+ read_unlock_bh(pm_lock);
+ goto invalid_pm_state;
+ }
+
+ mhi_write_reg(mhi_cntrl, base, BHI_STATUS, 0);
+ mhi_write_reg(mhi_cntrl, base, BHI_IMGADDR_HIGH,
+ upper_32_bits(dma_addr));
+ mhi_write_reg(mhi_cntrl, base, BHI_IMGADDR_LOW,
+ lower_32_bits(dma_addr));
+ mhi_write_reg(mhi_cntrl, base, BHI_IMGSIZE, size);
+ mhi_cntrl->session_id = prandom_u32() & BHI_TXDB_SEQNUM_BMSK;
+ mhi_write_reg(mhi_cntrl, base, BHI_IMGTXDB, mhi_cntrl->session_id);
+ read_unlock_bh(pm_lock);
+
+ MHI_LOG("Waiting for image transfer completion\n");
+
+ /* waiting for image download completion */
+ wait_event_timeout(mhi_cntrl->state_event,
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) ||
+ mhi_read_reg_field(mhi_cntrl, base, BHI_STATUS,
+ BHI_STATUS_MASK, BHI_STATUS_SHIFT,
+ &tx_status) || tx_status,
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+ goto invalid_pm_state;
+
+ if (tx_status == BHI_STATUS_ERROR) {
+ MHI_ERR("Image transfer failed\n");
+ read_lock_bh(pm_lock);
+ if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
+ for (i = 0; error_reg[i].name; i++) {
+ ret = mhi_read_reg(mhi_cntrl, base,
+ error_reg[i].offset, &val);
+ if (ret)
+ break;
+ MHI_ERR("reg:%s value:0x%x\n",
+ error_reg[i].name, val);
+ }
+ }
+ read_unlock_bh(pm_lock);
+ goto invalid_pm_state;
+ }
+
+ return (tx_status == BHI_STATUS_SUCCESS) ? 0 : -ETIMEDOUT;
+
+invalid_pm_state:
+
+ return -EIO;
+}
+
+void mhi_free_bhie_table(struct mhi_controller *mhi_cntrl,
+ struct image_info *image_info)
+{
+ int i;
+ struct mhi_buf *mhi_buf = image_info->mhi_buf;
+
+ for (i = 0; i < image_info->entries; i++, mhi_buf++)
+ mhi_free_coherent(mhi_cntrl, mhi_buf->len, mhi_buf->buf,
+ mhi_buf->dma_addr);
+
+ kfree(image_info->mhi_buf);
+ kfree(image_info);
+}
+
+int mhi_alloc_bhie_table(struct mhi_controller *mhi_cntrl,
+ struct image_info **image_info, size_t alloc_size)
+{
+ size_t seg_size = mhi_cntrl->seg_len;
+ /* requier additional entry for vec table */
+ int segments = DIV_ROUND_UP(alloc_size, seg_size) + 1;
+ int i;
+ struct image_info *img_info;
+ struct mhi_buf *mhi_buf;
+
+ MHI_LOG("Allocating bytes:%zu seg_size:%zu total_seg:%u\n",
+ alloc_size, seg_size, segments);
+
+ img_info = kzalloc(sizeof(*img_info), GFP_KERNEL);
+ if (!img_info)
+ return -ENOMEM;
+
+ /* allocate memory for entries */
+ img_info->mhi_buf = kcalloc(segments, sizeof(*img_info->mhi_buf),
+ GFP_KERNEL);
+ if (!img_info->mhi_buf)
+ goto error_alloc_mhi_buf;
+
+ /* allocate and populate vector table */
+ mhi_buf = img_info->mhi_buf;
+ for (i = 0; i < segments; i++, mhi_buf++) {
+ size_t vec_size = seg_size;
+
+ /* last entry is for vector table */
+ if (i == segments - 1)
+ vec_size = sizeof(struct bhi_vec_entry) * i;
+
+ mhi_buf->len = vec_size;
+ mhi_buf->buf = mhi_alloc_coherent(mhi_cntrl, vec_size,
+ &mhi_buf->dma_addr,
+ GFP_KERNEL);
+ if (!mhi_buf->buf)
+ goto error_alloc_segment;
+
+ MHI_LOG("Entry:%d Address:0x%llx size:%zu\n", i,
+ (unsigned long long)mhi_buf->dma_addr, mhi_buf->len);
+ }
+
+ img_info->bhi_vec = img_info->mhi_buf[segments - 1].buf;
+ img_info->entries = segments;
+ *image_info = img_info;
+
+ MHI_LOG("Successfully allocated bhi vec table\n");
+
+ return 0;
+
+error_alloc_segment:
+ for (--i, --mhi_buf; i >= 0; i--, mhi_buf--)
+ mhi_free_coherent(mhi_cntrl, mhi_buf->len, mhi_buf->buf,
+ mhi_buf->dma_addr);
+
+error_alloc_mhi_buf:
+ kfree(img_info);
+
+ return -ENOMEM;
+}
+
+static void mhi_firmware_copy(struct mhi_controller *mhi_cntrl,
+ const struct firmware *firmware,
+ struct image_info *img_info)
+{
+ size_t remainder = firmware->size;
+ size_t to_cpy;
+ const u8 *buf = firmware->data;
+ int i = 0;
+ struct mhi_buf *mhi_buf = img_info->mhi_buf;
+ struct bhi_vec_entry *bhi_vec = img_info->bhi_vec;
+
+ while (remainder) {
+ MHI_ASSERT(i >= img_info->entries, "malformed vector table");
+
+ to_cpy = min(remainder, mhi_buf->len);
+ memcpy(mhi_buf->buf, buf, to_cpy);
+ bhi_vec->dma_addr = mhi_buf->dma_addr;
+ bhi_vec->size = to_cpy;
+
+ MHI_VERB("Setting Vector:0x%llx size: %llu\n",
+ bhi_vec->dma_addr, bhi_vec->size);
+ buf += to_cpy;
+ remainder -= to_cpy;
+ i++;
+ bhi_vec++;
+ mhi_buf++;
+ }
+}
+
+void mhi_fw_load_worker(struct work_struct *work)
+{
+ int ret;
+ struct mhi_controller *mhi_cntrl;
+ const char *fw_name;
+ const struct firmware *firmware;
+ struct image_info *image_info;
+ void *buf;
+ dma_addr_t dma_addr;
+ size_t size;
+
+ mhi_cntrl = container_of(work, struct mhi_controller, fw_worker);
+
+ MHI_LOG("Waiting for device to enter PBL from EE:%s\n",
+ TO_MHI_EXEC_STR(mhi_cntrl->ee));
+
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ MHI_IN_PBL(mhi_cntrl->ee) ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ MHI_ERR("MHI is not in valid state\n");
+ return;
+ }
+
+ MHI_LOG("Device current EE:%s\n", TO_MHI_EXEC_STR(mhi_cntrl->ee));
+
+ /* if device in pthru, we do not have to load firmware */
+ if (mhi_cntrl->ee == MHI_EE_PTHRU)
+ return;
+
+ fw_name = (mhi_cntrl->ee == MHI_EE_EDL) ?
+ mhi_cntrl->edl_image : mhi_cntrl->fw_image;
+
+ if (!fw_name || (mhi_cntrl->fbc_download && (!mhi_cntrl->sbl_size ||
+ !mhi_cntrl->seg_len))) {
+ MHI_ERR("No firmware image defined or !sbl_size || !seg_len\n");
+ return;
+ }
+
+ ret = request_firmware(&firmware, fw_name, mhi_cntrl->dev);
+ if (ret) {
+ MHI_ERR("Error loading firmware, ret:%d\n", ret);
+ return;
+ }
+
+ size = (mhi_cntrl->fbc_download) ? mhi_cntrl->sbl_size : firmware->size;
+
+ /* the sbl size provided is maximum size, not necessarily image size */
+ if (size > firmware->size)
+ size = firmware->size;
+
+ buf = mhi_alloc_coherent(mhi_cntrl, size, &dma_addr, GFP_KERNEL);
+ if (!buf) {
+ MHI_ERR("Could not allocate memory for image\n");
+ release_firmware(firmware);
+ return;
+ }
+
+ /* load sbl image */
+ memcpy(buf, firmware->data, size);
+ ret = mhi_fw_load_sbl(mhi_cntrl, dma_addr, size);
+ mhi_free_coherent(mhi_cntrl, size, buf, dma_addr);
+
+ if (!mhi_cntrl->fbc_download || ret || mhi_cntrl->ee == MHI_EE_EDL)
+ release_firmware(firmware);
+
+ /* error or in edl, we're done */
+ if (ret || mhi_cntrl->ee == MHI_EE_EDL)
+ return;
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ mhi_cntrl->dev_state = MHI_STATE_RESET;
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+
+ /*
+ * if we're doing fbc, populate vector tables while
+ * device transitioning into MHI READY state
+ */
+ if (mhi_cntrl->fbc_download) {
+ ret = mhi_alloc_bhie_table(mhi_cntrl, &mhi_cntrl->fbc_image,
+ firmware->size);
+ if (ret) {
+ MHI_ERR("Error alloc size of %zu\n", firmware->size);
+ goto error_alloc_fw_table;
+ }
+
+ MHI_LOG("Copying firmware image into vector table\n");
+
+ /* load the firmware into BHIE vec table */
+ mhi_firmware_copy(mhi_cntrl, firmware, mhi_cntrl->fbc_image);
+ }
+
+ /* transitioning into MHI RESET->READY state */
+ ret = mhi_ready_state_transition(mhi_cntrl);
+
+ MHI_LOG("To Reset->Ready PM_STATE:%s MHI_STATE:%s EE:%s, ret:%d\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ TO_MHI_EXEC_STR(mhi_cntrl->ee), ret);
+
+ if (!mhi_cntrl->fbc_download)
+ return;
+
+ if (ret) {
+ MHI_ERR("Did not transition to READY state\n");
+ goto error_read;
+ }
+
+ /* wait for SBL event */
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->ee == MHI_EE_SBL ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ MHI_ERR("MHI did not enter BHIE\n");
+ goto error_read;
+ }
+
+ /* start full firmware image download */
+ image_info = mhi_cntrl->fbc_image;
+ ret = mhi_fw_load_amss(mhi_cntrl,
+ /* last entry is vec table */
+ &image_info->mhi_buf[image_info->entries - 1]);
+
+ MHI_LOG("amss fw_load, ret:%d\n", ret);
+
+ release_firmware(firmware);
+
+ return;
+
+error_read:
+ mhi_free_bhie_table(mhi_cntrl, mhi_cntrl->fbc_image);
+ mhi_cntrl->fbc_image = NULL;
+
+error_alloc_fw_table:
+ release_firmware(firmware);
+}
diff --git a/drivers/bus/mhi/core/mhi_dtr.c b/drivers/bus/mhi/core/mhi_dtr.c
new file mode 100644
index 000000000000..e6f07d4cc329
--- /dev/null
+++ b/drivers/bus/mhi/core/mhi_dtr.c
@@ -0,0 +1,223 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.*/
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/of.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/termios.h>
+#include <linux/wait.h>
+#include <linux/mhi.h>
+#include "mhi_internal.h"
+
+struct __packed dtr_ctrl_msg {
+ u32 preamble;
+ u32 msg_id;
+ u32 dest_id;
+ u32 size;
+ u32 msg;
+};
+
+#define CTRL_MAGIC (0x4C525443)
+#define CTRL_MSG_DTR BIT(0)
+#define CTRL_MSG_RTS BIT(1)
+#define CTRL_MSG_DCD BIT(0)
+#define CTRL_MSG_DSR BIT(1)
+#define CTRL_MSG_RI BIT(3)
+#define CTRL_HOST_STATE (0x10)
+#define CTRL_DEVICE_STATE (0x11)
+#define CTRL_GET_CHID(dtr) (dtr->dest_id & 0xFF)
+
+static int mhi_dtr_tiocmset(struct mhi_controller *mhi_cntrl,
+ struct mhi_device *mhi_dev, u32 tiocm)
+{
+ struct dtr_ctrl_msg *dtr_msg = NULL;
+ struct mhi_chan *dtr_chan = mhi_cntrl->dtr_dev->ul_chan;
+ spinlock_t *res_lock = &mhi_dev->dev.devres_lock;
+ u32 cur_tiocm;
+ int ret = 0;
+
+ cur_tiocm = mhi_dev->tiocm & ~(TIOCM_CD | TIOCM_DSR | TIOCM_RI);
+
+ tiocm &= (TIOCM_DTR | TIOCM_RTS);
+
+ /* state did not changed */
+ if (cur_tiocm == tiocm)
+ return 0;
+
+ mutex_lock(&dtr_chan->mutex);
+
+ dtr_msg = kzalloc(sizeof(*dtr_msg), GFP_KERNEL);
+ if (!dtr_msg) {
+ ret = -ENOMEM;
+ goto tiocm_exit;
+ }
+
+ dtr_msg->preamble = CTRL_MAGIC;
+ dtr_msg->msg_id = CTRL_HOST_STATE;
+ dtr_msg->dest_id = mhi_dev->ul_chan_id;
+ dtr_msg->size = sizeof(u32);
+ if (tiocm & TIOCM_DTR)
+ dtr_msg->msg |= CTRL_MSG_DTR;
+ if (tiocm & TIOCM_RTS)
+ dtr_msg->msg |= CTRL_MSG_RTS;
+
+ reinit_completion(&dtr_chan->completion);
+ ret = mhi_queue_transfer(mhi_cntrl->dtr_dev, DMA_TO_DEVICE, dtr_msg,
+ sizeof(*dtr_msg), MHI_EOT);
+ if (ret)
+ goto tiocm_exit;
+
+ ret = wait_for_completion_timeout(&dtr_chan->completion,
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+ if (!ret) {
+ MHI_ERR("Failed to receive transfer callback\n");
+ ret = -EIO;
+ goto tiocm_exit;
+ }
+
+ ret = 0;
+ spin_lock_irq(res_lock);
+ mhi_dev->tiocm &= ~(TIOCM_DTR | TIOCM_RTS);
+ mhi_dev->tiocm |= tiocm;
+ spin_unlock_irq(res_lock);
+
+tiocm_exit:
+ kfree(dtr_msg);
+ mutex_unlock(&dtr_chan->mutex);
+
+ return ret;
+}
+
+long mhi_ioctl(struct mhi_device *mhi_dev, unsigned int cmd, unsigned long arg)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ int ret;
+
+ /* ioctl not supported by this controller */
+ if (!mhi_cntrl->dtr_dev)
+ return -EIO;
+
+ switch (cmd) {
+ case TIOCMGET:
+ return mhi_dev->tiocm;
+ case TIOCMSET:
+ {
+ u32 tiocm;
+
+ ret = get_user(tiocm, (u32 *) arg);
+ if (ret)
+ return ret;
+
+ return mhi_dtr_tiocmset(mhi_cntrl, mhi_dev, tiocm);
+ }
+ default:
+ break;
+ }
+
+ return -EINVAL;
+}
+EXPORT_SYMBOL(mhi_ioctl);
+
+static void mhi_dtr_dl_xfer_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct dtr_ctrl_msg *dtr_msg = mhi_result->buf_addr;
+ u32 chan;
+ spinlock_t *res_lock;
+
+ if (mhi_result->bytes_xferd != sizeof(*dtr_msg)) {
+ MHI_ERR("Unexpected length %zu received\n",
+ mhi_result->bytes_xferd);
+ return;
+ }
+
+ MHI_VERB("preamble:0x%x msg_id:%u dest_id:%u msg:0x%x\n",
+ dtr_msg->preamble, dtr_msg->msg_id, dtr_msg->dest_id,
+ dtr_msg->msg);
+
+ chan = CTRL_GET_CHID(dtr_msg);
+ if (chan >= mhi_cntrl->max_chan)
+ return;
+
+ mhi_dev = mhi_cntrl->mhi_chan[chan].mhi_dev;
+ if (!mhi_dev)
+ return;
+
+ res_lock = &mhi_dev->dev.devres_lock;
+ spin_lock_irq(res_lock);
+ mhi_dev->tiocm &= ~(TIOCM_CD | TIOCM_DSR | TIOCM_RI);
+
+ if (dtr_msg->msg & CTRL_MSG_DCD)
+ mhi_dev->tiocm |= TIOCM_CD;
+
+ if (dtr_msg->msg & CTRL_MSG_DSR)
+ mhi_dev->tiocm |= TIOCM_DSR;
+
+ if (dtr_msg->msg & CTRL_MSG_RI)
+ mhi_dev->tiocm |= TIOCM_RI;
+ spin_unlock_irq(res_lock);
+}
+
+static void mhi_dtr_ul_xfer_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_chan *dtr_chan = mhi_cntrl->dtr_dev->ul_chan;
+
+ MHI_VERB("Received with status:%d\n", mhi_result->transaction_status);
+ if (!mhi_result->transaction_status)
+ complete(&dtr_chan->completion);
+}
+
+static void mhi_dtr_remove(struct mhi_device *mhi_dev)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+
+ mhi_cntrl->dtr_dev = NULL;
+}
+
+static int mhi_dtr_probe(struct mhi_device *mhi_dev,
+ const struct mhi_device_id *id)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ int ret;
+
+ MHI_LOG("Enter for DTR control channel\n");
+
+ ret = mhi_prepare_for_transfer(mhi_dev);
+ if (!ret)
+ mhi_cntrl->dtr_dev = mhi_dev;
+
+ MHI_LOG("Exit with ret:%d\n", ret);
+
+ return ret;
+}
+
+static const struct mhi_device_id mhi_dtr_table[] = {
+ { .chan = "IP_CTRL" },
+ { },
+};
+
+static struct mhi_driver mhi_dtr_driver = {
+ .id_table = mhi_dtr_table,
+ .remove = mhi_dtr_remove,
+ .probe = mhi_dtr_probe,
+ .ul_xfer_cb = mhi_dtr_ul_xfer_cb,
+ .dl_xfer_cb = mhi_dtr_dl_xfer_cb,
+ .driver = {
+ .name = "MHI_DTR",
+ .owner = THIS_MODULE,
+ }
+};
+
+int __init mhi_dtr_init(void)
+{
+ return mhi_driver_register(&mhi_dtr_driver);
+}
diff --git a/drivers/bus/mhi/core/mhi_init.c b/drivers/bus/mhi/core/mhi_init.c
new file mode 100644
index 000000000000..ec6103eb66ed
--- /dev/null
+++ b/drivers/bus/mhi/core/mhi_init.c
@@ -0,0 +1,1901 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. */
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/of.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/wait.h>
+#include <linux/mhi.h>
+#include "mhi_internal.h"
+
+const char *const mhi_ee_str[MHI_EE_MAX] = {
+ [MHI_EE_PBL] = "PBL",
+ [MHI_EE_SBL] = "SBL",
+ [MHI_EE_AMSS] = "AMSS",
+ [MHI_EE_RDDM] = "RDDM",
+ [MHI_EE_WFW] = "WFW",
+ [MHI_EE_PTHRU] = "PASS THRU",
+ [MHI_EE_EDL] = "EDL",
+ [MHI_EE_DISABLE_TRANSITION] = "DISABLE",
+};
+
+const char *const mhi_state_tran_str[MHI_ST_TRANSITION_MAX] = {
+ [MHI_ST_TRANSITION_PBL] = "PBL",
+ [MHI_ST_TRANSITION_READY] = "READY",
+ [MHI_ST_TRANSITION_SBL] = "SBL",
+ [MHI_ST_TRANSITION_MISSION_MODE] = "MISSION MODE",
+};
+
+const char *const mhi_state_str[MHI_STATE_MAX] = {
+ [MHI_STATE_RESET] = "RESET",
+ [MHI_STATE_READY] = "READY",
+ [MHI_STATE_M0] = "M0",
+ [MHI_STATE_M1] = "M1",
+ [MHI_STATE_M2] = "M2",
+ [MHI_STATE_M3] = "M3",
+ [MHI_STATE_BHI] = "BHI",
+ [MHI_STATE_SYS_ERR] = "SYS_ERR",
+};
+
+static const char *const mhi_pm_state_str[] = {
+ [MHI_PM_BIT_DISABLE] = "DISABLE",
+ [MHI_PM_BIT_POR] = "POR",
+ [MHI_PM_BIT_M0] = "M0",
+ [MHI_PM_BIT_M2] = "M2",
+ [MHI_PM_BIT_M3_ENTER] = "M?->M3",
+ [MHI_PM_BIT_M3] = "M3",
+ [MHI_PM_BIT_M3_EXIT] = "M3->M0",
+ [MHI_PM_BIT_FW_DL_ERR] = "FW DL Error",
+ [MHI_PM_BIT_SYS_ERR_DETECT] = "SYS_ERR Detect",
+ [MHI_PM_BIT_SYS_ERR_PROCESS] = "SYS_ERR Process",
+ [MHI_PM_BIT_SHUTDOWN_PROCESS] = "SHUTDOWN Process",
+ [MHI_PM_BIT_LD_ERR_FATAL_DETECT] = "LD or Error Fatal Detect",
+};
+
+#ifndef CONFIG_OF
+struct mhi_chan_init {
+ u32 chan;
+ const char *name;
+ size_t tre_ring_elements;
+ size_t buf_ring_elements;
+ u32 er_index;
+ enum dma_data_direction dir;
+ u32 ee_mask;
+ u32 pollcfg;
+ enum MHI_XFER_TYPE xfer_type;
+ bool lpm_notify;
+ bool offload_ch;
+ bool reset_req;
+ bool pre_alloc;
+ bool auto_start;
+ bool wake_capable;
+ enum MHI_BRSTMODE brstmode;
+};
+
+struct mhi_chan_init mhi_chan_init_array[] = {
+ { 0, "LOOPBACK", 64, 0, 2, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 1, "LOOPBACK", 64, 0, 2, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 4, "DIAG", 64, 0, 1, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 5, "DIAG", 64, 0, 3, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+#ifdef CONFIG_MHI_MBIM
+ { 12, "MBIM", 64, 0, 3, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 13, "MBIM", 64, 0, 3, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+#endif
+ { 14, "QMI0", 64, 0, 1, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 15, "QMI0", 64, 0, 2, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 16, "QMI1", 64, 0, 3, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 17, "QMI1", 64, 0, 3, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 18, "IP_CTRL", 64, 0, 1, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 19, "IP_CTRL", 64, 0, 1, 2, 4, 0, 0, 0, 0, 0, 1, 0, 0, 2 },
+ { 20, "IPCR", 64, 0, 2, 1, 4, 0, 1, 0, 0, 0, 0, 1, 0, 2 },
+ { 21, "IPCR", 64, 0, 2, 2, 4, 0, 0, 0, 0, 0, 1, 1, 0, 2 },
+ { 32, "AT", 64, 0, 3, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 33, "AT", 64, 0, 3, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
+ { 46, "IP_SW0", 512, 0, 4, 1, 4, 0, 1, 0, 0, 0, 0, 0, 0, 2 },
+ { 47, "IP_SW0", 512, 0, 5, 2, 4, 0, 4, 0, 0, 0, 0, 0, 0, 2 },
+ { 100, "IP_HW0", 512, 0, 6, 1, 4, 0, 1, 0, 0, 1, 0, 0, 0, 3 },
+ { 101, "IP_HW0", 512, 0, 7, 2, 4, 0, 4, 0, 0, 0, 0, 0, 0, 3 },
+ { 102, "IP_HW_ADPL", 0, 0, 8, 2, 4, 0, 3, 1, 1, 0, 0, 0, 0, 0 },
+};
+
+struct mhi_event_init {
+ size_t elements;
+ u32 intmod;
+ u32 msi;
+ int chan;
+ u32 priority;
+ enum MHI_BRSTMODE brstmode;
+ enum mhi_er_data_type data_type;
+ bool hw_ring;
+ bool cl_manage;
+ bool offload_ev;
+};
+
+struct mhi_event_init mhi_event_init_array[] = {
+ { 32, 1, 1, 0, 1, 2, 1, 0, 0, 0 },
+ { 256, 1, 2, 0, 1, 2, 0, 0, 0, 0 },
+ { 256, 1, 3, 0, 1, 2, 0, 0, 0, 0 },
+ { 256, 1, 4, 0, 1, 2, 0, 0, 0, 0 },
+ { 1024, 5, 5, 46, 1, 2, 0, 0, 0, 0 },
+ { 1024, 5, 6, 47, 1, 2, 0, 0, 1, 0 },
+ { 1024, 5, 5, 100, 1, 3, 0, 1, 0, 0 },
+ { 1024, 5, 6, 101, 1, 3, 0, 1, 1, 0 },
+ { 0, 0, 0, 102, 1, 3, 0, 1, 1, 1 },
+ { 1024, 5, 7, 103, 1, 2, 0, 1, 0, 0 },
+ { 0, 0, 0, 105, 1, 3, 0, 1, 1, 1 },
+ { 0, 0, 0, 106, 1, 3, 0, 1, 1, 1 },
+ { 0, 0, 0, 107, 1, 3, 0, 1, 1, 1 },
+ { 0, 0, 0, 108, 1, 3, 0, 1, 1, 1 },
+};
+#endif
+
+struct mhi_bus mhi_bus;
+
+const char *to_mhi_pm_state_str(enum MHI_PM_STATE state)
+{
+ int index = find_last_bit((unsigned long *)&state, 32);
+
+ if (index >= ARRAY_SIZE(mhi_pm_state_str))
+ return "Invalid State";
+
+ return mhi_pm_state_str[index];
+}
+
+/* MHI protocol require transfer ring to be aligned to ring length */
+static int mhi_alloc_aligned_ring(struct mhi_controller *mhi_cntrl,
+ struct mhi_ring *ring, u64 len)
+{
+ ring->alloc_size = len + (len - 1);
+ ring->pre_aligned = mhi_alloc_coherent(mhi_cntrl, ring->alloc_size,
+ &ring->dma_handle, GFP_KERNEL);
+ if (!ring->pre_aligned)
+ return -ENOMEM;
+
+ ring->iommu_base = (ring->dma_handle + (len - 1)) & ~(len - 1);
+ ring->base = ring->pre_aligned + (ring->iommu_base - ring->dma_handle);
+ return 0;
+}
+
+void mhi_deinit_free_irq(struct mhi_controller *mhi_cntrl)
+{
+ int i;
+ struct mhi_event *mhi_event = mhi_cntrl->mhi_event;
+
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
+ if (mhi_event->offload_ev)
+ continue;
+
+ free_irq(mhi_cntrl->irq[mhi_event->msi], mhi_event);
+ }
+
+ free_irq(mhi_cntrl->irq[0], mhi_cntrl);
+}
+
+int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl)
+{
+ int i;
+ int ret;
+ struct mhi_event *mhi_event = mhi_cntrl->mhi_event;
+ unsigned long irqflags =
+ (mhi_cntrl->msi_allocated > 1) ? IRQF_ONESHOT : IRQF_SHARED;
+
+ /* for BHI INTVEC msi */
+ ret = request_threaded_irq(mhi_cntrl->irq[0], mhi_intvec_handlr,
+ mhi_intvec_threaded_handlr, irqflags,
+ "mhi", mhi_cntrl);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
+ if (mhi_event->offload_ev)
+ continue;
+
+ if (mhi_cntrl->msi_allocated == 1)
+ mhi_event->msi = 0;
+
+ ret = request_irq(mhi_cntrl->irq[mhi_event->msi],
+ mhi_msi_handlr, IRQF_SHARED, "mhi",
+ mhi_event);
+ if (ret) {
+ MHI_ERR("Error requesting irq:%d for ev:%d\n",
+ mhi_cntrl->irq[mhi_event->msi], i);
+ goto error_request;
+ }
+ }
+
+ return 0;
+
+error_request:
+ for (--i, --mhi_event; i >= 0; i--, mhi_event--) {
+ if (mhi_event->offload_ev)
+ continue;
+
+ free_irq(mhi_cntrl->irq[mhi_event->msi], mhi_event);
+ }
+ free_irq(mhi_cntrl->irq[0], mhi_cntrl);
+
+ return ret;
+}
+
+void mhi_deinit_dev_ctxt(struct mhi_controller *mhi_cntrl)
+{
+ int i;
+ struct mhi_ctxt *mhi_ctxt = mhi_cntrl->mhi_ctxt;
+ struct mhi_cmd *mhi_cmd;
+ struct mhi_event *mhi_event;
+ struct mhi_ring *ring;
+
+ mhi_cmd = mhi_cntrl->mhi_cmd;
+ for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++) {
+ ring = &mhi_cmd->ring;
+ mhi_free_coherent(mhi_cntrl, ring->alloc_size,
+ ring->pre_aligned, ring->dma_handle);
+ ring->base = NULL;
+ ring->iommu_base = 0;
+ }
+
+ mhi_free_coherent(mhi_cntrl,
+ sizeof(*mhi_ctxt->cmd_ctxt) * NR_OF_CMD_RINGS,
+ mhi_ctxt->cmd_ctxt, mhi_ctxt->cmd_ctxt_addr);
+
+ mhi_event = mhi_cntrl->mhi_event;
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
+ if (mhi_event->offload_ev)
+ continue;
+
+ ring = &mhi_event->ring;
+ mhi_free_coherent(mhi_cntrl, ring->alloc_size,
+ ring->pre_aligned, ring->dma_handle);
+ ring->base = NULL;
+ ring->iommu_base = 0;
+ }
+
+ mhi_free_coherent(mhi_cntrl, sizeof(*mhi_ctxt->er_ctxt) *
+ mhi_cntrl->total_ev_rings, mhi_ctxt->er_ctxt,
+ mhi_ctxt->er_ctxt_addr);
+
+ mhi_free_coherent(mhi_cntrl, sizeof(*mhi_ctxt->chan_ctxt) *
+ mhi_cntrl->max_chan, mhi_ctxt->chan_ctxt,
+ mhi_ctxt->chan_ctxt_addr);
+
+ kfree(mhi_ctxt);
+ mhi_cntrl->mhi_ctxt = NULL;
+}
+
+static int mhi_init_debugfs_mhi_states_open(struct inode *inode,
+ struct file *fp)
+{
+ return single_open(fp, mhi_debugfs_mhi_states_show, inode->i_private);
+}
+
+static int mhi_init_debugfs_mhi_event_open(struct inode *inode, struct file *fp)
+{
+ return single_open(fp, mhi_debugfs_mhi_event_show, inode->i_private);
+}
+
+static int mhi_init_debugfs_mhi_chan_open(struct inode *inode, struct file *fp)
+{
+ return single_open(fp, mhi_debugfs_mhi_chan_show, inode->i_private);
+}
+
+static const struct file_operations debugfs_state_ops = {
+ .open = mhi_init_debugfs_mhi_states_open,
+ .release = single_release,
+ .read = seq_read,
+};
+
+static const struct file_operations debugfs_ev_ops = {
+ .open = mhi_init_debugfs_mhi_event_open,
+ .release = single_release,
+ .read = seq_read,
+};
+
+static const struct file_operations debugfs_chan_ops = {
+ .open = mhi_init_debugfs_mhi_chan_open,
+ .release = single_release,
+ .read = seq_read,
+};
+
+DEFINE_SIMPLE_ATTRIBUTE(debugfs_trigger_reset_fops, NULL,
+ mhi_debugfs_trigger_reset, "%llu\n");
+
+void mhi_init_debugfs(struct mhi_controller *mhi_cntrl)
+{
+ struct dentry *dentry;
+ char node[32];
+
+ if (!mhi_cntrl->parent)
+ return;
+
+ snprintf(node, sizeof(node), "%04x_%02u:%02u.%02u",
+ mhi_cntrl->dev_id, mhi_cntrl->domain, mhi_cntrl->bus,
+ mhi_cntrl->slot);
+
+ dentry = debugfs_create_dir(node, mhi_cntrl->parent);
+ if (IS_ERR_OR_NULL(dentry))
+ return;
+
+ debugfs_create_file("states", 0444, dentry, mhi_cntrl,
+ &debugfs_state_ops);
+ debugfs_create_file("events", 0444, dentry, mhi_cntrl, &debugfs_ev_ops);
+ debugfs_create_file("chan", 0444, dentry, mhi_cntrl, &debugfs_chan_ops);
+ debugfs_create_file("reset", 0444, dentry, mhi_cntrl,
+ &debugfs_trigger_reset_fops);
+ mhi_cntrl->dentry = dentry;
+}
+
+void mhi_deinit_debugfs(struct mhi_controller *mhi_cntrl)
+{
+ debugfs_remove_recursive(mhi_cntrl->dentry);
+ mhi_cntrl->dentry = NULL;
+}
+
+int mhi_init_dev_ctxt(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_ctxt *mhi_ctxt;
+ struct mhi_chan_ctxt *chan_ctxt;
+ struct mhi_event_ctxt *er_ctxt;
+ struct mhi_cmd_ctxt *cmd_ctxt;
+ struct mhi_chan *mhi_chan;
+ struct mhi_event *mhi_event;
+ struct mhi_cmd *mhi_cmd;
+ int ret = -ENOMEM, i;
+
+ atomic_set(&mhi_cntrl->dev_wake, 0);
+ atomic_set(&mhi_cntrl->alloc_size, 0);
+
+ mhi_ctxt = kzalloc(sizeof(*mhi_ctxt), GFP_KERNEL);
+ if (!mhi_ctxt)
+ return -ENOMEM;
+
+ /* setup channel ctxt */
+ mhi_ctxt->chan_ctxt = mhi_alloc_coherent(mhi_cntrl,
+ sizeof(*mhi_ctxt->chan_ctxt) * mhi_cntrl->max_chan,
+ &mhi_ctxt->chan_ctxt_addr, GFP_KERNEL);
+ if (!mhi_ctxt->chan_ctxt)
+ goto error_alloc_chan_ctxt;
+
+ mhi_chan = mhi_cntrl->mhi_chan;
+ chan_ctxt = mhi_ctxt->chan_ctxt;
+ for (i = 0; i < mhi_cntrl->max_chan; i++, chan_ctxt++, mhi_chan++) {
+ /* If it's offload channel skip this step */
+ if (mhi_chan->offload_ch)
+ continue;
+
+ chan_ctxt->chstate = MHI_CH_STATE_DISABLED;
+ chan_ctxt->brstmode = mhi_chan->db_cfg.brstmode;
+ chan_ctxt->pollcfg = mhi_chan->db_cfg.pollcfg;
+ chan_ctxt->chtype = mhi_chan->type;
+ chan_ctxt->erindex = mhi_chan->er_index;
+
+ mhi_chan->ch_state = MHI_CH_STATE_DISABLED;
+ mhi_chan->tre_ring.db_addr = &chan_ctxt->wp;
+ }
+
+ /* setup event context */
+ mhi_ctxt->er_ctxt = mhi_alloc_coherent(mhi_cntrl,
+ sizeof(*mhi_ctxt->er_ctxt) * mhi_cntrl->total_ev_rings,
+ &mhi_ctxt->er_ctxt_addr, GFP_KERNEL);
+ if (!mhi_ctxt->er_ctxt)
+ goto error_alloc_er_ctxt;
+
+ er_ctxt = mhi_ctxt->er_ctxt;
+ mhi_event = mhi_cntrl->mhi_event;
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++,
+ er_ctxt++, mhi_event++) {
+ struct mhi_ring *ring = &mhi_event->ring;
+
+ /* it's a satellite ev, we do not touch it */
+ if (mhi_event->offload_ev)
+ continue;
+
+ if (mhi_cntrl->msi_allocated == 1)
+ mhi_event->msi = 0;
+
+ er_ctxt->intmodc = 0;
+ er_ctxt->intmodt = mhi_event->intmod;
+ er_ctxt->ertype = MHI_ER_TYPE_VALID;
+ er_ctxt->msivec = mhi_event->msi;
+ mhi_event->db_cfg.db_mode = true;
+
+ ring->el_size = sizeof(struct mhi_tre);
+ ring->len = ring->el_size * ring->elements;
+ ret = mhi_alloc_aligned_ring(mhi_cntrl, ring, ring->len);
+ if (ret)
+ goto error_alloc_er;
+
+ ring->rptr = ring->wptr = ring->base;
+ er_ctxt->rbase = ring->iommu_base;
+ er_ctxt->rp = er_ctxt->wp = er_ctxt->rbase;
+ er_ctxt->rlen = ring->len;
+ ring->ctxt_wp = &er_ctxt->wp;
+ }
+
+ /* setup cmd context */
+ mhi_ctxt->cmd_ctxt = mhi_alloc_coherent(mhi_cntrl,
+ sizeof(*mhi_ctxt->cmd_ctxt) * NR_OF_CMD_RINGS,
+ &mhi_ctxt->cmd_ctxt_addr, GFP_KERNEL);
+ if (!mhi_ctxt->cmd_ctxt)
+ goto error_alloc_er;
+
+ mhi_cmd = mhi_cntrl->mhi_cmd;
+ cmd_ctxt = mhi_ctxt->cmd_ctxt;
+ for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++, cmd_ctxt++) {
+ struct mhi_ring *ring = &mhi_cmd->ring;
+
+ ring->el_size = sizeof(struct mhi_tre);
+ ring->elements = CMD_EL_PER_RING;
+ ring->len = ring->el_size * ring->elements;
+ ret = mhi_alloc_aligned_ring(mhi_cntrl, ring, ring->len);
+ if (ret)
+ goto error_alloc_cmd;
+
+ ring->rptr = ring->wptr = ring->base;
+ cmd_ctxt->rbase = ring->iommu_base;
+ cmd_ctxt->rp = cmd_ctxt->wp = cmd_ctxt->rbase;
+ cmd_ctxt->rlen = ring->len;
+ ring->ctxt_wp = &cmd_ctxt->wp;
+ }
+
+ mhi_cntrl->mhi_ctxt = mhi_ctxt;
+
+ return 0;
+
+error_alloc_cmd:
+ for (--i, --mhi_cmd; i >= 0; i--, mhi_cmd--) {
+ struct mhi_ring *ring = &mhi_cmd->ring;
+
+ mhi_free_coherent(mhi_cntrl, ring->alloc_size,
+ ring->pre_aligned, ring->dma_handle);
+ }
+ mhi_free_coherent(mhi_cntrl,
+ sizeof(*mhi_ctxt->cmd_ctxt) * NR_OF_CMD_RINGS,
+ mhi_ctxt->cmd_ctxt, mhi_ctxt->cmd_ctxt_addr);
+ i = mhi_cntrl->total_ev_rings;
+ mhi_event = mhi_cntrl->mhi_event + i;
+
+error_alloc_er:
+ for (--i, --mhi_event; i >= 0; i--, mhi_event--) {
+ struct mhi_ring *ring = &mhi_event->ring;
+
+ if (mhi_event->offload_ev)
+ continue;
+
+ mhi_free_coherent(mhi_cntrl, ring->alloc_size,
+ ring->pre_aligned, ring->dma_handle);
+ }
+ mhi_free_coherent(mhi_cntrl,
+ sizeof(*mhi_ctxt->er_ctxt) * mhi_cntrl->total_ev_rings,
+ mhi_ctxt->er_ctxt, mhi_ctxt->er_ctxt_addr);
+
+error_alloc_er_ctxt:
+ mhi_free_coherent(mhi_cntrl,
+ sizeof(*mhi_ctxt->chan_ctxt) * mhi_cntrl->max_chan,
+ mhi_ctxt->chan_ctxt, mhi_ctxt->chan_ctxt_addr);
+
+error_alloc_chan_ctxt:
+ kfree(mhi_ctxt);
+
+ return ret;
+}
+
+static int mhi_get_tsync_er_cfg(struct mhi_controller *mhi_cntrl)
+{
+ int i;
+ struct mhi_event *mhi_event = mhi_cntrl->mhi_event;
+
+ /* find event ring with timesync support */
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++)
+ if (mhi_event->data_type == MHI_ER_TSYNC_ELEMENT_TYPE)
+ return mhi_event->er_index;
+
+ return -ENOENT;
+}
+
+int mhi_init_timesync(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_timesync *mhi_tsync;
+ u32 time_offset, db_offset;
+ int ret;
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+
+ if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
+ ret = -EIO;
+ goto exit_timesync;
+ }
+
+ ret = mhi_get_capability_offset(mhi_cntrl, TIMESYNC_CAP_ID,
+ &time_offset);
+ if (ret) {
+ MHI_LOG("No timesync capability found\n");
+ goto exit_timesync;
+ }
+
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ if (!mhi_cntrl->time_get || !mhi_cntrl->lpm_disable ||
+ !mhi_cntrl->lpm_enable)
+ return -EINVAL;
+
+ /* register method supported */
+ mhi_tsync = kzalloc(sizeof(*mhi_tsync), GFP_KERNEL);
+ if (!mhi_tsync)
+ return -ENOMEM;
+
+ spin_lock_init(&mhi_tsync->lock);
+ INIT_LIST_HEAD(&mhi_tsync->head);
+ init_completion(&mhi_tsync->completion);
+
+ /* save time_offset for obtaining time */
+ MHI_LOG("TIME OFFS:0x%x\n", time_offset);
+ mhi_tsync->time_reg = mhi_cntrl->regs + time_offset
+ + TIMESYNC_TIME_LOW_OFFSET;
+
+ mhi_cntrl->mhi_tsync = mhi_tsync;
+
+ ret = mhi_create_timesync_sysfs(mhi_cntrl);
+ if (unlikely(ret)) {
+ /* kernel method still work */
+ MHI_ERR("Failed to create timesync sysfs nodes\n");
+ }
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+
+ if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
+ ret = -EIO;
+ goto exit_timesync;
+ }
+
+ /* get DB offset if supported, else return */
+ ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs,
+ time_offset + TIMESYNC_DB_OFFSET, &db_offset);
+ if (ret || !db_offset) {
+ ret = 0;
+ goto exit_timesync;
+ }
+
+ MHI_LOG("TIMESYNC_DB OFFS:0x%x\n", db_offset);
+ mhi_tsync->db = mhi_cntrl->regs + db_offset;
+
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ /* get time-sync event ring configuration */
+ ret = mhi_get_tsync_er_cfg(mhi_cntrl);
+ if (ret < 0) {
+ MHI_LOG("Could not find timesync event ring\n");
+ return ret;
+ }
+
+ mhi_tsync->er_index = ret;
+
+ ret = mhi_send_cmd(mhi_cntrl, NULL, MHI_CMD_TIMSYNC_CFG);
+ if (ret) {
+ MHI_ERR("Failed to send time sync cfg cmd\n");
+ return ret;
+ }
+
+ ret = wait_for_completion_timeout(&mhi_tsync->completion,
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (!ret || mhi_tsync->ccs != MHI_EV_CC_SUCCESS) {
+ MHI_ERR("Failed to get time cfg cmd completion\n");
+ return -EIO;
+ }
+
+ return 0;
+
+exit_timesync:
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return ret;
+}
+
+int mhi_init_mmio(struct mhi_controller *mhi_cntrl)
+{
+ u32 val;
+ int i, ret;
+ struct mhi_chan *mhi_chan;
+ struct mhi_event *mhi_event;
+ void __iomem *base = mhi_cntrl->regs;
+ struct {
+ u32 offset;
+ u32 mask;
+ u32 shift;
+ u32 val;
+ } reg_info[] = {
+ {
+ CCABAP_HIGHER, U32_MAX, 0,
+ upper_32_bits(mhi_cntrl->mhi_ctxt->chan_ctxt_addr),
+ },
+ {
+ CCABAP_LOWER, U32_MAX, 0,
+ lower_32_bits(mhi_cntrl->mhi_ctxt->chan_ctxt_addr),
+ },
+ {
+ ECABAP_HIGHER, U32_MAX, 0,
+ upper_32_bits(mhi_cntrl->mhi_ctxt->er_ctxt_addr),
+ },
+ {
+ ECABAP_LOWER, U32_MAX, 0,
+ lower_32_bits(mhi_cntrl->mhi_ctxt->er_ctxt_addr),
+ },
+ {
+ CRCBAP_HIGHER, U32_MAX, 0,
+ upper_32_bits(mhi_cntrl->mhi_ctxt->cmd_ctxt_addr),
+ },
+ {
+ CRCBAP_LOWER, U32_MAX, 0,
+ lower_32_bits(mhi_cntrl->mhi_ctxt->cmd_ctxt_addr),
+ },
+ {
+ MHICFG, MHICFG_NER_MASK, MHICFG_NER_SHIFT,
+ mhi_cntrl->total_ev_rings,
+ },
+ {
+ MHICFG, MHICFG_NHWER_MASK, MHICFG_NHWER_SHIFT,
+ mhi_cntrl->hw_ev_rings,
+ },
+ {
+ MHICTRLBASE_HIGHER, U32_MAX, 0,
+ upper_32_bits(mhi_cntrl->iova_start),
+ },
+ {
+ MHICTRLBASE_LOWER, U32_MAX, 0,
+ lower_32_bits(mhi_cntrl->iova_start),
+ },
+ {
+ MHIDATABASE_HIGHER, U32_MAX, 0,
+ upper_32_bits(mhi_cntrl->iova_start),
+ },
+ {
+ MHIDATABASE_LOWER, U32_MAX, 0,
+ lower_32_bits(mhi_cntrl->iova_start),
+ },
+ {
+ MHICTRLLIMIT_HIGHER, U32_MAX, 0,
+ upper_32_bits(mhi_cntrl->iova_stop),
+ },
+ {
+ MHICTRLLIMIT_LOWER, U32_MAX, 0,
+ lower_32_bits(mhi_cntrl->iova_stop),
+ },
+ {
+ MHIDATALIMIT_HIGHER, U32_MAX, 0,
+ upper_32_bits(mhi_cntrl->iova_stop),
+ },
+ {
+ MHIDATALIMIT_LOWER, U32_MAX, 0,
+ lower_32_bits(mhi_cntrl->iova_stop),
+ },
+ { 0, 0, 0 }
+ };
+
+ MHI_LOG("Initializing MMIO\n");
+
+ /* set up DB register for all the chan rings */
+ ret = mhi_read_reg_field(mhi_cntrl, base, CHDBOFF, CHDBOFF_CHDBOFF_MASK,
+ CHDBOFF_CHDBOFF_SHIFT, &val);
+ if (ret)
+ return -EIO;
+
+ MHI_LOG("CHDBOFF:0x%x\n", val);
+
+ /* setup wake db */
+ mhi_cntrl->wake_db = base + val + (8 * MHI_DEV_WAKE_DB);
+ mhi_write_reg(mhi_cntrl, mhi_cntrl->wake_db, 4, 0);
+ mhi_write_reg(mhi_cntrl, mhi_cntrl->wake_db, 0, 0);
+ mhi_cntrl->wake_set = false;
+
+ /* setup channel db addresses */
+ mhi_chan = mhi_cntrl->mhi_chan;
+ for (i = 0; i < mhi_cntrl->max_chan; i++, val += 8, mhi_chan++)
+ mhi_chan->tre_ring.db_addr = base + val;
+
+ /* setup event ring db addresses */
+ ret = mhi_read_reg_field(mhi_cntrl, base, ERDBOFF, ERDBOFF_ERDBOFF_MASK,
+ ERDBOFF_ERDBOFF_SHIFT, &val);
+ if (ret)
+ return -EIO;
+
+ MHI_LOG("ERDBOFF:0x%x\n", val);
+
+ mhi_event = mhi_cntrl->mhi_event;
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++, val += 8, mhi_event++) {
+ if (mhi_event->offload_ev)
+ continue;
+
+ mhi_event->ring.db_addr = base + val;
+ }
+
+ /* set up DB register for primary CMD rings */
+ mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING].ring.db_addr = base + CRDB_LOWER;
+
+ MHI_LOG("Programming all MMIO values.\n");
+ for (i = 0; reg_info[i].offset; i++)
+ mhi_write_reg_field(mhi_cntrl, base, reg_info[i].offset,
+ reg_info[i].mask, reg_info[i].shift,
+ reg_info[i].val);
+
+ return 0;
+}
+
+void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan)
+{
+ struct mhi_ring *buf_ring;
+ struct mhi_ring *tre_ring;
+ struct mhi_chan_ctxt *chan_ctxt;
+
+ buf_ring = &mhi_chan->buf_ring;
+ tre_ring = &mhi_chan->tre_ring;
+ chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan];
+
+ mhi_free_coherent(mhi_cntrl, tre_ring->alloc_size,
+ tre_ring->pre_aligned, tre_ring->dma_handle);
+ kfree(buf_ring->base);
+
+ buf_ring->base = tre_ring->base = NULL;
+ chan_ctxt->rbase = 0;
+}
+
+int mhi_init_chan_ctxt(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan)
+{
+ struct mhi_ring *buf_ring;
+ struct mhi_ring *tre_ring;
+ struct mhi_chan_ctxt *chan_ctxt;
+ int ret;
+
+ buf_ring = &mhi_chan->buf_ring;
+ tre_ring = &mhi_chan->tre_ring;
+ tre_ring->el_size = sizeof(struct mhi_tre);
+ tre_ring->len = tre_ring->el_size * tre_ring->elements;
+ chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan];
+ ret = mhi_alloc_aligned_ring(mhi_cntrl, tre_ring, tre_ring->len);
+ if (ret)
+ return -ENOMEM;
+
+ buf_ring->el_size = sizeof(struct mhi_buf_info);
+ buf_ring->len = buf_ring->el_size * buf_ring->elements;
+ buf_ring->base = kzalloc(buf_ring->len, GFP_KERNEL);
+
+ if (!buf_ring->base) {
+ mhi_free_coherent(mhi_cntrl, tre_ring->alloc_size,
+ tre_ring->pre_aligned, tre_ring->dma_handle);
+ return -ENOMEM;
+ }
+
+ chan_ctxt->chstate = MHI_CH_STATE_ENABLED;
+ chan_ctxt->rbase = tre_ring->iommu_base;
+ chan_ctxt->rp = chan_ctxt->wp = chan_ctxt->rbase;
+ chan_ctxt->rlen = tre_ring->len;
+ tre_ring->ctxt_wp = &chan_ctxt->wp;
+
+ tre_ring->rptr = tre_ring->wptr = tre_ring->base;
+ buf_ring->rptr = buf_ring->wptr = buf_ring->base;
+ mhi_chan->db_cfg.db_mode = true;
+
+ /* update to all cores */
+ smp_wmb();
+
+ return 0;
+}
+
+int mhi_device_configure(struct mhi_device *mhi_dev,
+ enum dma_data_direction dir,
+ struct mhi_buf *cfg_tbl, int elements)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_chan *mhi_chan;
+ struct mhi_event_ctxt *er_ctxt;
+ struct mhi_chan_ctxt *ch_ctxt;
+ int er_index, chan;
+
+ switch (dir) {
+ case DMA_TO_DEVICE:
+ mhi_chan = mhi_dev->ul_chan;
+ break;
+ case DMA_BIDIRECTIONAL:
+ case DMA_FROM_DEVICE:
+ case DMA_NONE:
+ mhi_chan = mhi_dev->dl_chan;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ er_index = mhi_chan->er_index;
+ chan = mhi_chan->chan;
+
+ for (; elements > 0; elements--, cfg_tbl++) {
+ /* update event context array */
+ if (!strcmp(cfg_tbl->name, "ECA")) {
+ er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[er_index];
+ if (sizeof(*er_ctxt) != cfg_tbl->len) {
+ MHI_ERR
+ ("Invalid ECA size: %zu, expected: %zu\n",
+ cfg_tbl->len, sizeof(*er_ctxt));
+ return -EINVAL;
+ }
+ memcpy((void *)er_ctxt, cfg_tbl->buf, sizeof(*er_ctxt));
+ continue;
+ }
+
+ /* update channel context array */
+ if (!strcmp(cfg_tbl->name, "CCA")) {
+ ch_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[chan];
+ if (cfg_tbl->len != sizeof(*ch_ctxt)) {
+ MHI_ERR
+ ("Invalid CCA size: %zu, expected: %zu\n",
+ cfg_tbl->len, sizeof(*ch_ctxt));
+ return -EINVAL;
+ }
+ memcpy((void *)ch_ctxt, cfg_tbl->buf, sizeof(*ch_ctxt));
+ continue;
+ }
+
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int of_parse_ev_cfg(struct mhi_controller *mhi_cntrl,
+ struct device_node *of_node)
+{
+ int i, num = 0;
+ struct mhi_event *mhi_event;
+#ifdef CONFIG_OF
+ int ret;
+ struct device_node *child;
+#else
+ int index;
+#endif
+
+#ifdef CONFIG_OF
+ of_node = of_find_node_by_name(of_node, "mhi_events");
+ if (!of_node)
+ return -EINVAL;
+
+ for_each_available_child_of_node(of_node, child) {
+ if (!strcmp(child->name, "mhi_event"))
+ num++;
+ }
+#else
+ num = ARRAY_SIZE(mhi_event_init_array);
+#endif
+
+ if (!num)
+ return -EINVAL;
+
+ mhi_cntrl->total_ev_rings = num;
+ mhi_cntrl->mhi_event = kcalloc(num, sizeof(*mhi_cntrl->mhi_event),
+ GFP_KERNEL);
+ if (!mhi_cntrl->mhi_event)
+ return -ENOMEM;
+
+ /* populate ev ring */
+ mhi_event = mhi_cntrl->mhi_event;
+ i = 0;
+#ifdef CONFIG_OF
+ for_each_available_child_of_node(of_node, child) {
+ if (strcmp(child->name, "mhi_event"))
+ continue;
+
+ mhi_event->er_index = i++;
+ ret = of_property_read_u32(child, "mhi,num-elements",
+ (u32 *)&mhi_event->ring.elements);
+ if (ret)
+ goto error_ev_cfg;
+
+ ret = of_property_read_u32(child, "mhi,intmod",
+ &mhi_event->intmod);
+ if (ret)
+ goto error_ev_cfg;
+
+ ret = of_property_read_u32(child, "mhi,msi", &mhi_event->msi);
+ if (ret)
+ goto error_ev_cfg;
+
+ ret = of_property_read_u32(child, "mhi,chan", &mhi_event->chan);
+ if (!ret) {
+ if (mhi_event->chan >= mhi_cntrl->max_chan)
+ goto error_ev_cfg;
+ /* this event ring has a dedicated channel */
+ mhi_event->mhi_chan =
+ &mhi_cntrl->mhi_chan[mhi_event->chan];
+ }
+
+ ret = of_property_read_u32(child, "mhi,priority",
+ &mhi_event->priority);
+ if (ret)
+ goto error_ev_cfg;
+
+ ret = of_property_read_u32(child, "mhi,brstmode",
+ &mhi_event->db_cfg.brstmode);
+ if (ret || MHI_INVALID_BRSTMODE(mhi_event->db_cfg.brstmode))
+ goto error_ev_cfg;
+
+ mhi_event->db_cfg.process_db =
+ (mhi_event->db_cfg.brstmode == MHI_BRSTMODE_ENABLE) ?
+ mhi_db_brstmode : mhi_db_brstmode_disable;
+
+ ret = of_property_read_u32(child, "mhi,data-type",
+ &mhi_event->data_type);
+ if (ret)
+ mhi_event->data_type = MHI_ER_DATA_ELEMENT_TYPE;
+
+ if (mhi_event->data_type > MHI_ER_DATA_TYPE_MAX)
+ goto error_ev_cfg;
+
+ switch (mhi_event->data_type) {
+ case MHI_ER_DATA_ELEMENT_TYPE:
+ mhi_event->process_event = mhi_process_data_event_ring;
+ break;
+ case MHI_ER_CTRL_ELEMENT_TYPE:
+ mhi_event->process_event = mhi_process_ctrl_ev_ring;
+ break;
+ case MHI_ER_TSYNC_ELEMENT_TYPE:
+ mhi_event->process_event = mhi_process_tsync_event_ring;
+ break;
+ }
+
+ mhi_event->hw_ring = of_property_read_bool(child, "mhi,hw-ev");
+ if (mhi_event->hw_ring)
+ mhi_cntrl->hw_ev_rings++;
+ else
+ mhi_cntrl->sw_ev_rings++;
+ mhi_event->cl_manage = of_property_read_bool(child,
+ "mhi,client-manage");
+ mhi_event->offload_ev = of_property_read_bool(child,
+ "mhi,offload");
+ mhi_event++;
+ }
+#else
+ /* for_each_available_child_of_node(of_node, child) { */
+ for (index = 0; index < ARRAY_SIZE(mhi_event_init_array); index++) {
+
+ mhi_event->er_index = i++;
+ mhi_event->ring.elements = mhi_event_init_array[index].elements;
+
+ mhi_event->intmod = mhi_event_init_array[index].intmod;
+
+ mhi_event->msi = mhi_event_init_array[index].msi;
+
+ mhi_event->chan = mhi_event_init_array[index].chan;
+
+ if (mhi_event->chan) {
+ if (mhi_event->chan >= mhi_cntrl->max_chan)
+ goto error_ev_cfg;
+ /* this event ring has a dedicated channel */
+ mhi_event->mhi_chan =
+ &mhi_cntrl->mhi_chan[mhi_event->chan];
+ }
+
+ mhi_event->priority = mhi_event_init_array[index].priority;
+
+ mhi_event->db_cfg.brstmode =
+ mhi_event_init_array[index].brstmode;
+
+ if (MHI_INVALID_BRSTMODE(mhi_event->db_cfg.brstmode))
+ goto error_ev_cfg;
+
+ mhi_event->db_cfg.process_db =
+ (mhi_event->db_cfg.brstmode == MHI_BRSTMODE_ENABLE) ?
+ mhi_db_brstmode : mhi_db_brstmode_disable;
+
+ mhi_event->data_type = mhi_event_init_array[index].data_type;
+
+ if (!mhi_event->data_type)
+ mhi_event->data_type = MHI_ER_DATA_ELEMENT_TYPE;
+
+ if (mhi_event->data_type > MHI_ER_DATA_TYPE_MAX)
+ goto error_ev_cfg;
+
+ switch (mhi_event->data_type) {
+ case MHI_ER_DATA_ELEMENT_TYPE:
+ mhi_event->process_event = mhi_process_data_event_ring;
+ break;
+ case MHI_ER_CTRL_ELEMENT_TYPE:
+ mhi_event->process_event = mhi_process_ctrl_ev_ring;
+ break;
+ case MHI_ER_TSYNC_ELEMENT_TYPE:
+ mhi_event->process_event = mhi_process_tsync_event_ring;
+ break;
+ }
+
+ mhi_event->hw_ring = mhi_event_init_array[index].hw_ring;
+
+ if (mhi_event->hw_ring)
+ mhi_cntrl->hw_ev_rings++;
+ else
+ mhi_cntrl->sw_ev_rings++;
+ mhi_event->cl_manage = mhi_event_init_array[index].cl_manage;
+ mhi_event->offload_ev = mhi_event_init_array[index].offload_ev;
+ mhi_event++;
+ }
+#endif
+
+ /* we need msi for each event ring + additional one for BHI */
+ mhi_cntrl->msi_required = mhi_cntrl->total_ev_rings + 1;
+
+ return 0;
+
+error_ev_cfg:
+
+ kfree(mhi_cntrl->mhi_event);
+ return -EINVAL;
+}
+
+#ifdef CONFIG_OF
+static int of_parse_ch_cfg(struct mhi_controller *mhi_cntrl,
+ struct device_node *of_node)
+{
+ int ret;
+ struct device_node *child;
+ u32 chan;
+
+ ret = of_property_read_u32(of_node, "mhi,max-channels",
+ &mhi_cntrl->max_chan);
+ if (ret)
+ return ret;
+
+ of_node = of_find_node_by_name(of_node, "mhi_channels");
+ if (!of_node)
+ return -EINVAL;
+
+ mhi_cntrl->mhi_chan = kcalloc(mhi_cntrl->max_chan,
+ sizeof(*mhi_cntrl->mhi_chan), GFP_KERNEL);
+ if (!mhi_cntrl->mhi_chan)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&mhi_cntrl->lpm_chans);
+
+ /* populate channel configurations */
+ for_each_available_child_of_node(of_node, child) {
+ struct mhi_chan *mhi_chan;
+
+ if (strcmp(child->name, "mhi_chan"))
+ continue;
+
+ ret = of_property_read_u32(child, "reg", &chan);
+ if (ret || chan >= mhi_cntrl->max_chan)
+ goto error_chan_cfg;
+
+ mhi_chan = &mhi_cntrl->mhi_chan[chan];
+
+ ret = of_property_read_string(child, "label", &mhi_chan->name);
+ if (ret)
+ goto error_chan_cfg;
+
+ mhi_chan->chan = chan;
+
+ ret = of_property_read_u32(child, "mhi,num-elements",
+ (u32 *)&mhi_chan->tre_ring.elements);
+ if (!ret && !mhi_chan->tre_ring.elements)
+ goto error_chan_cfg;
+
+ /*
+ * For some channels, local ring len should be bigger than
+ * transfer ring len due to internal logical channels in device.
+ * So host can queue much more buffers than transfer ring len.
+ * Example, RSC channels should have a larger local channel
+ * than transfer ring length.
+ */
+ ret = of_property_read_u32(child, "mhi,local-elements",
+ (u32 *)&mhi_chan->buf_ring.elements);
+ if (ret)
+ mhi_chan->buf_ring.elements =
+ mhi_chan->tre_ring.elements;
+
+ ret = of_property_read_u32(child, "mhi,event-ring",
+ &mhi_chan->er_index);
+ if (ret)
+ goto error_chan_cfg;
+
+ ret = of_property_read_u32(child, "mhi,chan-dir",
+ &mhi_chan->dir);
+ if (ret)
+ goto error_chan_cfg;
+
+ /*
+ * For most channels, chtype is identical to channel directions,
+ * if not defined, assign ch direction to chtype
+ */
+ ret = of_property_read_u32(child, "mhi,chan-type",
+ &mhi_chan->type);
+ if (ret)
+ mhi_chan->type = (enum mhi_ch_type)mhi_chan->dir;
+
+ ret = of_property_read_u32(child, "mhi,ee", &mhi_chan->ee_mask);
+ if (ret)
+ goto error_chan_cfg;
+
+ of_property_read_u32(child, "mhi,pollcfg",
+ &mhi_chan->db_cfg.pollcfg);
+
+ ret = of_property_read_u32(child, "mhi,data-type",
+ &mhi_chan->xfer_type);
+ if (ret)
+ goto error_chan_cfg;
+
+ switch (mhi_chan->xfer_type) {
+ case MHI_XFER_BUFFER:
+ mhi_chan->gen_tre = mhi_gen_tre;
+ mhi_chan->queue_xfer = mhi_queue_buf;
+ break;
+ case MHI_XFER_SKB:
+ mhi_chan->queue_xfer = mhi_queue_skb;
+ break;
+ case MHI_XFER_SCLIST:
+ mhi_chan->gen_tre = mhi_gen_tre;
+ mhi_chan->queue_xfer = mhi_queue_sclist;
+ break;
+ case MHI_XFER_NOP:
+ mhi_chan->queue_xfer = mhi_queue_nop;
+ break;
+ case MHI_XFER_DMA:
+ case MHI_XFER_RSC_DMA:
+ mhi_chan->queue_xfer = mhi_queue_dma;
+ break;
+ default:
+ goto error_chan_cfg;
+ }
+
+ mhi_chan->lpm_notify = of_property_read_bool(child,
+ "mhi,lpm-notify");
+ mhi_chan->offload_ch = of_property_read_bool(child,
+ "mhi,offload-chan");
+ mhi_chan->db_cfg.reset_req = of_property_read_bool(child,
+ "mhi,db-mode-switch");
+ mhi_chan->pre_alloc = of_property_read_bool(child,
+ "mhi,auto-queue");
+ mhi_chan->auto_start = of_property_read_bool(child,
+ "mhi,auto-start");
+ mhi_chan->wake_capable = of_property_read_bool(child,
+ "mhi,wake-capable");
+
+ if (mhi_chan->pre_alloc &&
+ (mhi_chan->dir != DMA_FROM_DEVICE ||
+ mhi_chan->xfer_type != MHI_XFER_BUFFER))
+ goto error_chan_cfg;
+
+ /* bi-dir and dirctionless channels must be a offload chan */
+ if ((mhi_chan->dir == DMA_BIDIRECTIONAL ||
+ mhi_chan->dir == DMA_NONE) && !mhi_chan->offload_ch)
+ goto error_chan_cfg;
+
+ /* if mhi host allocate the buffers then client cannot queue */
+ if (mhi_chan->pre_alloc)
+ mhi_chan->queue_xfer = mhi_queue_nop;
+
+ if (!mhi_chan->offload_ch) {
+ ret = of_property_read_u32(child, "mhi,doorbell-mode",
+ &mhi_chan->db_cfg.brstmode);
+ if (ret ||
+ MHI_INVALID_BRSTMODE(mhi_chan->db_cfg.brstmode))
+ goto error_chan_cfg;
+
+ mhi_chan->db_cfg.process_db =
+ (mhi_chan->db_cfg.brstmode ==
+ MHI_BRSTMODE_ENABLE) ?
+ mhi_db_brstmode : mhi_db_brstmode_disable;
+ }
+
+ mhi_chan->configured = true;
+
+ if (mhi_chan->lpm_notify)
+ list_add_tail(&mhi_chan->node, &mhi_cntrl->lpm_chans);
+ }
+
+ return 0;
+
+error_chan_cfg:
+ kfree(mhi_cntrl->mhi_chan);
+
+ return -EINVAL;
+}
+
+static int of_parse_dt(struct mhi_controller *mhi_cntrl,
+ struct device_node *of_node)
+{
+ int ret;
+
+ /* parse MHI channel configuration */
+ ret = of_parse_ch_cfg(mhi_cntrl, of_node);
+ if (ret)
+ return ret;
+
+ /* parse MHI event configuration */
+ ret = of_parse_ev_cfg(mhi_cntrl, of_node);
+ if (ret)
+ goto error_ev_cfg;
+
+ ret = of_property_read_u32(of_node, "mhi,timeout",
+ &mhi_cntrl->timeout_ms);
+ if (ret)
+ mhi_cntrl->timeout_ms = MHI_TIMEOUT_MS;
+
+ mhi_cntrl->bounce_buf = of_property_read_bool(of_node, "mhi,use-bb");
+ ret = of_property_read_u32(of_node, "mhi,buffer-len",
+ (u32 *)&mhi_cntrl->buffer_len);
+ if (ret)
+ mhi_cntrl->buffer_len = MHI_MAX_MTU;
+
+ return 0;
+
+error_ev_cfg:
+ kfree(mhi_cntrl->mhi_chan);
+
+ return ret;
+}
+#else
+static int of_parse_ch_cfg(struct mhi_controller *mhi_cntrl,
+ struct device_node *of_node)
+{
+ u32 chan;
+ int i;
+
+ mhi_cntrl->max_chan = 110;
+ mhi_cntrl->mhi_chan = kcalloc(mhi_cntrl->max_chan,
+ sizeof(*mhi_cntrl->mhi_chan), GFP_KERNEL);
+ if (!mhi_cntrl->mhi_chan)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&mhi_cntrl->lpm_chans);
+
+ /* populate channel configurations */
+ for (i = 0; i < ARRAY_SIZE(mhi_chan_init_array); i++) {
+
+ struct mhi_chan *mhi_chan;
+
+ chan = mhi_chan_init_array[i].chan;
+
+ if (chan >= mhi_cntrl->max_chan)
+ goto error_chan_cfg;
+
+ mhi_chan = &mhi_cntrl->mhi_chan[chan];
+
+ mhi_chan->name = mhi_chan_init_array[i].name;
+
+ mhi_chan->chan = chan;
+
+ mhi_chan->tre_ring.elements =
+ mhi_chan_init_array[i].tre_ring_elements;
+
+ /*
+ * For some channels, local ring len should be bigger than
+ * transfer ring len due to internal logical channels in device.
+ * So host can queue much more buffers than transfer ring len.
+ * Example, RSC channels should have a larger local channel
+ * than transfer ring length.
+ */
+ mhi_chan->buf_ring.elements = mhi_chan->tre_ring.elements;
+
+ mhi_chan->er_index = mhi_chan_init_array[i].er_index;
+
+ mhi_chan->dir = mhi_chan_init_array[i].dir;
+
+ /*
+ * For most channels, chtype is identical to channel directions,
+ * if not defined, assign ch direction to chtype
+ */
+ mhi_chan->type = (enum mhi_ch_type)mhi_chan->dir;
+
+ mhi_chan->ee_mask = mhi_chan_init_array[i].ee_mask;
+
+ mhi_chan->db_cfg.pollcfg = mhi_chan_init_array[i].pollcfg;
+
+ mhi_chan->xfer_type = mhi_chan_init_array[i].xfer_type;
+
+ switch (mhi_chan->xfer_type) {
+ case MHI_XFER_BUFFER:
+ mhi_chan->gen_tre = mhi_gen_tre;
+ mhi_chan->queue_xfer = mhi_queue_buf;
+ break;
+ case MHI_XFER_SKB:
+ mhi_chan->queue_xfer = mhi_queue_skb;
+ break;
+ case MHI_XFER_SCLIST:
+ mhi_chan->gen_tre = mhi_gen_tre;
+ mhi_chan->queue_xfer = mhi_queue_sclist;
+ break;
+ case MHI_XFER_NOP:
+ mhi_chan->queue_xfer = mhi_queue_nop;
+ break;
+ case MHI_XFER_DMA:
+ case MHI_XFER_RSC_DMA:
+ mhi_chan->queue_xfer = mhi_queue_dma;
+ break;
+ default:
+ goto error_chan_cfg;
+ }
+
+ mhi_chan->lpm_notify = mhi_chan_init_array[i].lpm_notify;
+
+ mhi_chan->offload_ch = mhi_chan_init_array[i].offload_ch;
+
+ mhi_chan->db_cfg.reset_req = mhi_chan_init_array[i].reset_req;
+
+ mhi_chan->pre_alloc = mhi_chan_init_array[i].pre_alloc;
+
+ mhi_chan->auto_start = mhi_chan_init_array[i].auto_start;
+
+ mhi_chan->wake_capable = mhi_chan_init_array[i].wake_capable;
+
+ if (mhi_chan->pre_alloc &&
+ (mhi_chan->dir != DMA_FROM_DEVICE ||
+ mhi_chan->xfer_type != MHI_XFER_BUFFER))
+ goto error_chan_cfg;
+
+ /* bi-dir and dirctionless channels must be a offload chan */
+ if ((mhi_chan->dir == DMA_BIDIRECTIONAL ||
+ mhi_chan->dir == DMA_NONE) && !mhi_chan->offload_ch)
+ goto error_chan_cfg;
+
+ /* if mhi host allocate the buffers then client cannot queue */
+ if (mhi_chan->pre_alloc)
+ mhi_chan->queue_xfer = mhi_queue_nop;
+
+ if (!mhi_chan->offload_ch) {
+ mhi_chan->db_cfg.brstmode =
+ mhi_chan_init_array[i].brstmode;
+
+ if (MHI_INVALID_BRSTMODE(mhi_chan->db_cfg.brstmode))
+ goto error_chan_cfg;
+
+ mhi_chan->db_cfg.process_db =
+ (mhi_chan->db_cfg.brstmode ==
+ MHI_BRSTMODE_ENABLE) ?
+ mhi_db_brstmode : mhi_db_brstmode_disable;
+ }
+
+ mhi_chan->configured = true;
+
+ if (mhi_chan->lpm_notify)
+ list_add_tail(&mhi_chan->node, &mhi_cntrl->lpm_chans);
+ }
+
+ return 0;
+
+error_chan_cfg:
+ kfree(mhi_cntrl->mhi_chan);
+
+ return -EINVAL;
+}
+
+static int of_parse_dt(struct mhi_controller *mhi_cntrl,
+ struct device_node *of_node)
+{
+ int ret;
+
+ /* parse MHI channel configuration */
+ ret = of_parse_ch_cfg(mhi_cntrl, of_node);
+ if (ret)
+ return ret;
+
+ /* parse MHI event configuration */
+ ret = of_parse_ev_cfg(mhi_cntrl, of_node);
+ if (ret)
+ goto error_ev_cfg;
+
+ mhi_cntrl->timeout_ms = 600;
+
+ mhi_cntrl->bounce_buf = 0;
+
+ mhi_cntrl->buffer_len = MHI_MAX_MTU;
+
+ return 0;
+
+error_ev_cfg:
+ kfree(mhi_cntrl->mhi_chan);
+
+ return ret;
+}
+#endif
+
+int of_register_mhi_controller(struct mhi_controller *mhi_cntrl)
+{
+ int ret;
+ int i;
+ struct mhi_event *mhi_event;
+ struct mhi_chan *mhi_chan;
+ struct mhi_cmd *mhi_cmd;
+ struct mhi_device *mhi_dev;
+
+#ifdef CONFIG_OF
+ if (!mhi_cntrl->of_node)
+ return -EINVAL;
+#endif
+
+ if (!mhi_cntrl->runtime_get || !mhi_cntrl->runtime_put)
+ return -EINVAL;
+
+ if (!mhi_cntrl->status_cb || !mhi_cntrl->link_status)
+ return -EINVAL;
+
+ ret = of_parse_dt(mhi_cntrl, mhi_cntrl->of_node);
+ if (ret)
+ return -EINVAL;
+
+ mhi_cntrl->mhi_cmd = kcalloc(NR_OF_CMD_RINGS,
+ sizeof(*mhi_cntrl->mhi_cmd), GFP_KERNEL);
+ if (!mhi_cntrl->mhi_cmd) {
+ ret = -ENOMEM;
+ goto error_alloc_cmd;
+ }
+
+ INIT_LIST_HEAD(&mhi_cntrl->transition_list);
+ mutex_init(&mhi_cntrl->pm_mutex);
+ rwlock_init(&mhi_cntrl->pm_lock);
+ spin_lock_init(&mhi_cntrl->transition_lock);
+ spin_lock_init(&mhi_cntrl->wlock);
+ INIT_WORK(&mhi_cntrl->st_worker, mhi_pm_st_worker);
+ INIT_WORK(&mhi_cntrl->fw_worker, mhi_fw_load_worker);
+ INIT_WORK(&mhi_cntrl->syserr_worker, mhi_pm_sys_err_worker);
+ init_waitqueue_head(&mhi_cntrl->state_event);
+
+ mhi_cmd = mhi_cntrl->mhi_cmd;
+ for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++)
+ spin_lock_init(&mhi_cmd->lock);
+
+ mhi_event = mhi_cntrl->mhi_event;
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
+ if (mhi_event->offload_ev)
+ continue;
+
+ mhi_event->mhi_cntrl = mhi_cntrl;
+ spin_lock_init(&mhi_event->lock);
+ if (mhi_event->data_type == MHI_ER_CTRL_ELEMENT_TYPE)
+ tasklet_init(&mhi_event->task, mhi_ctrl_ev_task,
+ (ulong)mhi_event);
+ else
+ tasklet_init(&mhi_event->task, mhi_ev_task,
+ (ulong)mhi_event);
+ }
+
+ mhi_chan = mhi_cntrl->mhi_chan;
+ for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) {
+ mutex_init(&mhi_chan->mutex);
+ init_completion(&mhi_chan->completion);
+ rwlock_init(&mhi_chan->lock);
+ }
+
+ if (mhi_cntrl->bounce_buf) {
+ mhi_cntrl->map_single = mhi_map_single_use_bb;
+ mhi_cntrl->unmap_single = mhi_unmap_single_use_bb;
+ } else {
+ mhi_cntrl->map_single = mhi_map_single_no_bb;
+ mhi_cntrl->unmap_single = mhi_unmap_single_no_bb;
+ }
+
+ /* register controller with mhi_bus */
+ mhi_dev = mhi_alloc_device(mhi_cntrl);
+ if (!mhi_dev) {
+ ret = -ENOMEM;
+ goto error_alloc_dev;
+ }
+
+ mhi_dev->dev_type = MHI_CONTROLLER_TYPE;
+ mhi_dev->mhi_cntrl = mhi_cntrl;
+ dev_set_name(&mhi_dev->dev, "%04x_%02u.%02u.%02u", mhi_dev->dev_id,
+ mhi_dev->domain, mhi_dev->bus, mhi_dev->slot);
+ ret = device_add(&mhi_dev->dev);
+ if (ret)
+ goto error_add_dev;
+
+ mhi_cntrl->mhi_dev = mhi_dev;
+
+ mhi_cntrl->parent = debugfs_lookup(mhi_bus_type.name, NULL);
+ mhi_cntrl->klog_lvl = MHI_MSG_LVL_ERROR;
+
+ /* adding it to this list only for debug purpose */
+ mutex_lock(&mhi_bus.lock);
+ list_add_tail(&mhi_cntrl->node, &mhi_bus.controller_list);
+ mutex_unlock(&mhi_bus.lock);
+
+ return 0;
+
+error_add_dev:
+ mhi_dealloc_device(mhi_cntrl, mhi_dev);
+
+error_alloc_dev:
+ kfree(mhi_cntrl->mhi_cmd);
+
+error_alloc_cmd:
+ kfree(mhi_cntrl->mhi_chan);
+ kfree(mhi_cntrl->mhi_event);
+
+ return ret;
+};
+EXPORT_SYMBOL(of_register_mhi_controller);
+
+void mhi_unregister_mhi_controller(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_device *mhi_dev = mhi_cntrl->mhi_dev;
+
+ kfree(mhi_cntrl->mhi_cmd);
+ kfree(mhi_cntrl->mhi_event);
+ kfree(mhi_cntrl->mhi_chan);
+ kfree(mhi_cntrl->mhi_tsync);
+
+ device_del(&mhi_dev->dev);
+ put_device(&mhi_dev->dev);
+
+ mutex_lock(&mhi_bus.lock);
+ list_del(&mhi_cntrl->node);
+ mutex_unlock(&mhi_bus.lock);
+}
+EXPORT_SYMBOL(mhi_unregister_mhi_controller);
+
+/* set ptr to control private data */
+static inline void mhi_controller_set_devdata(struct mhi_controller *mhi_cntrl,
+ void *priv)
+{
+ mhi_cntrl->priv_data = priv;
+}
+
+/* allocate mhi controller to register */
+struct mhi_controller *mhi_alloc_controller(size_t size)
+{
+ struct mhi_controller *mhi_cntrl;
+
+ mhi_cntrl = kzalloc(size + sizeof(*mhi_cntrl), GFP_KERNEL);
+
+ if (mhi_cntrl && size)
+ mhi_controller_set_devdata(mhi_cntrl, mhi_cntrl + 1);
+
+ return mhi_cntrl;
+}
+EXPORT_SYMBOL(mhi_alloc_controller);
+
+int mhi_prepare_for_power_up(struct mhi_controller *mhi_cntrl)
+{
+ int ret;
+ u32 bhie_off;
+
+ mutex_lock(&mhi_cntrl->pm_mutex);
+
+ ret = mhi_init_dev_ctxt(mhi_cntrl);
+ if (ret) {
+ MHI_ERR("Error with init dev_ctxt\n");
+ goto error_dev_ctxt;
+ }
+
+ ret = mhi_init_irq_setup(mhi_cntrl);
+ if (ret) {
+ MHI_ERR("Error setting up irq\n");
+ goto error_setup_irq;
+ }
+
+ /*
+ * allocate rddm table if specified, this table is for debug purpose
+ * so we'll ignore erros
+ */
+ if (mhi_cntrl->rddm_size) {
+ mhi_alloc_bhie_table(mhi_cntrl, &mhi_cntrl->rddm_image,
+ mhi_cntrl->rddm_size);
+
+ /*
+ * This controller supports rddm, we need to manually clear
+ * BHIE RX registers since por values are undefined.
+ */
+ ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, BHIEOFF,
+ &bhie_off);
+ if (ret) {
+ MHI_ERR("Error getting bhie offset\n");
+ goto bhie_error;
+ }
+
+ memset_io(mhi_cntrl->regs + bhie_off + BHIE_RXVECADDR_LOW_OFFS,
+ 0, BHIE_RXVECSTATUS_OFFS - BHIE_RXVECADDR_LOW_OFFS +
+ 4);
+ }
+
+ mhi_cntrl->pre_init = true;
+
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+
+ return 0;
+
+bhie_error:
+ if (mhi_cntrl->rddm_image) {
+ mhi_free_bhie_table(mhi_cntrl, mhi_cntrl->rddm_image);
+ mhi_cntrl->rddm_image = NULL;
+ }
+ mhi_deinit_free_irq(mhi_cntrl);
+
+error_setup_irq:
+ mhi_deinit_dev_ctxt(mhi_cntrl);
+
+error_dev_ctxt:
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL(mhi_prepare_for_power_up);
+
+void mhi_unprepare_after_power_down(struct mhi_controller *mhi_cntrl)
+{
+ if (mhi_cntrl->fbc_image) {
+ mhi_free_bhie_table(mhi_cntrl, mhi_cntrl->fbc_image);
+ mhi_cntrl->fbc_image = NULL;
+ }
+
+ if (mhi_cntrl->rddm_image) {
+ mhi_free_bhie_table(mhi_cntrl, mhi_cntrl->rddm_image);
+ mhi_cntrl->rddm_image = NULL;
+ }
+
+ mhi_deinit_free_irq(mhi_cntrl);
+ mhi_deinit_dev_ctxt(mhi_cntrl);
+ mhi_cntrl->pre_init = false;
+}
+
+/* match dev to drv */
+static int mhi_match(struct device *dev, struct device_driver *drv)
+{
+ struct mhi_device *mhi_dev = to_mhi_device(dev);
+ struct mhi_driver *mhi_drv = to_mhi_driver(drv);
+ const struct mhi_device_id *id;
+
+ /* if controller type there is no client driver associated with it */
+ if (mhi_dev->dev_type == MHI_CONTROLLER_TYPE)
+ return 0;
+
+ for (id = mhi_drv->id_table; id->chan[0]; id++)
+ if (!strcmp(mhi_dev->chan_name, id->chan)) {
+ mhi_dev->id = id;
+ return 1;
+ }
+
+ return 0;
+};
+
+static void mhi_release_device(struct device *dev)
+{
+ struct mhi_device *mhi_dev = to_mhi_device(dev);
+
+ if (mhi_dev->ul_chan)
+ mhi_dev->ul_chan->mhi_dev = NULL;
+
+ if (mhi_dev->dl_chan)
+ mhi_dev->dl_chan->mhi_dev = NULL;
+
+ kfree(mhi_dev);
+}
+
+struct bus_type mhi_bus_type = {
+ .name = "mhi",
+ .dev_name = "mhi",
+ .match = mhi_match,
+};
+
+static int mhi_driver_probe(struct device *dev)
+{
+ struct mhi_device *mhi_dev = to_mhi_device(dev);
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct device_driver *drv = dev->driver;
+ struct mhi_driver *mhi_drv = to_mhi_driver(drv);
+ struct mhi_event *mhi_event;
+ struct mhi_chan *ul_chan = mhi_dev->ul_chan;
+ struct mhi_chan *dl_chan = mhi_dev->dl_chan;
+ bool auto_start = false;
+ int ret;
+
+ /* bring device out of lpm */
+ ret = mhi_device_get_sync(mhi_dev);
+ if (ret)
+ return ret;
+
+ ret = -EINVAL;
+ if (ul_chan) {
+ /* lpm notification require status_cb */
+ if (ul_chan->lpm_notify && !mhi_drv->status_cb)
+ goto exit_probe;
+
+ if (!ul_chan->offload_ch && !mhi_drv->ul_xfer_cb)
+ goto exit_probe;
+
+ ul_chan->xfer_cb = mhi_drv->ul_xfer_cb;
+ mhi_dev->status_cb = mhi_drv->status_cb;
+ auto_start = ul_chan->auto_start;
+ }
+
+ if (dl_chan) {
+ if (dl_chan->lpm_notify && !mhi_drv->status_cb)
+ goto exit_probe;
+
+ if (!dl_chan->offload_ch && !mhi_drv->dl_xfer_cb)
+ goto exit_probe;
+
+ mhi_event = &mhi_cntrl->mhi_event[dl_chan->er_index];
+
+ /*
+ * if this channal event ring manage by client, then
+ * status_cb must be defined so we can send the async
+ * cb whenever there are pending data
+ */
+ if (mhi_event->cl_manage && !mhi_drv->status_cb)
+ goto exit_probe;
+
+ dl_chan->xfer_cb = mhi_drv->dl_xfer_cb;
+
+ /* ul & dl uses same status cb */
+ mhi_dev->status_cb = mhi_drv->status_cb;
+ auto_start = (auto_start || dl_chan->auto_start);
+ }
+
+ ret = mhi_drv->probe(mhi_dev, mhi_dev->id);
+
+ if (!ret && auto_start)
+ mhi_prepare_for_transfer(mhi_dev);
+
+exit_probe:
+ mhi_device_put(mhi_dev);
+
+ return ret;
+}
+
+static int mhi_driver_remove(struct device *dev)
+{
+ struct mhi_device *mhi_dev = to_mhi_device(dev);
+ struct mhi_driver *mhi_drv = to_mhi_driver(dev->driver);
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_chan *mhi_chan;
+ enum MHI_CH_STATE ch_state[] = {
+ MHI_CH_STATE_DISABLED,
+ MHI_CH_STATE_DISABLED
+ };
+ int dir;
+
+ /* control device has no work to do */
+ if (mhi_dev->dev_type == MHI_CONTROLLER_TYPE)
+ return 0;
+
+ MHI_LOG("Removing device for chan:%s\n", mhi_dev->chan_name);
+
+ /* reset both channels */
+ for (dir = 0; dir < 2; dir++) {
+ mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan;
+
+ if (!mhi_chan)
+ continue;
+
+ /* wake all threads waiting for completion */
+ write_lock_irq(&mhi_chan->lock);
+ mhi_chan->ccs = MHI_EV_CC_INVALID;
+ complete_all(&mhi_chan->completion);
+ write_unlock_irq(&mhi_chan->lock);
+
+ /* move channel state to disable, no more processing */
+ mutex_lock(&mhi_chan->mutex);
+ write_lock_irq(&mhi_chan->lock);
+ ch_state[dir] = mhi_chan->ch_state;
+ mhi_chan->ch_state = MHI_CH_STATE_SUSPENDED;
+ write_unlock_irq(&mhi_chan->lock);
+
+ /* reset the channel */
+ if (!mhi_chan->offload_ch)
+ mhi_reset_chan(mhi_cntrl, mhi_chan);
+
+ mutex_unlock(&mhi_chan->mutex);
+ }
+
+ /* destroy the device */
+ mhi_drv->remove(mhi_dev);
+
+ /* de_init channel if it was enabled */
+ for (dir = 0; dir < 2; dir++) {
+ mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan;
+
+ if (!mhi_chan)
+ continue;
+
+ mutex_lock(&mhi_chan->mutex);
+
+ if (ch_state[dir] == MHI_CH_STATE_ENABLED &&
+ !mhi_chan->offload_ch)
+ mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan);
+
+ mhi_chan->ch_state = MHI_CH_STATE_DISABLED;
+
+ mutex_unlock(&mhi_chan->mutex);
+ }
+
+ if (mhi_cntrl->tsync_dev == mhi_dev)
+ mhi_cntrl->tsync_dev = NULL;
+
+ /* relinquish any pending votes */
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ while (atomic_read(&mhi_dev->dev_wake))
+ mhi_device_put(mhi_dev);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return 0;
+}
+
+int mhi_driver_register(struct mhi_driver *mhi_drv)
+{
+ struct device_driver *driver = &mhi_drv->driver;
+
+ if (!mhi_drv->probe || !mhi_drv->remove)
+ return -EINVAL;
+
+ driver->bus = &mhi_bus_type;
+ driver->probe = mhi_driver_probe;
+ driver->remove = mhi_driver_remove;
+ return driver_register(driver);
+}
+EXPORT_SYMBOL(mhi_driver_register);
+
+void mhi_driver_unregister(struct mhi_driver *mhi_drv)
+{
+ driver_unregister(&mhi_drv->driver);
+}
+EXPORT_SYMBOL(mhi_driver_unregister);
+
+struct mhi_device *mhi_alloc_device(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_device *mhi_dev = kzalloc(sizeof(*mhi_dev), GFP_KERNEL);
+ struct device *dev;
+
+ if (!mhi_dev)
+ return NULL;
+
+ dev = &mhi_dev->dev;
+ device_initialize(dev);
+ dev->bus = &mhi_bus_type;
+ dev->release = mhi_release_device;
+ dev->parent = mhi_cntrl->dev;
+ mhi_dev->mhi_cntrl = mhi_cntrl;
+ mhi_dev->dev_id = mhi_cntrl->dev_id;
+ mhi_dev->domain = mhi_cntrl->domain;
+ mhi_dev->bus = mhi_cntrl->bus;
+ mhi_dev->slot = mhi_cntrl->slot;
+ mhi_dev->mtu = MHI_MAX_MTU;
+ atomic_set(&mhi_dev->dev_wake, 0);
+
+ return mhi_dev;
+}
+
+static int __init mhi_init(void)
+{
+ int ret;
+
+ mutex_init(&mhi_bus.lock);
+ INIT_LIST_HEAD(&mhi_bus.controller_list);
+
+ /* parent directory */
+ debugfs_create_dir(mhi_bus_type.name, NULL);
+
+ ret = bus_register(&mhi_bus_type);
+
+ if (!ret)
+ mhi_dtr_init();
+ return ret;
+}
+
+module_init(mhi_init);
+
+static void __exit mhi_exit(void)
+{
+ struct dentry *dentry;
+
+ bus_unregister(&mhi_bus_type);
+ dentry = debugfs_lookup(mhi_bus_type.name, NULL);
+ debugfs_remove(dentry);
+}
+
+module_exit(mhi_exit);
+
+MODULE_AUTHOR("Qualcomm Corporation");
+MODULE_DESCRIPTION("Qualcomm Modem Host Interface Bus Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/bus/mhi/core/mhi_internal.h b/drivers/bus/mhi/core/mhi_internal.h
new file mode 100644
index 000000000000..380712ba451e
--- /dev/null
+++ b/drivers/bus/mhi/core/mhi_internal.h
@@ -0,0 +1,826 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. */
+
+#ifndef _MHI_INT_H
+#define _MHI_INT_H
+
+extern struct bus_type mhi_bus_type;
+
+/* MHI mmio register mapping */
+#define PCI_INVALID_READ(val) (val == U32_MAX)
+
+#define MHIREGLEN (0x0)
+#define MHIREGLEN_MHIREGLEN_MASK (0xFFFFFFFF)
+#define MHIREGLEN_MHIREGLEN_SHIFT (0)
+
+#define MHIVER (0x8)
+#define MHIVER_MHIVER_MASK (0xFFFFFFFF)
+#define MHIVER_MHIVER_SHIFT (0)
+
+#define MHICFG (0x10)
+#define MHICFG_NHWER_MASK (0xFF000000)
+#define MHICFG_NHWER_SHIFT (24)
+#define MHICFG_NER_MASK (0xFF0000)
+#define MHICFG_NER_SHIFT (16)
+#define MHICFG_NHWCH_MASK (0xFF00)
+#define MHICFG_NHWCH_SHIFT (8)
+#define MHICFG_NCH_MASK (0xFF)
+#define MHICFG_NCH_SHIFT (0)
+
+#define CHDBOFF (0x18)
+#define CHDBOFF_CHDBOFF_MASK (0xFFFFFFFF)
+#define CHDBOFF_CHDBOFF_SHIFT (0)
+
+#define ERDBOFF (0x20)
+#define ERDBOFF_ERDBOFF_MASK (0xFFFFFFFF)
+#define ERDBOFF_ERDBOFF_SHIFT (0)
+
+#define BHIOFF (0x28)
+#define BHIOFF_BHIOFF_MASK (0xFFFFFFFF)
+#define BHIOFF_BHIOFF_SHIFT (0)
+
+#define BHIEOFF (0x2C)
+#define BHIEOFF_BHIEOFF_MASK (0xFFFFFFFF)
+#define BHIEOFF_BHIEOFF_SHIFT (0)
+
+#define DEBUGOFF (0x30)
+#define DEBUGOFF_DEBUGOFF_MASK (0xFFFFFFFF)
+#define DEBUGOFF_DEBUGOFF_SHIFT (0)
+
+#define MHICTRL (0x38)
+#define MHICTRL_MHISTATE_MASK (0x0000FF00)
+#define MHICTRL_MHISTATE_SHIFT (8)
+#define MHICTRL_RESET_MASK (0x2)
+#define MHICTRL_RESET_SHIFT (1)
+
+#define MHISTATUS (0x48)
+#define MHISTATUS_MHISTATE_MASK (0x0000FF00)
+#define MHISTATUS_MHISTATE_SHIFT (8)
+#define MHISTATUS_SYSERR_MASK (0x4)
+#define MHISTATUS_SYSERR_SHIFT (2)
+#define MHISTATUS_READY_MASK (0x1)
+#define MHISTATUS_READY_SHIFT (0)
+
+#define CCABAP_LOWER (0x58)
+#define CCABAP_LOWER_CCABAP_LOWER_MASK (0xFFFFFFFF)
+#define CCABAP_LOWER_CCABAP_LOWER_SHIFT (0)
+
+#define CCABAP_HIGHER (0x5C)
+#define CCABAP_HIGHER_CCABAP_HIGHER_MASK (0xFFFFFFFF)
+#define CCABAP_HIGHER_CCABAP_HIGHER_SHIFT (0)
+
+#define ECABAP_LOWER (0x60)
+#define ECABAP_LOWER_ECABAP_LOWER_MASK (0xFFFFFFFF)
+#define ECABAP_LOWER_ECABAP_LOWER_SHIFT (0)
+
+#define ECABAP_HIGHER (0x64)
+#define ECABAP_HIGHER_ECABAP_HIGHER_MASK (0xFFFFFFFF)
+#define ECABAP_HIGHER_ECABAP_HIGHER_SHIFT (0)
+
+#define CRCBAP_LOWER (0x68)
+#define CRCBAP_LOWER_CRCBAP_LOWER_MASK (0xFFFFFFFF)
+#define CRCBAP_LOWER_CRCBAP_LOWER_SHIFT (0)
+
+#define CRCBAP_HIGHER (0x6C)
+#define CRCBAP_HIGHER_CRCBAP_HIGHER_MASK (0xFFFFFFFF)
+#define CRCBAP_HIGHER_CRCBAP_HIGHER_SHIFT (0)
+
+#define CRDB_LOWER (0x70)
+#define CRDB_LOWER_CRDB_LOWER_MASK (0xFFFFFFFF)
+#define CRDB_LOWER_CRDB_LOWER_SHIFT (0)
+
+#define CRDB_HIGHER (0x74)
+#define CRDB_HIGHER_CRDB_HIGHER_MASK (0xFFFFFFFF)
+#define CRDB_HIGHER_CRDB_HIGHER_SHIFT (0)
+
+#define MHICTRLBASE_LOWER (0x80)
+#define MHICTRLBASE_LOWER_MHICTRLBASE_LOWER_MASK (0xFFFFFFFF)
+#define MHICTRLBASE_LOWER_MHICTRLBASE_LOWER_SHIFT (0)
+
+#define MHICTRLBASE_HIGHER (0x84)
+#define MHICTRLBASE_HIGHER_MHICTRLBASE_HIGHER_MASK (0xFFFFFFFF)
+#define MHICTRLBASE_HIGHER_MHICTRLBASE_HIGHER_SHIFT (0)
+
+#define MHICTRLLIMIT_LOWER (0x88)
+#define MHICTRLLIMIT_LOWER_MHICTRLLIMIT_LOWER_MASK (0xFFFFFFFF)
+#define MHICTRLLIMIT_LOWER_MHICTRLLIMIT_LOWER_SHIFT (0)
+
+#define MHICTRLLIMIT_HIGHER (0x8C)
+#define MHICTRLLIMIT_HIGHER_MHICTRLLIMIT_HIGHER_MASK (0xFFFFFFFF)
+#define MHICTRLLIMIT_HIGHER_MHICTRLLIMIT_HIGHER_SHIFT (0)
+
+#define MHIDATABASE_LOWER (0x98)
+#define MHIDATABASE_LOWER_MHIDATABASE_LOWER_MASK (0xFFFFFFFF)
+#define MHIDATABASE_LOWER_MHIDATABASE_LOWER_SHIFT (0)
+
+#define MHIDATABASE_HIGHER (0x9C)
+#define MHIDATABASE_HIGHER_MHIDATABASE_HIGHER_MASK (0xFFFFFFFF)
+#define MHIDATABASE_HIGHER_MHIDATABASE_HIGHER_SHIFT (0)
+
+#define MHIDATALIMIT_LOWER (0xA0)
+#define MHIDATALIMIT_LOWER_MHIDATALIMIT_LOWER_MASK (0xFFFFFFFF)
+#define MHIDATALIMIT_LOWER_MHIDATALIMIT_LOWER_SHIFT (0)
+
+#define MHIDATALIMIT_HIGHER (0xA4)
+#define MHIDATALIMIT_HIGHER_MHIDATALIMIT_HIGHER_MASK (0xFFFFFFFF)
+#define MHIDATALIMIT_HIGHER_MHIDATALIMIT_HIGHER_SHIFT (0)
+
+/* Host request register */
+#define MHI_SOC_RESET_REQ_OFFSET (0xB0)
+#define MHI_SOC_RESET_REQ BIT(0)
+
+/* MHI misc capability registers */
+#define MISC_OFFSET (0x24)
+#define MISC_CAP_MASK (0xFFFFFFFF)
+#define MISC_CAP_SHIFT (0)
+
+#define CAP_CAPID_MASK (0xFF000000)
+#define CAP_CAPID_SHIFT (24)
+#define CAP_NEXT_CAP_MASK (0x00FFF000)
+#define CAP_NEXT_CAP_SHIFT (12)
+
+/* MHI Timesync offsets */
+#define TIMESYNC_CFG_OFFSET (0x00)
+#define TIMESYNC_CFG_CAPID_MASK (CAP_CAPID_MASK)
+#define TIMESYNC_CFG_CAPID_SHIFT (CAP_CAPID_SHIFT)
+#define TIMESYNC_CFG_NEXT_OFF_MASK (CAP_NEXT_CAP_MASK)
+#define TIMESYNC_CFG_NEXT_OFF_SHIFT (CAP_NEXT_CAP_SHIFT)
+#define TIMESYNC_CFG_NUMCMD_MASK (0xFF)
+#define TIMESYNC_CFG_NUMCMD_SHIFT (0)
+#define TIMESYNC_DB_OFFSET (0x4)
+#define TIMESYNC_TIME_LOW_OFFSET (0x8)
+#define TIMESYNC_TIME_HIGH_OFFSET (0xC)
+
+#define TIMESYNC_CAP_ID (2)
+
+/* MHI BHI offfsets */
+#define BHI_BHIVERSION_MINOR (0x00)
+#define BHI_BHIVERSION_MAJOR (0x04)
+#define BHI_IMGADDR_LOW (0x08)
+#define BHI_IMGADDR_HIGH (0x0C)
+#define BHI_IMGSIZE (0x10)
+#define BHI_RSVD1 (0x14)
+#define BHI_IMGTXDB (0x18)
+#define BHI_TXDB_SEQNUM_BMSK (0x3FFFFFFF)
+#define BHI_TXDB_SEQNUM_SHFT (0)
+#define BHI_RSVD2 (0x1C)
+#define BHI_INTVEC (0x20)
+#define BHI_RSVD3 (0x24)
+#define BHI_EXECENV (0x28)
+#define BHI_STATUS (0x2C)
+#define BHI_ERRCODE (0x30)
+#define BHI_ERRDBG1 (0x34)
+#define BHI_ERRDBG2 (0x38)
+#define BHI_ERRDBG3 (0x3C)
+#define BHI_SERIALNU (0x40)
+#define BHI_SBLANTIROLLVER (0x44)
+#define BHI_NUMSEG (0x48)
+#define BHI_MSMHWID(n) (0x4C + (0x4 * n))
+#define BHI_OEMPKHASH(n) (0x64 + (0x4 * n))
+#define BHI_RSVD5 (0xC4)
+#define BHI_STATUS_MASK (0xC0000000)
+#define BHI_STATUS_SHIFT (30)
+#define BHI_STATUS_ERROR (3)
+#define BHI_STATUS_SUCCESS (2)
+#define BHI_STATUS_RESET (0)
+
+/* MHI BHIE offsets */
+#define BHIE_MSMSOCID_OFFS (0x0000)
+#define BHIE_TXVECADDR_LOW_OFFS (0x002C)
+#define BHIE_TXVECADDR_HIGH_OFFS (0x0030)
+#define BHIE_TXVECSIZE_OFFS (0x0034)
+#define BHIE_TXVECDB_OFFS (0x003C)
+#define BHIE_TXVECDB_SEQNUM_BMSK (0x3FFFFFFF)
+#define BHIE_TXVECDB_SEQNUM_SHFT (0)
+#define BHIE_TXVECSTATUS_OFFS (0x0044)
+#define BHIE_TXVECSTATUS_SEQNUM_BMSK (0x3FFFFFFF)
+#define BHIE_TXVECSTATUS_SEQNUM_SHFT (0)
+#define BHIE_TXVECSTATUS_STATUS_BMSK (0xC0000000)
+#define BHIE_TXVECSTATUS_STATUS_SHFT (30)
+#define BHIE_TXVECSTATUS_STATUS_RESET (0x00)
+#define BHIE_TXVECSTATUS_STATUS_XFER_COMPL (0x02)
+#define BHIE_TXVECSTATUS_STATUS_ERROR (0x03)
+#define BHIE_RXVECADDR_LOW_OFFS (0x0060)
+#define BHIE_RXVECADDR_HIGH_OFFS (0x0064)
+#define BHIE_RXVECSIZE_OFFS (0x0068)
+#define BHIE_RXVECDB_OFFS (0x0070)
+#define BHIE_RXVECDB_SEQNUM_BMSK (0x3FFFFFFF)
+#define BHIE_RXVECDB_SEQNUM_SHFT (0)
+#define BHIE_RXVECSTATUS_OFFS (0x0078)
+#define BHIE_RXVECSTATUS_SEQNUM_BMSK (0x3FFFFFFF)
+#define BHIE_RXVECSTATUS_SEQNUM_SHFT (0)
+#define BHIE_RXVECSTATUS_STATUS_BMSK (0xC0000000)
+#define BHIE_RXVECSTATUS_STATUS_SHFT (30)
+#define BHIE_RXVECSTATUS_STATUS_RESET (0x00)
+#define BHIE_RXVECSTATUS_STATUS_XFER_COMPL (0x02)
+#define BHIE_RXVECSTATUS_STATUS_ERROR (0x03)
+
+/* convert ticks to micro seconds by dividing by 19.2 */
+#define TIME_TICKS_TO_US(x) (div_u64((x) * 10, 192))
+
+struct mhi_event_ctxt {
+ u32 reserved:8;
+ u32 intmodc:8;
+ u32 intmodt:16;
+ u32 ertype;
+ u32 msivec;
+
+ u64 rbase __packed __aligned(4);
+ u64 rlen __packed __aligned(4);
+ u64 rp __packed __aligned(4);
+ u64 wp __packed __aligned(4);
+};
+
+struct mhi_chan_ctxt {
+ u32 chstate:8;
+ u32 brstmode:2;
+ u32 pollcfg:6;
+ u32 reserved:16;
+ u32 chtype;
+ u32 erindex;
+
+ u64 rbase __packed __aligned(4);
+ u64 rlen __packed __aligned(4);
+ u64 rp __packed __aligned(4);
+ u64 wp __packed __aligned(4);
+};
+
+struct mhi_cmd_ctxt {
+ u32 reserved0;
+ u32 reserved1;
+ u32 reserved2;
+
+ u64 rbase __packed __aligned(4);
+ u64 rlen __packed __aligned(4);
+ u64 rp __packed __aligned(4);
+ u64 wp __packed __aligned(4);
+};
+
+struct mhi_tre {
+ u64 ptr;
+ u32 dword[2];
+};
+
+struct bhi_vec_entry {
+ u64 dma_addr;
+ u64 size;
+};
+
+enum mhi_cmd_type {
+ MHI_CMD_TYPE_NOP = 1,
+ MHI_CMD_TYPE_RESET = 16,
+ MHI_CMD_TYPE_STOP = 17,
+ MHI_CMD_TYPE_START = 18,
+ MHI_CMD_TYPE_TSYNC = 24,
+};
+
+/* no operation command */
+#define MHI_TRE_CMD_NOOP_PTR (0)
+#define MHI_TRE_CMD_NOOP_DWORD0 (0)
+#define MHI_TRE_CMD_NOOP_DWORD1 (MHI_CMD_TYPE_NOP << 16)
+
+/* channel reset command */
+#define MHI_TRE_CMD_RESET_PTR (0)
+#define MHI_TRE_CMD_RESET_DWORD0 (0)
+#define MHI_TRE_CMD_RESET_DWORD1(chid) ((chid << 24) | \
+ (MHI_CMD_TYPE_RESET << 16))
+
+/* channel stop command */
+#define MHI_TRE_CMD_STOP_PTR (0)
+#define MHI_TRE_CMD_STOP_DWORD0 (0)
+#define MHI_TRE_CMD_STOP_DWORD1(chid) ((chid << 24) | (MHI_CMD_TYPE_STOP << 16))
+
+/* channel start command */
+#define MHI_TRE_CMD_START_PTR (0)
+#define MHI_TRE_CMD_START_DWORD0 (0)
+#define MHI_TRE_CMD_START_DWORD1(chid) ((chid << 24) | \
+ (MHI_CMD_TYPE_START << 16))
+
+/* time sync cfg command */
+#define MHI_TRE_CMD_TSYNC_CFG_PTR (0)
+#define MHI_TRE_CMD_TSYNC_CFG_DWORD0 (0)
+#define MHI_TRE_CMD_TSYNC_CFG_DWORD1(er) ((MHI_CMD_TYPE_TSYNC << 16) | \
+ (er << 24))
+
+#define MHI_TRE_GET_CMD_CHID(tre) (((tre)->dword[1] >> 24) & 0xFF)
+#define MHI_TRE_GET_CMD_TYPE(tre) (((tre)->dword[1] >> 16) & 0xFF)
+
+/* event descriptor macros */
+#define MHI_TRE_EV_PTR(ptr) (ptr)
+#define MHI_TRE_EV_DWORD0(code, len) ((code << 24) | len)
+#define MHI_TRE_EV_DWORD1(chid, type) ((chid << 24) | (type << 16))
+#define MHI_TRE_GET_EV_PTR(tre) ((tre)->ptr)
+#define MHI_TRE_GET_EV_CODE(tre) (((tre)->dword[0] >> 24) & 0xFF)
+#define MHI_TRE_GET_EV_LEN(tre) ((tre)->dword[0] & 0xFFFF)
+#define MHI_TRE_GET_EV_CHID(tre) (((tre)->dword[1] >> 24) & 0xFF)
+#define MHI_TRE_GET_EV_TYPE(tre) (((tre)->dword[1] >> 16) & 0xFF)
+#define MHI_TRE_GET_EV_STATE(tre) (((tre)->dword[0] >> 24) & 0xFF)
+#define MHI_TRE_GET_EV_EXECENV(tre) (((tre)->dword[0] >> 24) & 0xFF)
+#define MHI_TRE_GET_EV_SEQ(tre) ((tre)->dword[0])
+#define MHI_TRE_GET_EV_TIME(tre) ((tre)->ptr)
+#define MHI_TRE_GET_EV_COOKIE(tre) lower_32_bits((tre)->ptr)
+#define MHI_TRE_GET_EV_VEID(tre) (((tre)->dword[0] >> 16) & 0xFF)
+
+/* transfer descriptor macros */
+#define MHI_TRE_DATA_PTR(ptr) (ptr)
+#define MHI_TRE_DATA_DWORD0(len) (len & MHI_MAX_MTU)
+#define MHI_TRE_DATA_DWORD1(bei, ieot, ieob, chain) ((2 << 16) | (bei << 10) \
+ | (ieot << 9) | (ieob << 8) | chain)
+
+/* rsc transfer descriptor macros */
+#define MHI_RSCTRE_DATA_PTR(ptr, len) (((u64)len << 48) | ptr)
+#define MHI_RSCTRE_DATA_DWORD0(cookie) (cookie)
+#define MHI_RSCTRE_DATA_DWORD1 (MHI_PKT_TYPE_COALESCING << 16)
+
+enum MHI_CMD {
+ MHI_CMD_RESET_CHAN,
+ MHI_CMD_START_CHAN,
+ MHI_CMD_TIMSYNC_CFG,
+};
+
+enum MHI_PKT_TYPE {
+ MHI_PKT_TYPE_INVALID = 0x0,
+ MHI_PKT_TYPE_NOOP_CMD = 0x1,
+ MHI_PKT_TYPE_TRANSFER = 0x2,
+ MHI_PKT_TYPE_COALESCING = 0x8,
+ MHI_PKT_TYPE_RESET_CHAN_CMD = 0x10,
+ MHI_PKT_TYPE_STOP_CHAN_CMD = 0x11,
+ MHI_PKT_TYPE_START_CHAN_CMD = 0x12,
+ MHI_PKT_TYPE_STATE_CHANGE_EVENT = 0x20,
+ MHI_PKT_TYPE_CMD_COMPLETION_EVENT = 0x21,
+ MHI_PKT_TYPE_TX_EVENT = 0x22,
+ MHI_PKT_TYPE_RSC_TX_EVENT = 0x28,
+ MHI_PKT_TYPE_EE_EVENT = 0x40,
+ MHI_PKT_TYPE_TSYNC_EVENT = 0x48,
+ MHI_PKT_TYPE_STALE_EVENT, /* internal event */
+};
+
+/* MHI transfer completion events */
+enum MHI_EV_CCS {
+ MHI_EV_CC_INVALID = 0x0,
+ MHI_EV_CC_SUCCESS = 0x1,
+ MHI_EV_CC_EOT = 0x2,
+ MHI_EV_CC_OVERFLOW = 0x3,
+ MHI_EV_CC_EOB = 0x4,
+ MHI_EV_CC_OOB = 0x5,
+ MHI_EV_CC_DB_MODE = 0x6,
+ MHI_EV_CC_UNDEFINED_ERR = 0x10,
+ MHI_EV_CC_BAD_TRE = 0x11,
+};
+
+enum MHI_CH_STATE {
+ MHI_CH_STATE_DISABLED = 0x0,
+ MHI_CH_STATE_ENABLED = 0x1,
+ MHI_CH_STATE_RUNNING = 0x2,
+ MHI_CH_STATE_SUSPENDED = 0x3,
+ MHI_CH_STATE_STOP = 0x4,
+ MHI_CH_STATE_ERROR = 0x5,
+};
+
+enum MHI_BRSTMODE {
+ MHI_BRSTMODE_DISABLE = 0x2,
+ MHI_BRSTMODE_ENABLE = 0x3,
+};
+
+#define MHI_INVALID_BRSTMODE(mode) (mode != MHI_BRSTMODE_DISABLE && \
+ mode != MHI_BRSTMODE_ENABLE)
+
+#define MHI_IN_PBL(ee) (ee == MHI_EE_PBL || ee == MHI_EE_PTHRU || \
+ ee == MHI_EE_EDL)
+
+#define MHI_IN_MISSION_MODE(ee) (ee == MHI_EE_AMSS || ee == MHI_EE_WFW)
+
+enum MHI_ST_TRANSITION {
+ MHI_ST_TRANSITION_PBL,
+ MHI_ST_TRANSITION_READY,
+ MHI_ST_TRANSITION_SBL,
+ MHI_ST_TRANSITION_MISSION_MODE,
+ MHI_ST_TRANSITION_MAX,
+};
+
+extern const char *const mhi_state_tran_str[MHI_ST_TRANSITION_MAX];
+#define TO_MHI_STATE_TRANS_STR(state) (((state) >= MHI_ST_TRANSITION_MAX) ? \
+ "INVALID_STATE" : mhi_state_tran_str[state])
+
+extern const char *const mhi_state_str[MHI_STATE_MAX];
+#define TO_MHI_STATE_STR(state) ((state >= MHI_STATE_MAX || \
+ !mhi_state_str[state]) ? \
+ "INVALID_STATE" : mhi_state_str[state])
+
+enum {
+ MHI_PM_BIT_DISABLE,
+ MHI_PM_BIT_POR,
+ MHI_PM_BIT_M0,
+ MHI_PM_BIT_M2,
+ MHI_PM_BIT_M3_ENTER,
+ MHI_PM_BIT_M3,
+ MHI_PM_BIT_M3_EXIT,
+ MHI_PM_BIT_FW_DL_ERR,
+ MHI_PM_BIT_SYS_ERR_DETECT,
+ MHI_PM_BIT_SYS_ERR_PROCESS,
+ MHI_PM_BIT_SHUTDOWN_PROCESS,
+ MHI_PM_BIT_LD_ERR_FATAL_DETECT,
+ MHI_PM_BIT_MAX
+};
+
+/* internal power states */
+enum MHI_PM_STATE {
+ MHI_PM_DISABLE = BIT(MHI_PM_BIT_DISABLE), /* MHI is not enabled */
+ MHI_PM_POR = BIT(MHI_PM_BIT_POR), /* reset state */
+ MHI_PM_M0 = BIT(MHI_PM_BIT_M0),
+ MHI_PM_M2 = BIT(MHI_PM_BIT_M2),
+ MHI_PM_M3_ENTER = BIT(MHI_PM_BIT_M3_ENTER),
+ MHI_PM_M3 = BIT(MHI_PM_BIT_M3),
+ MHI_PM_M3_EXIT = BIT(MHI_PM_BIT_M3_EXIT),
+ /* firmware download failure state */
+ MHI_PM_FW_DL_ERR = BIT(MHI_PM_BIT_FW_DL_ERR),
+ MHI_PM_SYS_ERR_DETECT = BIT(MHI_PM_BIT_SYS_ERR_DETECT),
+ MHI_PM_SYS_ERR_PROCESS = BIT(MHI_PM_BIT_SYS_ERR_PROCESS),
+ MHI_PM_SHUTDOWN_PROCESS = BIT(MHI_PM_BIT_SHUTDOWN_PROCESS),
+ /* link not accessible */
+ MHI_PM_LD_ERR_FATAL_DETECT = BIT(MHI_PM_BIT_LD_ERR_FATAL_DETECT),
+};
+
+#define MHI_REG_ACCESS_VALID(pm_state) ((pm_state & (MHI_PM_POR | MHI_PM_M0 | \
+ MHI_PM_M2 | MHI_PM_M3_ENTER | MHI_PM_M3_EXIT | \
+ MHI_PM_SYS_ERR_DETECT | MHI_PM_SYS_ERR_PROCESS | \
+ MHI_PM_SHUTDOWN_PROCESS | MHI_PM_FW_DL_ERR)))
+#define MHI_PM_IN_ERROR_STATE(pm_state) (pm_state >= MHI_PM_FW_DL_ERR)
+#define MHI_PM_IN_FATAL_STATE(pm_state) (pm_state == MHI_PM_LD_ERR_FATAL_DETECT)
+#define MHI_DB_ACCESS_VALID(pm_state) (pm_state & MHI_PM_M0)
+#define MHI_WAKE_DB_CLEAR_VALID(pm_state) (pm_state & (MHI_PM_M0 | \
+ MHI_PM_M2 | MHI_PM_M3_EXIT))
+#define MHI_WAKE_DB_SET_VALID(pm_state) (pm_state & MHI_PM_M2)
+#define MHI_WAKE_DB_FORCE_SET_VALID(pm_state) MHI_WAKE_DB_CLEAR_VALID(pm_state)
+#define MHI_EVENT_ACCESS_INVALID(pm_state) (pm_state == MHI_PM_DISABLE || \
+ MHI_PM_IN_ERROR_STATE(pm_state))
+#define MHI_PM_IN_SUSPEND_STATE(pm_state) (pm_state & \
+ (MHI_PM_M3_ENTER | MHI_PM_M3))
+
+/* accepted buffer type for the channel */
+enum MHI_XFER_TYPE {
+ MHI_XFER_BUFFER,
+ MHI_XFER_SKB,
+ MHI_XFER_SCLIST,
+ MHI_XFER_NOP, /* CPU offload channel, host does not accept transfer */
+ MHI_XFER_DMA, /* receive dma address, already mapped by client */
+ MHI_XFER_RSC_DMA, /* RSC type, accept premapped buffer */
+};
+
+#define NR_OF_CMD_RINGS (1)
+#define CMD_EL_PER_RING (128)
+#define PRIMARY_CMD_RING (0)
+#define MHI_DEV_WAKE_DB (127)
+#define MHI_MAX_MTU (0xffff)
+
+enum MHI_ER_TYPE {
+ MHI_ER_TYPE_INVALID = 0x0,
+ MHI_ER_TYPE_VALID = 0x1,
+};
+
+enum mhi_er_data_type {
+ MHI_ER_DATA_ELEMENT_TYPE,
+ MHI_ER_CTRL_ELEMENT_TYPE,
+ MHI_ER_TSYNC_ELEMENT_TYPE,
+ MHI_ER_DATA_TYPE_MAX = MHI_ER_TSYNC_ELEMENT_TYPE,
+};
+
+enum mhi_ch_ee_mask {
+ MHI_CH_EE_PBL = BIT(MHI_EE_PBL),
+ MHI_CH_EE_SBL = BIT(MHI_EE_SBL),
+ MHI_CH_EE_AMSS = BIT(MHI_EE_AMSS),
+ MHI_CH_EE_RDDM = BIT(MHI_EE_RDDM),
+ MHI_CH_EE_PTHRU = BIT(MHI_EE_PTHRU),
+ MHI_CH_EE_WFW = BIT(MHI_EE_WFW),
+ MHI_CH_EE_EDL = BIT(MHI_EE_EDL),
+};
+
+enum mhi_ch_type {
+ MHI_CH_TYPE_INVALID = 0,
+ MHI_CH_TYPE_OUTBOUND = DMA_TO_DEVICE,
+ MHI_CH_TYPE_INBOUND = DMA_FROM_DEVICE,
+ MHI_CH_TYPE_INBOUND_COALESCED = 3,
+};
+
+struct db_cfg {
+ bool reset_req;
+ bool db_mode;
+ u32 pollcfg;
+ enum MHI_BRSTMODE brstmode;
+ dma_addr_t db_val;
+ void (*process_db)(struct mhi_controller *mhi_cntrl,
+ struct db_cfg *db_cfg, void __iomem *io_addr,
+ dma_addr_t db_val);
+};
+
+struct mhi_pm_transitions {
+ enum MHI_PM_STATE from_state;
+ u32 to_states;
+};
+
+struct state_transition {
+ struct list_head node;
+ enum MHI_ST_TRANSITION state;
+};
+
+struct mhi_ctxt {
+ struct mhi_event_ctxt *er_ctxt;
+ struct mhi_chan_ctxt *chan_ctxt;
+ struct mhi_cmd_ctxt *cmd_ctxt;
+ dma_addr_t er_ctxt_addr;
+ dma_addr_t chan_ctxt_addr;
+ dma_addr_t cmd_ctxt_addr;
+};
+
+struct mhi_ring {
+ dma_addr_t dma_handle;
+ dma_addr_t iommu_base;
+ u64 *ctxt_wp; /* point to ctxt wp */
+ void *pre_aligned;
+ void *base;
+ void *rptr;
+ void *wptr;
+ size_t el_size;
+ size_t len;
+ size_t elements;
+ size_t alloc_size;
+ void __iomem *db_addr;
+};
+
+struct mhi_cmd {
+ struct mhi_ring ring;
+ spinlock_t lock;
+};
+
+struct mhi_buf_info {
+ dma_addr_t p_addr;
+ void *v_addr;
+ void *bb_addr;
+ void *wp;
+ size_t len;
+ void *cb_buf;
+ bool used; /* indicate element is free to use */
+ bool pre_mapped; /* already pre-mapped by client */
+ enum dma_data_direction dir;
+};
+
+struct mhi_event {
+ u32 er_index;
+ u32 intmod;
+ u32 msi;
+ int chan; /* this event ring is dedicated to a channel */
+ u32 priority;
+ enum mhi_er_data_type data_type;
+ struct mhi_ring ring;
+ struct db_cfg db_cfg;
+ bool hw_ring;
+ bool cl_manage;
+ bool offload_ev; /* managed by a device driver */
+ spinlock_t lock;
+ struct mhi_chan *mhi_chan; /* dedicated to channel */
+ struct tasklet_struct task;
+ int (*process_event)(struct mhi_controller *mhi_cntrl,
+ struct mhi_event *mhi_event, u32 event_quota);
+ struct mhi_controller *mhi_cntrl;
+};
+
+struct mhi_chan {
+ u32 chan;
+ const char *name;
+ /*
+ * important, when consuming increment tre_ring first, when releasing
+ * decrement buf_ring first. If tre_ring has space, buf_ring
+ * guranteed to have space so we do not need to check both rings.
+ */
+ struct mhi_ring buf_ring;
+ struct mhi_ring tre_ring;
+ u32 er_index;
+ u32 intmod;
+ enum mhi_ch_type type;
+ enum dma_data_direction dir;
+ struct db_cfg db_cfg;
+ u32 ee_mask;
+ enum MHI_XFER_TYPE xfer_type;
+ enum MHI_CH_STATE ch_state;
+ enum MHI_EV_CCS ccs;
+ bool lpm_notify;
+ bool configured;
+ bool offload_ch;
+ bool pre_alloc;
+ bool auto_start;
+ bool wake_capable; /* channel should wake up system */
+ /* functions that generate the transfer ring elements */
+ int (*gen_tre)(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan, void *buf, void *cb,
+ size_t len, enum MHI_FLAGS flags);
+ int (*queue_xfer)(struct mhi_device *mhi_dev,
+ struct mhi_chan *mhi_chan, void *buf,
+ size_t len, enum MHI_FLAGS flags);
+ /* xfer call back */
+ struct mhi_device *mhi_dev;
+ void (*xfer_cb)(struct mhi_device *mhi_dev,
+ struct mhi_result *result);
+ struct mutex mutex;
+ struct completion completion;
+ rwlock_t lock;
+ struct list_head node;
+};
+
+struct tsync_node {
+ struct list_head node;
+ u32 sequence;
+ u64 local_time;
+ u64 remote_time;
+ struct mhi_device *mhi_dev;
+ void (*cb_func)(struct mhi_device *mhi_dev, u32 sequence,
+ u64 local_time, u64 remote_time);
+};
+
+struct mhi_timesync {
+ u32 er_index;
+ void __iomem *db;
+ void __iomem *time_reg;
+ enum MHI_EV_CCS ccs;
+ struct completion completion;
+ spinlock_t lock; /* list protection */
+ struct mutex lpm_mutex; /* lpm protection */
+ struct list_head head;
+};
+
+struct mhi_bus {
+ struct list_head controller_list;
+ struct mutex lock;
+};
+
+/* default MHI timeout */
+#define MHI_TIMEOUT_MS (1000)
+extern struct mhi_bus mhi_bus;
+
+/* debug fs related functions */
+int mhi_debugfs_mhi_chan_show(struct seq_file *m, void *d);
+int mhi_debugfs_mhi_event_show(struct seq_file *m, void *d);
+int mhi_debugfs_mhi_states_show(struct seq_file *m, void *d);
+int mhi_debugfs_trigger_reset(void *data, u64 val);
+
+void mhi_deinit_debugfs(struct mhi_controller *mhi_cntrl);
+void mhi_init_debugfs(struct mhi_controller *mhi_cntrl);
+
+/* power management apis */
+enum MHI_PM_STATE __must_check mhi_tryset_pm_state(struct mhi_controller
+ *mhi_cntrl,
+ enum MHI_PM_STATE state);
+const char *to_mhi_pm_state_str(enum MHI_PM_STATE state);
+void mhi_reset_chan(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan);
+enum mhi_ee mhi_get_exec_env(struct mhi_controller *mhi_cntrl);
+int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl,
+ enum MHI_ST_TRANSITION state);
+void mhi_pm_st_worker(struct work_struct *work);
+void mhi_fw_load_worker(struct work_struct *work);
+void mhi_pm_sys_err_worker(struct work_struct *work);
+int mhi_ready_state_transition(struct mhi_controller *mhi_cntrl);
+void mhi_ctrl_ev_task(unsigned long data);
+int mhi_pm_m0_transition(struct mhi_controller *mhi_cntrl);
+void mhi_pm_m1_transition(struct mhi_controller *mhi_cntrl);
+int mhi_pm_m3_transition(struct mhi_controller *mhi_cntrl);
+void mhi_notify(struct mhi_device *mhi_dev, enum MHI_CB cb_reason);
+int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl,
+ struct mhi_event *mhi_event, u32 event_quota);
+int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
+ struct mhi_event *mhi_event, u32 event_quota);
+int mhi_process_tsync_event_ring(struct mhi_controller *mhi_cntrl,
+ struct mhi_event *mhi_event, u32 event_quota);
+int mhi_send_cmd(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
+ enum MHI_CMD cmd);
+int __mhi_device_get_sync(struct mhi_controller *mhi_cntrl);
+
+/* queue transfer buffer */
+int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan,
+ void *buf, void *cb, size_t buf_len, enum MHI_FLAGS flags);
+int mhi_queue_buf(struct mhi_device *mhi_dev, struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS mflags);
+int mhi_queue_skb(struct mhi_device *mhi_dev, struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS mflags);
+int mhi_queue_sclist(struct mhi_device *mhi_dev, struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS mflags);
+int mhi_queue_nop(struct mhi_device *mhi_dev, struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS mflags);
+int mhi_queue_dma(struct mhi_device *mhi_dev, struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS mflags);
+
+/* register access methods */
+void mhi_db_brstmode(struct mhi_controller *mhi_cntrl, struct db_cfg *db_cfg,
+ void __iomem *db_addr, dma_addr_t db_wp);
+void mhi_db_brstmode_disable(struct mhi_controller *mhi_cntrl,
+ struct db_cfg *db_mode, void __iomem *db_addr,
+ dma_addr_t db_wp);
+int __must_check mhi_read_reg(struct mhi_controller *mhi_cntrl,
+ void __iomem *base, u32 offset, u32 *out);
+int __must_check mhi_read_reg_field(struct mhi_controller *mhi_cntrl,
+ void __iomem *base, u32 offset, u32 mask,
+ u32 shift, u32 *out);
+void mhi_write_reg(struct mhi_controller *mhi_cntrl, void __iomem *base,
+ u32 offset, u32 val);
+void mhi_write_reg_field(struct mhi_controller *mhi_cntrl, void __iomem *base,
+ u32 offset, u32 mask, u32 shift, u32 val);
+void mhi_ring_er_db(struct mhi_event *mhi_event);
+void mhi_write_db(struct mhi_controller *mhi_cntrl, void __iomem *db_addr,
+ dma_addr_t db_wp);
+void mhi_ring_cmd_db(struct mhi_controller *mhi_cntrl, struct mhi_cmd *mhi_cmd);
+void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan);
+int mhi_get_capability_offset(struct mhi_controller *mhi_cntrl, u32 capability,
+ u32 *offset);
+int mhi_init_timesync(struct mhi_controller *mhi_cntrl);
+int mhi_create_timesync_sysfs(struct mhi_controller *mhi_cntrl);
+void mhi_destroy_timesync(struct mhi_controller *mhi_cntrl);
+
+static inline void *dma_zalloc_coherent(struct device *dev, size_t size,
+ dma_addr_t *dma_handle, gfp_t flag)
+{
+ void *ret = dma_alloc_coherent(dev, size, dma_handle,
+ flag | __GFP_ZERO);
+ return ret;
+}
+
+/* memory allocation methods */
+static inline void *mhi_alloc_coherent(struct mhi_controller *mhi_cntrl,
+ size_t size,
+ dma_addr_t *dma_handle, gfp_t gfp)
+{
+ void *buf = dma_zalloc_coherent(mhi_cntrl->dev, size, dma_handle, gfp);
+
+ if (buf)
+ atomic_add(size, &mhi_cntrl->alloc_size);
+
+ return buf;
+}
+
+static inline void mhi_free_coherent(struct mhi_controller *mhi_cntrl,
+ size_t size,
+ void *vaddr, dma_addr_t dma_handle)
+{
+ atomic_sub(size, &mhi_cntrl->alloc_size);
+ dma_free_coherent(mhi_cntrl->dev, size, vaddr, dma_handle);
+}
+
+struct mhi_device *mhi_alloc_device(struct mhi_controller *mhi_cntrl);
+static inline void mhi_dealloc_device(struct mhi_controller *mhi_cntrl,
+ struct mhi_device *mhi_dev)
+{
+ kfree(mhi_dev);
+}
+
+int mhi_destroy_device(struct device *dev, void *data);
+void mhi_create_devices(struct mhi_controller *mhi_cntrl);
+int mhi_alloc_bhie_table(struct mhi_controller *mhi_cntrl,
+ struct image_info **image_info, size_t alloc_size);
+void mhi_free_bhie_table(struct mhi_controller *mhi_cntrl,
+ struct image_info *image_info);
+
+int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl,
+ struct mhi_buf_info *buf_info);
+int mhi_map_single_use_bb(struct mhi_controller *mhi_cntrl,
+ struct mhi_buf_info *buf_info);
+void mhi_unmap_single_no_bb(struct mhi_controller *mhi_cntrl,
+ struct mhi_buf_info *buf_info);
+void mhi_unmap_single_use_bb(struct mhi_controller *mhi_cntrl,
+ struct mhi_buf_info *buf_info);
+
+/* initialization methods */
+int mhi_init_chan_ctxt(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan);
+void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan);
+int mhi_init_mmio(struct mhi_controller *mhi_cntrl);
+int mhi_init_dev_ctxt(struct mhi_controller *mhi_cntrl);
+void mhi_deinit_dev_ctxt(struct mhi_controller *mhi_cntrl);
+int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl);
+void mhi_deinit_free_irq(struct mhi_controller *mhi_cntrl);
+int mhi_dtr_init(void);
+
+/* isr handlers */
+irqreturn_t mhi_msi_handlr(int irq_number, void *dev);
+irqreturn_t mhi_intvec_threaded_handlr(int irq_number, void *dev);
+irqreturn_t mhi_intvec_handlr(int irq_number, void *dev);
+void mhi_ev_task(unsigned long data);
+
+#ifdef CONFIG_MHI_DEBUG
+
+#define MHI_ASSERT(cond, msg) do { \
+ if (cond) \
+ panic(msg); \
+} while (0)
+
+#else
+
+#define MHI_ASSERT(cond, msg) do { \
+ if (cond) { \
+ MHI_ERR(msg); \
+ WARN_ON(cond); \
+ } \
+} while (0)
+
+#endif
+
+#endif /* _MHI_INT_H */
diff --git a/drivers/bus/mhi/core/mhi_main.c b/drivers/bus/mhi/core/mhi_main.c
new file mode 100644
index 000000000000..de362b9272c2
--- /dev/null
+++ b/drivers/bus/mhi/core/mhi_main.c
@@ -0,0 +1,2261 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. */
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/of.h>
+#include <linux/module.h>
+#include <linux/skbuff.h>
+#include <linux/slab.h>
+#include <linux/mhi.h>
+#include "mhi_internal.h"
+
+static void __mhi_unprepare_channel(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan);
+
+int __must_check mhi_read_reg(struct mhi_controller *mhi_cntrl,
+ void __iomem *base, u32 offset, u32 *out)
+{
+ u32 tmp = readl_relaxed(base + offset);
+
+ /* unexpected value, query the link status */
+ if (PCI_INVALID_READ(tmp) &&
+ mhi_cntrl->link_status(mhi_cntrl, mhi_cntrl->priv_data))
+ return -EIO;
+
+ *out = tmp;
+
+ return 0;
+}
+
+int __must_check mhi_read_reg_field(struct mhi_controller *mhi_cntrl,
+ void __iomem *base,
+ u32 offset, u32 mask, u32 shift, u32 *out)
+{
+ u32 tmp;
+ int ret;
+
+ ret = mhi_read_reg(mhi_cntrl, base, offset, &tmp);
+ if (ret)
+ return ret;
+
+ *out = (tmp & mask) >> shift;
+
+ return 0;
+}
+
+int mhi_get_capability_offset(struct mhi_controller *mhi_cntrl,
+ u32 capability, u32 *offset)
+{
+ u32 cur_cap, next_offset;
+ int ret;
+
+ /* get the 1st supported capability offset */
+ ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, MISC_OFFSET,
+ MISC_CAP_MASK, MISC_CAP_SHIFT, offset);
+ if (ret)
+ return ret;
+ do {
+ ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, *offset,
+ CAP_CAPID_MASK, CAP_CAPID_SHIFT,
+ &cur_cap);
+ if (ret)
+ return ret;
+
+ if (cur_cap == capability)
+ return 0;
+
+ ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, *offset,
+ CAP_NEXT_CAP_MASK, CAP_NEXT_CAP_SHIFT,
+ &next_offset);
+ if (ret)
+ return ret;
+
+ *offset += next_offset;
+ } while (next_offset);
+
+ return -ENXIO;
+}
+
+void mhi_write_reg(struct mhi_controller *mhi_cntrl,
+ void __iomem *base, u32 offset, u32 val)
+{
+ writel_relaxed(val, base + offset);
+}
+
+void mhi_write_reg_field(struct mhi_controller *mhi_cntrl,
+ void __iomem *base,
+ u32 offset, u32 mask, u32 shift, u32 val)
+{
+ int ret;
+ u32 tmp;
+
+ ret = mhi_read_reg(mhi_cntrl, base, offset, &tmp);
+ if (ret)
+ return;
+
+ tmp &= ~mask;
+ tmp |= (val << shift);
+ mhi_write_reg(mhi_cntrl, base, offset, tmp);
+}
+
+void mhi_write_db(struct mhi_controller *mhi_cntrl,
+ void __iomem *db_addr, dma_addr_t wp)
+{
+ mhi_write_reg(mhi_cntrl, db_addr, 4, upper_32_bits(wp));
+ mhi_write_reg(mhi_cntrl, db_addr, 0, lower_32_bits(wp));
+}
+
+void mhi_db_brstmode(struct mhi_controller *mhi_cntrl,
+ struct db_cfg *db_cfg,
+ void __iomem *db_addr, dma_addr_t wp)
+{
+ if (db_cfg->db_mode) {
+ db_cfg->db_val = wp;
+ mhi_write_db(mhi_cntrl, db_addr, wp);
+ db_cfg->db_mode = false;
+ }
+}
+
+void mhi_db_brstmode_disable(struct mhi_controller *mhi_cntrl,
+ struct db_cfg *db_cfg,
+ void __iomem *db_addr, dma_addr_t wp)
+{
+ db_cfg->db_val = wp;
+ mhi_write_db(mhi_cntrl, db_addr, wp);
+}
+
+void mhi_ring_er_db(struct mhi_event *mhi_event)
+{
+ struct mhi_ring *ring = &mhi_event->ring;
+
+ mhi_event->db_cfg.process_db(mhi_event->mhi_cntrl, &mhi_event->db_cfg,
+ ring->db_addr, *ring->ctxt_wp);
+}
+
+void mhi_ring_cmd_db(struct mhi_controller *mhi_cntrl, struct mhi_cmd *mhi_cmd)
+{
+ dma_addr_t db;
+ struct mhi_ring *ring = &mhi_cmd->ring;
+
+ db = ring->iommu_base + (ring->wptr - ring->base);
+ *ring->ctxt_wp = db;
+ mhi_write_db(mhi_cntrl, ring->db_addr, db);
+}
+
+void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan)
+{
+ struct mhi_ring *ring = &mhi_chan->tre_ring;
+ dma_addr_t db;
+
+ db = ring->iommu_base + (ring->wptr - ring->base);
+ *ring->ctxt_wp = db;
+ mhi_chan->db_cfg.process_db(mhi_cntrl, &mhi_chan->db_cfg, ring->db_addr,
+ db);
+}
+
+enum mhi_ee mhi_get_exec_env(struct mhi_controller *mhi_cntrl)
+{
+ u32 exec;
+ int ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_EXECENV, &exec);
+
+ return (ret) ? MHI_EE_MAX : exec;
+}
+
+enum mhi_dev_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl)
+{
+ u32 state;
+ int ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, MHISTATUS,
+ MHISTATUS_MHISTATE_MASK,
+ MHISTATUS_MHISTATE_SHIFT, &state);
+ return ret ? MHI_STATE_MAX : state;
+}
+EXPORT_SYMBOL(mhi_get_mhi_state);
+
+int mhi_queue_sclist(struct mhi_device *mhi_dev,
+ struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS mflags)
+{
+ return -EINVAL;
+}
+
+int mhi_queue_nop(struct mhi_device *mhi_dev,
+ struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS mflags)
+{
+ return -EINVAL;
+}
+
+static void mhi_add_ring_element(struct mhi_controller *mhi_cntrl,
+ struct mhi_ring *ring)
+{
+ ring->wptr += ring->el_size;
+ if (ring->wptr >= (ring->base + ring->len))
+ ring->wptr = ring->base;
+ /* smp update */
+ smp_wmb();
+}
+
+static void mhi_del_ring_element(struct mhi_controller *mhi_cntrl,
+ struct mhi_ring *ring)
+{
+ ring->rptr += ring->el_size;
+ if (ring->rptr >= (ring->base + ring->len))
+ ring->rptr = ring->base;
+ /* smp update */
+ smp_wmb();
+}
+
+static int get_nr_avail_ring_elements(struct mhi_controller *mhi_cntrl,
+ struct mhi_ring *ring)
+{
+ int nr_el;
+
+ if (ring->wptr < ring->rptr)
+ nr_el = ((ring->rptr - ring->wptr) / ring->el_size) - 1;
+ else {
+ nr_el = (ring->rptr - ring->base) / ring->el_size;
+ nr_el += ((ring->base + ring->len - ring->wptr) /
+ ring->el_size) - 1;
+ }
+ return nr_el;
+}
+
+static void *mhi_to_virtual(struct mhi_ring *ring, dma_addr_t addr)
+{
+ return (addr - ring->iommu_base) + ring->base;
+}
+
+dma_addr_t mhi_to_physical(struct mhi_ring *ring, void *addr)
+{
+ return (addr - ring->base) + ring->iommu_base;
+}
+
+static void mhi_recycle_ev_ring_element(struct mhi_controller *mhi_cntrl,
+ struct mhi_ring *ring)
+{
+ dma_addr_t ctxt_wp;
+
+ /* update the WP */
+ ring->wptr += ring->el_size;
+ ctxt_wp = *ring->ctxt_wp + ring->el_size;
+
+ if (ring->wptr >= (ring->base + ring->len)) {
+ ring->wptr = ring->base;
+ ctxt_wp = ring->iommu_base;
+ }
+
+ *ring->ctxt_wp = ctxt_wp;
+
+ /* update the RP */
+ ring->rptr += ring->el_size;
+ if (ring->rptr >= (ring->base + ring->len))
+ ring->rptr = ring->base;
+
+ /* visible to other cores */
+ smp_wmb();
+}
+
+static bool mhi_is_ring_full(struct mhi_controller *mhi_cntrl,
+ struct mhi_ring *ring)
+{
+ void *tmp = ring->wptr + ring->el_size;
+
+ if (tmp >= (ring->base + ring->len))
+ tmp = ring->base;
+
+ return (tmp == ring->rptr);
+}
+
+int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl,
+ struct mhi_buf_info *buf_info)
+{
+ buf_info->p_addr = dma_map_single(mhi_cntrl->dev, buf_info->v_addr,
+ buf_info->len, buf_info->dir);
+ if (dma_mapping_error(mhi_cntrl->dev, buf_info->p_addr))
+ return -ENOMEM;
+
+ return 0;
+}
+
+int mhi_map_single_use_bb(struct mhi_controller *mhi_cntrl,
+ struct mhi_buf_info *buf_info)
+{
+ void *buf = mhi_alloc_coherent(mhi_cntrl, buf_info->len,
+ &buf_info->p_addr, GFP_ATOMIC);
+
+ if (!buf)
+ return -ENOMEM;
+
+ if (buf_info->dir == DMA_TO_DEVICE)
+ memcpy(buf, buf_info->v_addr, buf_info->len);
+
+ buf_info->bb_addr = buf;
+
+ return 0;
+}
+
+void mhi_unmap_single_no_bb(struct mhi_controller *mhi_cntrl,
+ struct mhi_buf_info *buf_info)
+{
+ dma_unmap_single(mhi_cntrl->dev, buf_info->p_addr, buf_info->len,
+ buf_info->dir);
+}
+
+void mhi_unmap_single_use_bb(struct mhi_controller *mhi_cntrl,
+ struct mhi_buf_info *buf_info)
+{
+ if (buf_info->dir == DMA_FROM_DEVICE)
+ memcpy(buf_info->v_addr, buf_info->bb_addr, buf_info->len);
+
+ mhi_free_coherent(mhi_cntrl, buf_info->len, buf_info->bb_addr,
+ buf_info->p_addr);
+}
+
+int mhi_queue_skb(struct mhi_device *mhi_dev,
+ struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS mflags)
+{
+ struct sk_buff *skb = buf;
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
+ struct mhi_ring *buf_ring = &mhi_chan->buf_ring;
+ struct mhi_buf_info *buf_info;
+ struct mhi_tre *mhi_tre;
+ bool assert_wake = false;
+ int ret;
+
+ if (mhi_is_ring_full(mhi_cntrl, tre_ring))
+ return -ENOMEM;
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) {
+ MHI_VERB("MHI is not in activate state, pm_state:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return -EIO;
+ }
+
+ /* we're in M3 or transitioning to M3 */
+ if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) {
+ mhi_cntrl->runtime_get(mhi_cntrl, mhi_cntrl->priv_data);
+ mhi_cntrl->runtime_put(mhi_cntrl, mhi_cntrl->priv_data);
+ }
+
+ /*
+ * For UL channels always assert WAKE until work is done,
+ * For DL channels only assert if MHI is in a LPM
+ */
+ if (mhi_chan->dir == DMA_TO_DEVICE ||
+ (mhi_chan->dir == DMA_FROM_DEVICE &&
+ mhi_cntrl->pm_state != MHI_PM_M0)) {
+ assert_wake = true;
+ mhi_cntrl->wake_get(mhi_cntrl, false);
+ }
+
+ /* generate the tre */
+ buf_info = buf_ring->wptr;
+ buf_info->v_addr = skb->data;
+ buf_info->cb_buf = skb;
+ buf_info->wp = tre_ring->wptr;
+ buf_info->dir = mhi_chan->dir;
+ buf_info->len = len;
+ ret = mhi_cntrl->map_single(mhi_cntrl, buf_info);
+ if (ret)
+ goto map_error;
+
+ mhi_tre = tre_ring->wptr;
+
+ mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
+ mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(buf_info->len);
+ mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(1, 1, 0, 0);
+
+ MHI_VERB("chan:%d WP:0x%llx TRE:0x%llx 0x%08x 0x%08x\n", mhi_chan->chan,
+ (u64)mhi_to_physical(tre_ring, mhi_tre), mhi_tre->ptr,
+ mhi_tre->dword[0], mhi_tre->dword[1]);
+
+ /* increment WP */
+ mhi_add_ring_element(mhi_cntrl, tre_ring);
+ mhi_add_ring_element(mhi_cntrl, buf_ring);
+
+ if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl->pm_state))) {
+ read_lock_bh(&mhi_chan->lock);
+ mhi_ring_chan_db(mhi_cntrl, mhi_chan);
+ read_unlock_bh(&mhi_chan->lock);
+ }
+
+ if (mhi_chan->dir == DMA_FROM_DEVICE && assert_wake)
+ mhi_cntrl->wake_put(mhi_cntrl, true);
+
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return 0;
+
+map_error:
+ if (assert_wake)
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return ret;
+}
+
+int mhi_queue_dma(struct mhi_device *mhi_dev,
+ struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS mflags)
+{
+ struct mhi_buf *mhi_buf = buf;
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
+ struct mhi_ring *buf_ring = &mhi_chan->buf_ring;
+ struct mhi_buf_info *buf_info;
+ struct mhi_tre *mhi_tre;
+ bool assert_wake = false;
+
+ if (mhi_is_ring_full(mhi_cntrl, tre_ring))
+ return -ENOMEM;
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) {
+ MHI_VERB("MHI is not in activate state, pm_state:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return -EIO;
+ }
+
+ /* we're in M3 or transitioning to M3 */
+ if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) {
+ mhi_cntrl->runtime_get(mhi_cntrl, mhi_cntrl->priv_data);
+ mhi_cntrl->runtime_put(mhi_cntrl, mhi_cntrl->priv_data);
+ }
+
+ /*
+ * For UL channels always assert WAKE until work is done,
+ * For DL channels only assert if MHI is in a LPM
+ */
+ if (mhi_chan->dir == DMA_TO_DEVICE ||
+ (mhi_chan->dir == DMA_FROM_DEVICE &&
+ mhi_cntrl->pm_state != MHI_PM_M0)) {
+ assert_wake = true;
+ mhi_cntrl->wake_get(mhi_cntrl, false);
+ }
+
+ /* generate the tre */
+ buf_info = buf_ring->wptr;
+ MHI_ASSERT(buf_info->used, "TRE Not Freed\n");
+ buf_info->p_addr = mhi_buf->dma_addr;
+ buf_info->pre_mapped = true;
+ buf_info->cb_buf = mhi_buf;
+ buf_info->wp = tre_ring->wptr;
+ buf_info->dir = mhi_chan->dir;
+ buf_info->len = len;
+
+ mhi_tre = tre_ring->wptr;
+
+ if (mhi_chan->xfer_type == MHI_XFER_RSC_DMA) {
+ buf_info->used = true;
+ mhi_tre->ptr =
+ MHI_RSCTRE_DATA_PTR(buf_info->p_addr, buf_info->len);
+ mhi_tre->dword[0] =
+ MHI_RSCTRE_DATA_DWORD0(buf_ring->wptr - buf_ring->base);
+ mhi_tre->dword[1] = MHI_RSCTRE_DATA_DWORD1;
+ } else {
+ mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
+ mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(buf_info->len);
+ mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(1, 1, 0, 0);
+ }
+
+ MHI_VERB("chan:%d WP:0x%llx TRE:0x%llx 0x%08x 0x%08x\n", mhi_chan->chan,
+ (u64)mhi_to_physical(tre_ring, mhi_tre), mhi_tre->ptr,
+ mhi_tre->dword[0], mhi_tre->dword[1]);
+
+ /* increment WP */
+ mhi_add_ring_element(mhi_cntrl, tre_ring);
+ mhi_add_ring_element(mhi_cntrl, buf_ring);
+
+ if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl->pm_state))) {
+ read_lock_bh(&mhi_chan->lock);
+ mhi_ring_chan_db(mhi_cntrl, mhi_chan);
+ read_unlock_bh(&mhi_chan->lock);
+ }
+
+ if (mhi_chan->dir == DMA_FROM_DEVICE && assert_wake)
+ mhi_cntrl->wake_put(mhi_cntrl, true);
+
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return 0;
+}
+
+int mhi_gen_tre(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan,
+ void *buf, void *cb, size_t buf_len, enum MHI_FLAGS flags)
+{
+ struct mhi_ring *buf_ring, *tre_ring;
+ struct mhi_tre *mhi_tre;
+ struct mhi_buf_info *buf_info;
+ int eot, eob, chain, bei;
+ int ret;
+
+ buf_ring = &mhi_chan->buf_ring;
+ tre_ring = &mhi_chan->tre_ring;
+
+ buf_info = buf_ring->wptr;
+ buf_info->v_addr = buf;
+ buf_info->cb_buf = cb;
+ buf_info->wp = tre_ring->wptr;
+ buf_info->dir = mhi_chan->dir;
+ buf_info->len = buf_len;
+
+ ret = mhi_cntrl->map_single(mhi_cntrl, buf_info);
+ if (ret)
+ return ret;
+
+ eob = !!(flags & MHI_EOB);
+ eot = !!(flags & MHI_EOT);
+ chain = !!(flags & MHI_CHAIN);
+ bei = !!(mhi_chan->intmod);
+
+ mhi_tre = tre_ring->wptr;
+ mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr);
+ mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(buf_len);
+ mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(bei, eot, eob, chain);
+
+ MHI_VERB("chan:%d WP:0x%llx TRE:0x%llx 0x%08x 0x%08x\n", mhi_chan->chan,
+ (u64)mhi_to_physical(tre_ring, mhi_tre), mhi_tre->ptr,
+ mhi_tre->dword[0], mhi_tre->dword[1]);
+
+ /* increment WP */
+ mhi_add_ring_element(mhi_cntrl, tre_ring);
+ mhi_add_ring_element(mhi_cntrl, buf_ring);
+
+ return 0;
+}
+
+int mhi_queue_buf(struct mhi_device *mhi_dev,
+ struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS mflags)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_ring *tre_ring;
+ unsigned long flags;
+ bool assert_wake = false;
+ int ret;
+
+ /*
+ * this check here only as a guard, it's always
+ * possible mhi can enter error while executing rest of function,
+ * which is not fatal so we do not need to hold pm_lock
+ */
+ if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) {
+ MHI_VERB("MHI is not in active state, pm_state:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+
+ return -EIO;
+ }
+
+ tre_ring = &mhi_chan->tre_ring;
+ if (mhi_is_ring_full(mhi_cntrl, tre_ring))
+ return -ENOMEM;
+
+ ret = mhi_chan->gen_tre(mhi_cntrl, mhi_chan, buf, buf, len, mflags);
+ if (unlikely(ret))
+ return ret;
+
+ read_lock_irqsave(&mhi_cntrl->pm_lock, flags);
+
+ /* we're in M3 or transitioning to M3 */
+ if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) {
+ mhi_cntrl->runtime_get(mhi_cntrl, mhi_cntrl->priv_data);
+ mhi_cntrl->runtime_put(mhi_cntrl, mhi_cntrl->priv_data);
+ }
+
+ /*
+ * For UL channels always assert WAKE until work is done,
+ * For DL channels only assert if MHI is in a LPM
+ */
+ if (mhi_chan->dir == DMA_TO_DEVICE ||
+ (mhi_chan->dir == DMA_FROM_DEVICE &&
+ mhi_cntrl->pm_state != MHI_PM_M0)) {
+ assert_wake = true;
+ mhi_cntrl->wake_get(mhi_cntrl, false);
+ }
+
+ if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl->pm_state))) {
+ unsigned long flags;
+
+ read_lock_irqsave(&mhi_chan->lock, flags);
+ mhi_ring_chan_db(mhi_cntrl, mhi_chan);
+ read_unlock_irqrestore(&mhi_chan->lock, flags);
+ }
+
+ if (mhi_chan->dir == DMA_FROM_DEVICE && assert_wake)
+ mhi_cntrl->wake_put(mhi_cntrl, true);
+
+ read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags);
+
+ return 0;
+}
+
+/* destroy specific device */
+int mhi_destroy_device(struct device *dev, void *data)
+{
+ struct mhi_device *mhi_dev;
+ struct mhi_controller *mhi_cntrl;
+
+ if (dev->bus != &mhi_bus_type)
+ return 0;
+
+ mhi_dev = to_mhi_device(dev);
+ mhi_cntrl = mhi_dev->mhi_cntrl;
+
+ /* only destroying virtual devices thats attached to bus */
+ if (mhi_dev->dev_type == MHI_CONTROLLER_TYPE)
+ return 0;
+
+ MHI_LOG("destroy device for chan:%s\n", mhi_dev->chan_name);
+
+ /* notify the client and remove the device from mhi bus */
+ device_del(dev);
+ put_device(dev);
+
+ return 0;
+}
+
+void mhi_notify(struct mhi_device *mhi_dev, enum MHI_CB cb_reason)
+{
+ struct mhi_driver *mhi_drv;
+
+ if (!mhi_dev->dev.driver)
+ return;
+
+ mhi_drv = to_mhi_driver(mhi_dev->dev.driver);
+
+ if (mhi_drv->status_cb)
+ mhi_drv->status_cb(mhi_dev, cb_reason);
+}
+
+static void mhi_assign_of_node(struct mhi_controller *mhi_cntrl,
+ struct mhi_device *mhi_dev)
+{
+ struct device_node *controller, *node;
+ const char *dt_name;
+ int ret;
+
+ controller = of_find_node_by_name(mhi_cntrl->of_node, "mhi_devices");
+ if (!controller)
+ return;
+
+ for_each_available_child_of_node(controller, node) {
+ ret = of_property_read_string(node, "mhi,chan", &dt_name);
+ if (ret)
+ continue;
+ if (!strcmp(mhi_dev->chan_name, dt_name)) {
+ mhi_dev->dev.of_node = node;
+ break;
+ }
+ }
+}
+
+static ssize_t time_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct mhi_device *mhi_dev = to_mhi_device(dev);
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ u64 t_host, t_device;
+ int ret;
+
+ ret = mhi_get_remote_time_sync(mhi_dev, &t_host, &t_device);
+ if (ret) {
+ MHI_ERR("Failed to obtain time, ret:%d\n", ret);
+ return ret;
+ }
+
+ return scnprintf(buf, PAGE_SIZE, "local: %llu remote: %llu (ticks)\n",
+ t_host, t_device);
+}
+
+static DEVICE_ATTR_RO(time);
+
+static ssize_t time_us_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct mhi_device *mhi_dev = to_mhi_device(dev);
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ u64 t_host, t_device;
+ int ret;
+
+ ret = mhi_get_remote_time_sync(mhi_dev, &t_host, &t_device);
+ if (ret) {
+ MHI_ERR("Failed to obtain time, ret:%d\n", ret);
+ return ret;
+ }
+
+ return scnprintf(buf, PAGE_SIZE, "local: %llu remote: %llu (us)\n",
+ TIME_TICKS_TO_US(t_host), TIME_TICKS_TO_US(t_device));
+}
+
+static DEVICE_ATTR_RO(time_us);
+
+static struct attribute *mhi_tsync_attrs[] = {
+ &dev_attr_time.attr,
+ &dev_attr_time_us.attr,
+ NULL,
+};
+
+static const struct attribute_group mhi_tsync_group = {
+ .attrs = mhi_tsync_attrs,
+};
+
+void mhi_destroy_timesync(struct mhi_controller *mhi_cntrl)
+{
+ if (mhi_cntrl->mhi_tsync) {
+ sysfs_remove_group(&mhi_cntrl->mhi_dev->dev.kobj,
+ &mhi_tsync_group);
+ kfree(mhi_cntrl->mhi_tsync);
+ mhi_cntrl->mhi_tsync = NULL;
+ }
+}
+
+int mhi_create_timesync_sysfs(struct mhi_controller *mhi_cntrl)
+{
+ return sysfs_create_group(&mhi_cntrl->mhi_dev->dev.kobj,
+ &mhi_tsync_group);
+}
+
+static void mhi_create_time_sync_dev(struct mhi_controller *mhi_cntrl)
+{
+ struct mhi_device *mhi_dev;
+ int ret;
+
+ if (!MHI_IN_MISSION_MODE(mhi_cntrl->ee))
+ return;
+
+ mhi_dev = mhi_alloc_device(mhi_cntrl);
+ if (!mhi_dev)
+ return;
+
+ mhi_dev->dev_type = MHI_TIMESYNC_TYPE;
+ mhi_dev->chan_name = "TIME_SYNC";
+ dev_set_name(&mhi_dev->dev, "%04x_%02u.%02u.%02u_%s", mhi_dev->dev_id,
+ mhi_dev->domain, mhi_dev->bus, mhi_dev->slot,
+ mhi_dev->chan_name);
+
+ /* add if there is a matching DT node */
+ mhi_assign_of_node(mhi_cntrl, mhi_dev);
+
+ ret = device_add(&mhi_dev->dev);
+ if (ret) {
+ MHI_ERR("Failed to register dev for chan:%s\n",
+ mhi_dev->chan_name);
+ mhi_dealloc_device(mhi_cntrl, mhi_dev);
+ return;
+ }
+
+ mhi_cntrl->tsync_dev = mhi_dev;
+}
+
+/* bind mhi channels into mhi devices */
+void mhi_create_devices(struct mhi_controller *mhi_cntrl)
+{
+ int i;
+ struct mhi_chan *mhi_chan;
+ struct mhi_device *mhi_dev;
+ int ret;
+
+ /*
+ * we need to create time sync device before creating other
+ * devices, because client may try to capture time during
+ * clint probe.
+ */
+ mhi_create_time_sync_dev(mhi_cntrl);
+
+ mhi_chan = mhi_cntrl->mhi_chan;
+ for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) {
+ if (!mhi_chan->configured || mhi_chan->mhi_dev ||
+ !(mhi_chan->ee_mask & BIT(mhi_cntrl->ee)))
+ continue;
+ mhi_dev = mhi_alloc_device(mhi_cntrl);
+ if (!mhi_dev)
+ return;
+
+ mhi_dev->dev_type = MHI_XFER_TYPE;
+ switch (mhi_chan->dir) {
+ case DMA_TO_DEVICE:
+ mhi_dev->ul_chan = mhi_chan;
+ mhi_dev->ul_chan_id = mhi_chan->chan;
+ mhi_dev->ul_xfer = mhi_chan->queue_xfer;
+ mhi_dev->ul_event_id = mhi_chan->er_index;
+ break;
+ case DMA_NONE:
+ case DMA_BIDIRECTIONAL:
+ mhi_dev->ul_chan_id = mhi_chan->chan;
+ mhi_dev->ul_event_id = mhi_chan->er_index;
+ fallthrough;
+ case DMA_FROM_DEVICE:
+ /* we use dl_chan for offload channels */
+ mhi_dev->dl_chan = mhi_chan;
+ mhi_dev->dl_chan_id = mhi_chan->chan;
+ mhi_dev->dl_xfer = mhi_chan->queue_xfer;
+ mhi_dev->dl_event_id = mhi_chan->er_index;
+ }
+
+ mhi_chan->mhi_dev = mhi_dev;
+
+ /* check next channel if it matches */
+ if ((i + 1) < mhi_cntrl->max_chan && mhi_chan[1].configured) {
+ if (!strcmp(mhi_chan[1].name, mhi_chan->name)) {
+ i++;
+ mhi_chan++;
+ if (mhi_chan->dir == DMA_TO_DEVICE) {
+ mhi_dev->ul_chan = mhi_chan;
+ mhi_dev->ul_chan_id = mhi_chan->chan;
+ mhi_dev->ul_xfer = mhi_chan->queue_xfer;
+ mhi_dev->ul_event_id =
+ mhi_chan->er_index;
+ } else {
+ mhi_dev->dl_chan = mhi_chan;
+ mhi_dev->dl_chan_id = mhi_chan->chan;
+ mhi_dev->dl_xfer = mhi_chan->queue_xfer;
+ mhi_dev->dl_event_id =
+ mhi_chan->er_index;
+ }
+ mhi_chan->mhi_dev = mhi_dev;
+ }
+ }
+
+ mhi_dev->chan_name = mhi_chan->name;
+ dev_set_name(&mhi_dev->dev, "%04x_%02u.%02u.%02u_%s",
+ mhi_dev->dev_id, mhi_dev->domain, mhi_dev->bus,
+ mhi_dev->slot, mhi_dev->chan_name);
+
+ /* add if there is a matching DT node */
+ mhi_assign_of_node(mhi_cntrl, mhi_dev);
+
+ /* init wake source */
+ if (mhi_dev->dl_chan && mhi_dev->dl_chan->wake_capable)
+ device_init_wakeup(&mhi_dev->dev, true);
+
+ ret = device_add(&mhi_dev->dev);
+ if (ret) {
+ MHI_ERR("Failed to register dev for chan:%s\n",
+ mhi_dev->chan_name);
+ mhi_dealloc_device(mhi_cntrl, mhi_dev);
+ }
+ }
+}
+
+static int parse_xfer_event(struct mhi_controller *mhi_cntrl,
+ struct mhi_tre *event, struct mhi_chan *mhi_chan)
+{
+ struct mhi_ring *buf_ring, *tre_ring;
+ u32 ev_code;
+ struct mhi_result result;
+ unsigned long flags = 0;
+
+ ev_code = MHI_TRE_GET_EV_CODE(event);
+ buf_ring = &mhi_chan->buf_ring;
+ tre_ring = &mhi_chan->tre_ring;
+
+ result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ?
+ -EOVERFLOW : 0;
+
+ /*
+ * if it's a DB Event then we need to grab the lock
+ * with preemption disable and as a write because we
+ * have to update db register and another thread could
+ * be doing same.
+ */
+ if (ev_code >= MHI_EV_CC_OOB)
+ write_lock_irqsave(&mhi_chan->lock, flags);
+ else
+ read_lock_bh(&mhi_chan->lock);
+
+ if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED)
+ goto end_process_tx_event;
+
+ switch (ev_code) {
+ case MHI_EV_CC_OVERFLOW:
+ case MHI_EV_CC_EOB:
+ case MHI_EV_CC_EOT:
+ {
+ dma_addr_t ptr = MHI_TRE_GET_EV_PTR(event);
+ struct mhi_tre *local_rp, *ev_tre;
+ void *dev_rp;
+ struct mhi_buf_info *buf_info;
+ u16 xfer_len;
+
+ /* Get the TRB this event points to */
+ ev_tre = mhi_to_virtual(tre_ring, ptr);
+
+ /* device rp after servicing the TREs */
+ dev_rp = ev_tre + 1;
+ if (dev_rp >= (tre_ring->base + tre_ring->len))
+ dev_rp = tre_ring->base;
+
+ result.dir = mhi_chan->dir;
+
+ /* local rp */
+ local_rp = tre_ring->rptr;
+ while (local_rp != dev_rp) {
+ buf_info = buf_ring->rptr;
+ /* Always get the get len from the event */
+ xfer_len = MHI_TRE_GET_EV_LEN(event);
+
+ /* unmap if it's not premapped by client */
+ if (likely(!buf_info->pre_mapped))
+ mhi_cntrl->unmap_single(mhi_cntrl, buf_info);
+
+ result.buf_addr = buf_info->cb_buf;
+ result.bytes_xferd = xfer_len;
+ mhi_del_ring_element(mhi_cntrl, buf_ring);
+ mhi_del_ring_element(mhi_cntrl, tre_ring);
+ local_rp = tre_ring->rptr;
+
+ /* notify client */
+ mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
+
+ if (mhi_chan->dir == DMA_TO_DEVICE) {
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ }
+
+ /*
+ * recycle the buffer if buffer is pre-allocated,
+ * if there is error, not much we can do apart from
+ * dropping the packet
+ */
+ if (mhi_chan->pre_alloc) {
+ if (mhi_queue_buf(mhi_chan->mhi_dev, mhi_chan,
+ buf_info->cb_buf,
+ buf_info->len, MHI_EOT)) {
+ MHI_ERR
+ ("Error recycling buf, chan: %d\n",
+ mhi_chan->chan);
+ kfree(buf_info->cb_buf);
+ }
+ }
+ }
+ break;
+ } /* CC_EOT */
+ case MHI_EV_CC_OOB:
+ case MHI_EV_CC_DB_MODE:
+ {
+ unsigned long flags;
+
+ MHI_VERB("DB_MODE/OOB Detected chan %d.\n", mhi_chan->chan);
+ mhi_chan->db_cfg.db_mode = true;
+ read_lock_irqsave(&mhi_cntrl->pm_lock, flags);
+ if (tre_ring->wptr != tre_ring->rptr &&
+ MHI_DB_ACCESS_VALID(mhi_cntrl->pm_state)) {
+ mhi_ring_chan_db(mhi_cntrl, mhi_chan);
+ }
+ read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags);
+ break;
+ }
+ case MHI_EV_CC_BAD_TRE:
+ MHI_ASSERT(1, "Received BAD TRE event for ring");
+ break;
+ default:
+ MHI_CRITICAL("Unknown TX completion.\n");
+
+ break;
+ } /* switch(MHI_EV_READ_CODE(event)) */
+
+end_process_tx_event:
+ if (ev_code >= MHI_EV_CC_OOB)
+ write_unlock_irqrestore(&mhi_chan->lock, flags);
+ else
+ read_unlock_bh(&mhi_chan->lock);
+
+ return 0;
+}
+
+static int parse_rsc_event(struct mhi_controller *mhi_cntrl,
+ struct mhi_tre *event, struct mhi_chan *mhi_chan)
+{
+ struct mhi_ring *buf_ring, *tre_ring;
+ struct mhi_buf_info *buf_info;
+ struct mhi_result result;
+ int ev_code;
+ u32 cookie; /* offset to local descriptor */
+ u16 xfer_len;
+
+ buf_ring = &mhi_chan->buf_ring;
+ tre_ring = &mhi_chan->tre_ring;
+
+ ev_code = MHI_TRE_GET_EV_CODE(event);
+ cookie = MHI_TRE_GET_EV_COOKIE(event);
+ xfer_len = MHI_TRE_GET_EV_LEN(event);
+
+ /* received out of bound cookie */
+ MHI_ASSERT(cookie >= buf_ring->len, "Invalid Cookie\n");
+
+ buf_info = buf_ring->base + cookie;
+
+ result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ?
+ -EOVERFLOW : 0;
+ result.bytes_xferd = xfer_len;
+ result.buf_addr = buf_info->cb_buf;
+ result.dir = mhi_chan->dir;
+
+ read_lock_bh(&mhi_chan->lock);
+
+ if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED)
+ goto end_process_rsc_event;
+
+ MHI_ASSERT(!buf_info->used, "TRE already Freed\n");
+
+ /* notify the client */
+ mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
+
+ /*
+ * Note: We're arbitrarily incrementing RP even though, completion
+ * packet we processed might not be the same one, reason we can do this
+ * is because device guaranteed to cache descriptors in order it
+ * receive, so even though completion event is different we can re-use
+ * all descriptors in between.
+ * Example:
+ * Transfer Ring has descriptors: A, B, C, D
+ * Last descriptor host queue is D (WP) and first descriptor
+ * host queue is A (RP).
+ * The completion event we just serviced is descriptor C.
+ * Then we can safely queue descriptors to replace A, B, and C
+ * even though host did not receive any completions.
+ */
+ mhi_del_ring_element(mhi_cntrl, tre_ring);
+ buf_info->used = false;
+
+end_process_rsc_event:
+ read_unlock_bh(&mhi_chan->lock);
+
+ return 0;
+}
+
+static void mhi_process_cmd_completion(struct mhi_controller *mhi_cntrl,
+ struct mhi_tre *tre)
+{
+ dma_addr_t ptr = MHI_TRE_GET_EV_PTR(tre);
+ struct mhi_cmd *cmd_ring = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING];
+ struct mhi_ring *mhi_ring = &cmd_ring->ring;
+ struct mhi_tre *cmd_pkt;
+ struct mhi_chan *mhi_chan;
+ struct mhi_timesync *mhi_tsync;
+ enum mhi_cmd_type type;
+ u32 chan;
+
+ cmd_pkt = mhi_to_virtual(mhi_ring, ptr);
+
+ /* out of order completion received */
+ MHI_ASSERT(cmd_pkt != mhi_ring->rptr, "Out of order cmd completion");
+
+ type = MHI_TRE_GET_CMD_TYPE(cmd_pkt);
+
+ if (type == MHI_CMD_TYPE_TSYNC) {
+ mhi_tsync = mhi_cntrl->mhi_tsync;
+ mhi_tsync->ccs = MHI_TRE_GET_EV_CODE(tre);
+ complete(&mhi_tsync->completion);
+ } else {
+ chan = MHI_TRE_GET_CMD_CHID(cmd_pkt);
+ mhi_chan = &mhi_cntrl->mhi_chan[chan];
+ write_lock_bh(&mhi_chan->lock);
+ mhi_chan->ccs = MHI_TRE_GET_EV_CODE(tre);
+ complete(&mhi_chan->completion);
+ write_unlock_bh(&mhi_chan->lock);
+ }
+
+ mhi_del_ring_element(mhi_cntrl, mhi_ring);
+}
+
+static void __mhi_process_ctrl_ev_type(struct mhi_controller *mhi_cntrl,
+ struct mhi_tre *local_rp,
+ enum MHI_PKT_TYPE type)
+{
+ MHI_VERB("Processing Event:0x%llx 0x%08x 0x%08x\n",
+ local_rp->ptr, local_rp->dword[0], local_rp->dword[1]);
+
+ switch (type) {
+ case MHI_PKT_TYPE_STATE_CHANGE_EVENT:
+ {
+ enum mhi_dev_state new_state;
+
+ new_state = MHI_TRE_GET_EV_STATE(local_rp);
+
+ MHI_LOG("MHI state change event to state:%s\n",
+ TO_MHI_STATE_STR(new_state));
+
+ switch (new_state) {
+ case MHI_STATE_M0:
+ mhi_pm_m0_transition(mhi_cntrl);
+ break;
+ case MHI_STATE_M1:
+ mhi_pm_m1_transition(mhi_cntrl);
+ break;
+ case MHI_STATE_M3:
+ mhi_pm_m3_transition(mhi_cntrl);
+ break;
+ case MHI_STATE_SYS_ERR:
+ {
+ enum MHI_PM_STATE new_state;
+
+ MHI_ERR("MHI system error detected\n");
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ new_state = mhi_tryset_pm_state(mhi_cntrl,
+ MHI_PM_SYS_ERR_DETECT);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ if (new_state == MHI_PM_SYS_ERR_DETECT)
+ schedule_work(&mhi_cntrl->syserr_worker);
+ break;
+ }
+ default:
+ MHI_ERR("Unsupported STE:%s\n",
+ TO_MHI_STATE_STR(new_state));
+ }
+
+ break;
+ }
+ case MHI_PKT_TYPE_CMD_COMPLETION_EVENT:
+ mhi_process_cmd_completion(mhi_cntrl, local_rp);
+ break;
+ case MHI_PKT_TYPE_EE_EVENT:
+ {
+ enum MHI_ST_TRANSITION st = MHI_ST_TRANSITION_MAX;
+ enum mhi_ee event = MHI_TRE_GET_EV_EXECENV(local_rp);
+
+ MHI_LOG("MHI EE received event:%s\n",
+ TO_MHI_EXEC_STR(event));
+ switch (event) {
+ case MHI_EE_SBL:
+ st = MHI_ST_TRANSITION_SBL;
+ break;
+ case MHI_EE_WFW:
+ case MHI_EE_AMSS:
+ st = MHI_ST_TRANSITION_MISSION_MODE;
+ break;
+ case MHI_EE_RDDM:
+ mhi_cntrl->status_cb(mhi_cntrl, mhi_cntrl->priv_data,
+ MHI_CB_EE_RDDM);
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ mhi_cntrl->ee = event;
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ wake_up_all(&mhi_cntrl->state_event);
+ break;
+ default:
+ MHI_ERR("Unhandled EE event:%s\n",
+ TO_MHI_EXEC_STR(event));
+ }
+ if (st != MHI_ST_TRANSITION_MAX)
+ mhi_queue_state_transition(mhi_cntrl, st);
+
+ break;
+ }
+ default:
+ MHI_ASSERT(1, "Unsupported ev type");
+ break;
+ }
+}
+
+int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl,
+ struct mhi_event *mhi_event, u32 event_quota)
+{
+ struct mhi_tre *dev_rp, *local_rp;
+ struct mhi_ring *ev_ring = &mhi_event->ring;
+ struct mhi_event_ctxt *er_ctxt =
+ &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
+ int count = 0;
+
+ /*
+ * this is a quick check to avoid unnecessary event processing
+ * in case we already in error state, but it's still possible
+ * to transition to error state while processing events
+ */
+ if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state))) {
+ MHI_ERR("No EV access, PM_STATE:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return -EIO;
+ }
+
+ dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp);
+ local_rp = ev_ring->rptr;
+
+ while (dev_rp != local_rp) {
+ enum MHI_PKT_TYPE type = MHI_TRE_GET_EV_TYPE(local_rp);
+
+ __mhi_process_ctrl_ev_type(mhi_cntrl, local_rp, type);
+
+ mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
+ local_rp = ev_ring->rptr;
+ dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp);
+ count++;
+ }
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl->pm_state)))
+ mhi_ring_er_db(mhi_event);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ MHI_VERB("exit er_index:%u\n", mhi_event->er_index);
+
+ return count;
+}
+
+int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl,
+ struct mhi_event *mhi_event, u32 event_quota)
+{
+ struct mhi_tre *dev_rp, *local_rp;
+ struct mhi_ring *ev_ring = &mhi_event->ring;
+ struct mhi_event_ctxt *er_ctxt =
+ &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
+ int count = 0;
+ u32 chan;
+ struct mhi_chan *mhi_chan;
+
+ if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state))) {
+ MHI_ERR("No EV access, PM_STATE:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return -EIO;
+ }
+
+ dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp);
+ local_rp = ev_ring->rptr;
+
+ while (dev_rp != local_rp && event_quota > 0) {
+ enum MHI_PKT_TYPE type = MHI_TRE_GET_EV_TYPE(local_rp);
+
+ MHI_VERB("Processing Event:0x%llx 0x%08x 0x%08x\n",
+ local_rp->ptr, local_rp->dword[0], local_rp->dword[1]);
+
+ chan = MHI_TRE_GET_EV_CHID(local_rp);
+ mhi_chan = &mhi_cntrl->mhi_chan[chan];
+
+ if (likely(type == MHI_PKT_TYPE_TX_EVENT)) {
+ parse_xfer_event(mhi_cntrl, local_rp, mhi_chan);
+ event_quota--;
+ } else if (type == MHI_PKT_TYPE_RSC_TX_EVENT) {
+ parse_rsc_event(mhi_cntrl, local_rp, mhi_chan);
+ event_quota--;
+ }
+
+ mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
+ local_rp = ev_ring->rptr;
+ dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp);
+ count++;
+ }
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl->pm_state)))
+ mhi_ring_er_db(mhi_event);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ MHI_VERB("exit er_index:%u\n", mhi_event->er_index);
+
+ return count;
+}
+
+int mhi_process_tsync_event_ring(struct mhi_controller *mhi_cntrl,
+ struct mhi_event *mhi_event, u32 event_quota)
+{
+ struct mhi_tre *dev_rp, *local_rp;
+ struct mhi_ring *ev_ring = &mhi_event->ring;
+ struct mhi_event_ctxt *er_ctxt =
+ &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
+ struct mhi_timesync *mhi_tsync = mhi_cntrl->mhi_tsync;
+ int count = 0;
+ u32 sequence;
+ u64 remote_time;
+
+ if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state))) {
+ MHI_ERR("No EV access, PM_STATE:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ return -EIO;
+ }
+
+ dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp);
+ local_rp = ev_ring->rptr;
+
+ while (dev_rp != local_rp) {
+ enum MHI_PKT_TYPE type = MHI_TRE_GET_EV_TYPE(local_rp);
+ struct tsync_node *tsync_node;
+
+ MHI_VERB("Processing Event:0x%llx 0x%08x 0x%08x\n",
+ local_rp->ptr, local_rp->dword[0], local_rp->dword[1]);
+
+ MHI_ASSERT(type != MHI_PKT_TYPE_TSYNC_EVENT, "!TSYNC event");
+
+ sequence = MHI_TRE_GET_EV_SEQ(local_rp);
+ remote_time = MHI_TRE_GET_EV_TIME(local_rp);
+
+ do {
+ spin_lock_irq(&mhi_tsync->lock);
+ tsync_node = list_first_entry_or_null(&mhi_tsync->head,
+ struct tsync_node,
+ node);
+ MHI_ASSERT(!tsync_node, "Unexpected Event");
+
+ if (unlikely(!tsync_node))
+ break;
+
+ list_del(&tsync_node->node);
+ spin_unlock_irq(&mhi_tsync->lock);
+
+ /*
+ * device may not able to process all time sync commands
+ * host issue and only process last command it receive
+ */
+ if (tsync_node->sequence == sequence) {
+ tsync_node->cb_func(tsync_node->mhi_dev,
+ sequence,
+ tsync_node->local_time,
+ remote_time);
+ kfree(tsync_node);
+ } else {
+ kfree(tsync_node);
+ }
+ } while (true);
+
+ mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring);
+ local_rp = ev_ring->rptr;
+ dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp);
+ count++;
+ }
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl->pm_state)))
+ mhi_ring_er_db(mhi_event);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ MHI_VERB("exit er_index:%u\n", mhi_event->er_index);
+
+ return count;
+}
+
+void mhi_ev_task(unsigned long data)
+{
+ struct mhi_event *mhi_event = (struct mhi_event *)data;
+ struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl;
+
+ MHI_VERB("Enter for ev_index:%d\n", mhi_event->er_index);
+
+ /* process all pending events */
+ spin_lock_bh(&mhi_event->lock);
+ mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX);
+ spin_unlock_bh(&mhi_event->lock);
+}
+
+void mhi_ctrl_ev_task(unsigned long data)
+{
+ struct mhi_event *mhi_event = (struct mhi_event *)data;
+ struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl;
+ enum mhi_dev_state state;
+ enum MHI_PM_STATE pm_state = 0;
+ int ret;
+
+ MHI_VERB("Enter for ev_index:%d\n", mhi_event->er_index);
+
+ /*
+ * we can check pm_state w/o a lock here because there is no way
+ * pm_state can change from reg access valid to no access while this
+ * therad being executed.
+ */
+ if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state))
+ return;
+
+ /* process ctrl events */
+ ret = mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX);
+
+ /*
+ * we received a MSI but no events to process maybe device went to
+ * SYS_ERR state, check the state
+ */
+ if (!ret) {
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ state = mhi_get_mhi_state(mhi_cntrl);
+ if (state == MHI_STATE_SYS_ERR) {
+ MHI_ERR("MHI system error detected\n");
+ pm_state = mhi_tryset_pm_state(mhi_cntrl,
+ MHI_PM_SYS_ERR_DETECT);
+ }
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ if (pm_state == MHI_PM_SYS_ERR_DETECT)
+ schedule_work(&mhi_cntrl->syserr_worker);
+ }
+}
+
+irqreturn_t mhi_msi_handlr(int irq_number, void *dev)
+{
+ struct mhi_event *mhi_event = dev;
+ struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl;
+ struct mhi_event_ctxt *er_ctxt =
+ &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index];
+ struct mhi_ring *ev_ring = &mhi_event->ring;
+ void *dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp);
+
+ /* confirm ER has pending events to process before scheduling work */
+ if (ev_ring->rptr == dev_rp)
+ return IRQ_HANDLED;
+
+ /* client managed event ring, notify pending data */
+ if (mhi_event->cl_manage) {
+ struct mhi_chan *mhi_chan = mhi_event->mhi_chan;
+ struct mhi_device *mhi_dev = mhi_chan->mhi_dev;
+
+ if (mhi_dev)
+ mhi_dev->status_cb(mhi_dev, MHI_CB_PENDING_DATA);
+ } else
+ tasklet_schedule(&mhi_event->task);
+
+ return IRQ_HANDLED;
+}
+
+/* this is the threaded fn */
+irqreturn_t mhi_intvec_threaded_handlr(int irq_number, void *dev)
+{
+ struct mhi_controller *mhi_cntrl = dev;
+ enum mhi_dev_state state = MHI_STATE_MAX;
+ enum MHI_PM_STATE pm_state = 0;
+ enum mhi_ee ee;
+
+ MHI_VERB("Enter\n");
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) {
+ state = mhi_get_mhi_state(mhi_cntrl);
+ ee = mhi_get_exec_env(mhi_cntrl);
+ MHI_LOG("device ee:%s dev_state:%s\n", TO_MHI_EXEC_STR(ee),
+ TO_MHI_STATE_STR(state));
+ }
+
+ /* tim.hc.lin, testing */
+ if (ee == MHI_EE_AMSS && state == MHI_STATE_READY)
+ mhi_queue_state_transition(mhi_cntrl, MHI_ST_TRANSITION_READY);
+
+ if (state == MHI_STATE_SYS_ERR) {
+ MHI_ERR("MHI system error detected\n");
+ pm_state = mhi_tryset_pm_state(mhi_cntrl,
+ MHI_PM_SYS_ERR_DETECT);
+ }
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ if (pm_state == MHI_PM_SYS_ERR_DETECT) {
+ wake_up_all(&mhi_cntrl->state_event);
+
+ /* for fatal errors, we let controller decide next step */
+ if (MHI_IN_PBL(ee))
+ mhi_cntrl->status_cb(mhi_cntrl, mhi_cntrl->priv_data,
+ MHI_CB_FATAL_ERROR);
+ else
+ schedule_work(&mhi_cntrl->syserr_worker);
+ }
+
+ MHI_VERB("Exit\n");
+
+ return IRQ_HANDLED;
+}
+
+irqreturn_t mhi_intvec_handlr(int irq_number, void *dev)
+{
+
+ struct mhi_controller *mhi_cntrl = dev;
+
+ /* wake up any events waiting for state change */
+ MHI_VERB("Enter\n");
+ wake_up_all(&mhi_cntrl->state_event);
+ MHI_VERB("Exit\n");
+
+ return IRQ_WAKE_THREAD;
+}
+
+int mhi_send_cmd(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan, enum MHI_CMD cmd)
+{
+ struct mhi_tre *cmd_tre = NULL;
+ struct mhi_cmd *mhi_cmd = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING];
+ struct mhi_ring *ring = &mhi_cmd->ring;
+ int chan = 0;
+
+ MHI_VERB("Entered, MHI pm_state:%s dev_state:%s ee:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ TO_MHI_EXEC_STR(mhi_cntrl->ee));
+
+ if (mhi_chan)
+ chan = mhi_chan->chan;
+
+ spin_lock_bh(&mhi_cmd->lock);
+ if (!get_nr_avail_ring_elements(mhi_cntrl, ring)) {
+ spin_unlock_bh(&mhi_cmd->lock);
+ return -ENOMEM;
+ }
+
+ /* prepare the cmd tre */
+ cmd_tre = ring->wptr;
+ switch (cmd) {
+ case MHI_CMD_RESET_CHAN:
+ cmd_tre->ptr = MHI_TRE_CMD_RESET_PTR;
+ cmd_tre->dword[0] = MHI_TRE_CMD_RESET_DWORD0;
+ cmd_tre->dword[1] = MHI_TRE_CMD_RESET_DWORD1(chan);
+ break;
+ case MHI_CMD_START_CHAN:
+ cmd_tre->ptr = MHI_TRE_CMD_START_PTR;
+ cmd_tre->dword[0] = MHI_TRE_CMD_START_DWORD0;
+ cmd_tre->dword[1] = MHI_TRE_CMD_START_DWORD1(chan);
+ break;
+ case MHI_CMD_TIMSYNC_CFG:
+ cmd_tre->ptr = MHI_TRE_CMD_TSYNC_CFG_PTR;
+ cmd_tre->dword[0] = MHI_TRE_CMD_TSYNC_CFG_DWORD0;
+ cmd_tre->dword[1] = MHI_TRE_CMD_TSYNC_CFG_DWORD1
+ (mhi_cntrl->mhi_tsync->er_index);
+ break;
+ }
+
+ MHI_VERB("WP:0x%llx TRE: 0x%llx 0x%08x 0x%08x\n",
+ (u64)mhi_to_physical(ring, cmd_tre), cmd_tre->ptr,
+ cmd_tre->dword[0], cmd_tre->dword[1]);
+
+ /* queue to hardware */
+ mhi_add_ring_element(mhi_cntrl, ring);
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl->pm_state)))
+ mhi_ring_cmd_db(mhi_cntrl, mhi_cmd);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ spin_unlock_bh(&mhi_cmd->lock);
+
+ return 0;
+}
+
+static int __mhi_prepare_channel(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan)
+{
+ int ret = 0;
+
+ MHI_LOG("Entered: preparing channel:%d\n", mhi_chan->chan);
+
+ if (!(BIT(mhi_cntrl->ee) & mhi_chan->ee_mask)) {
+ MHI_ERR("Current EE:%s Required EE Mask:0x%x for chan:%s\n",
+ TO_MHI_EXEC_STR(mhi_cntrl->ee), mhi_chan->ee_mask,
+ mhi_chan->name);
+ return -ENOTCONN;
+ }
+
+ mutex_lock(&mhi_chan->mutex);
+
+ /* if channel is not disable state do not allow to start */
+ if (mhi_chan->ch_state != MHI_CH_STATE_DISABLED) {
+ ret = -EIO;
+ MHI_LOG("channel:%d is not in disabled state, ch_state%d\n",
+ mhi_chan->chan, mhi_chan->ch_state);
+ goto error_init_chan;
+ }
+
+ /* client manages channel context for offload channels */
+ if (!mhi_chan->offload_ch) {
+ ret = mhi_init_chan_ctxt(mhi_cntrl, mhi_chan);
+ if (ret) {
+ MHI_ERR("Error with init chan\n");
+ goto error_init_chan;
+ }
+ }
+
+ reinit_completion(&mhi_chan->completion);
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ MHI_ERR("MHI host is not in active state\n");
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ ret = -EIO;
+ goto error_pm_state;
+ }
+
+ mhi_cntrl->wake_get(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->runtime_get(mhi_cntrl, mhi_cntrl->priv_data);
+ mhi_cntrl->runtime_put(mhi_cntrl, mhi_cntrl->priv_data);
+
+ ret = mhi_send_cmd(mhi_cntrl, mhi_chan, MHI_CMD_START_CHAN);
+ if (ret) {
+ MHI_ERR("Failed to send start chan cmd\n");
+ goto error_send_cmd;
+ }
+
+ ret = wait_for_completion_timeout(&mhi_chan->completion,
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+ if (!ret || mhi_chan->ccs != MHI_EV_CC_SUCCESS) {
+ MHI_ERR("Failed to receive cmd completion for chan:%d\n",
+ mhi_chan->chan);
+ ret = -EIO;
+ goto error_send_cmd;
+ }
+
+ write_lock_irq(&mhi_chan->lock);
+ mhi_chan->ch_state = MHI_CH_STATE_ENABLED;
+ write_unlock_irq(&mhi_chan->lock);
+
+ /* pre allocate buffer for xfer ring */
+ if (mhi_chan->pre_alloc) {
+ int nr_el = get_nr_avail_ring_elements(mhi_cntrl,
+ &mhi_chan->tre_ring);
+ size_t len = mhi_cntrl->buffer_len;
+
+ while (nr_el--) {
+ void *buf;
+
+ buf = kmalloc(len, GFP_KERNEL);
+ if (!buf) {
+ ret = -ENOMEM;
+ goto error_pre_alloc;
+ }
+
+ /* prepare transfer descriptors */
+ ret = mhi_chan->gen_tre(mhi_cntrl, mhi_chan, buf, buf,
+ len, MHI_EOT);
+ if (ret) {
+ MHI_ERR("Chan:%d error prepare buffer\n",
+ mhi_chan->chan);
+ kfree(buf);
+ goto error_pre_alloc;
+ }
+ }
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (MHI_DB_ACCESS_VALID(mhi_cntrl->pm_state)) {
+ read_lock_irq(&mhi_chan->lock);
+ mhi_ring_chan_db(mhi_cntrl, mhi_chan);
+ read_unlock_irq(&mhi_chan->lock);
+ }
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ }
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ mutex_unlock(&mhi_chan->mutex);
+
+ MHI_LOG("Chan:%d successfully moved to start state\n", mhi_chan->chan);
+
+ return 0;
+
+error_send_cmd:
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+error_pm_state:
+ if (!mhi_chan->offload_ch)
+ mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan);
+
+error_init_chan:
+ mutex_unlock(&mhi_chan->mutex);
+
+ return ret;
+
+error_pre_alloc:
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ mutex_unlock(&mhi_chan->mutex);
+ __mhi_unprepare_channel(mhi_cntrl, mhi_chan);
+
+ return ret;
+}
+
+static void mhi_mark_stale_events(struct mhi_controller *mhi_cntrl,
+ struct mhi_event *mhi_event,
+ struct mhi_event_ctxt *er_ctxt, int chan)
+{
+ struct mhi_tre *dev_rp, *local_rp;
+ struct mhi_ring *ev_ring;
+ unsigned long flags;
+
+ MHI_LOG("Marking all events for chan:%d as stale\n", chan);
+
+ ev_ring = &mhi_event->ring;
+
+ /* mark all stale events related to channel as STALE event */
+ spin_lock_irqsave(&mhi_event->lock, flags);
+ dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp);
+
+ local_rp = ev_ring->rptr;
+ while (dev_rp != local_rp) {
+ if (MHI_TRE_GET_EV_TYPE(local_rp) ==
+ MHI_PKT_TYPE_TX_EVENT &&
+ chan == MHI_TRE_GET_EV_CHID(local_rp))
+ local_rp->dword[1] = MHI_TRE_EV_DWORD1(chan,
+ MHI_PKT_TYPE_STALE_EVENT);
+ local_rp++;
+ if (local_rp == (ev_ring->base + ev_ring->len))
+ local_rp = ev_ring->base;
+ }
+
+ MHI_LOG("Finished marking events as stale events\n");
+ spin_unlock_irqrestore(&mhi_event->lock, flags);
+}
+
+static void mhi_reset_data_chan(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan)
+{
+ struct mhi_ring *buf_ring, *tre_ring;
+ struct mhi_result result;
+
+ /* reset any pending buffers */
+ buf_ring = &mhi_chan->buf_ring;
+ tre_ring = &mhi_chan->tre_ring;
+ result.transaction_status = -ENOTCONN;
+ result.bytes_xferd = 0;
+ while (tre_ring->rptr != tre_ring->wptr) {
+ struct mhi_buf_info *buf_info = buf_ring->rptr;
+
+ if (mhi_chan->dir == DMA_TO_DEVICE)
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ if (!buf_info->pre_mapped)
+ mhi_cntrl->unmap_single(mhi_cntrl, buf_info);
+ mhi_del_ring_element(mhi_cntrl, buf_ring);
+ mhi_del_ring_element(mhi_cntrl, tre_ring);
+
+ if (mhi_chan->pre_alloc) {
+ kfree(buf_info->cb_buf);
+ } else {
+ result.buf_addr = buf_info->cb_buf;
+ mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
+ }
+ }
+}
+
+static void mhi_reset_rsc_chan(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan)
+{
+ struct mhi_ring *buf_ring, *tre_ring;
+ struct mhi_result result;
+ struct mhi_buf_info *buf_info;
+
+ /* reset any pending buffers */
+ buf_ring = &mhi_chan->buf_ring;
+ tre_ring = &mhi_chan->tre_ring;
+ result.transaction_status = -ENOTCONN;
+ result.bytes_xferd = 0;
+
+ buf_info = buf_ring->base;
+ for (; (void *)buf_info < buf_ring->base + buf_ring->len; buf_info++) {
+ if (!buf_info->used)
+ continue;
+
+ result.buf_addr = buf_info->cb_buf;
+ mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result);
+ buf_info->used = false;
+ }
+}
+
+void mhi_reset_chan(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan)
+{
+
+ struct mhi_event *mhi_event;
+ struct mhi_event_ctxt *er_ctxt;
+ int chan = mhi_chan->chan;
+
+ /* nothing to reset, client don't queue buffers */
+ if (mhi_chan->offload_ch)
+ return;
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index];
+ er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_chan->er_index];
+
+ mhi_mark_stale_events(mhi_cntrl, mhi_event, er_ctxt, chan);
+
+ if (mhi_chan->xfer_type == MHI_XFER_RSC_DMA)
+ mhi_reset_rsc_chan(mhi_cntrl, mhi_chan);
+ else
+ mhi_reset_data_chan(mhi_cntrl, mhi_chan);
+
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ MHI_LOG("Reset complete.\n");
+}
+
+static void __mhi_unprepare_channel(struct mhi_controller *mhi_cntrl,
+ struct mhi_chan *mhi_chan)
+{
+ int ret;
+
+ MHI_LOG("Entered: unprepare channel:%d\n", mhi_chan->chan);
+
+ /* no more processing events for this channel */
+ mutex_lock(&mhi_chan->mutex);
+ write_lock_irq(&mhi_chan->lock);
+ if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED) {
+ MHI_LOG("chan:%d is already disabled\n", mhi_chan->chan);
+ write_unlock_irq(&mhi_chan->lock);
+ mutex_unlock(&mhi_chan->mutex);
+ return;
+ }
+
+ mhi_chan->ch_state = MHI_CH_STATE_DISABLED;
+ write_unlock_irq(&mhi_chan->lock);
+
+ reinit_completion(&mhi_chan->completion);
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ goto error_invalid_state;
+ }
+
+ mhi_cntrl->wake_get(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ mhi_cntrl->runtime_get(mhi_cntrl, mhi_cntrl->priv_data);
+ mhi_cntrl->runtime_put(mhi_cntrl, mhi_cntrl->priv_data);
+ ret = mhi_send_cmd(mhi_cntrl, mhi_chan, MHI_CMD_RESET_CHAN);
+ if (ret) {
+ MHI_ERR("Failed to send reset chan cmd\n");
+ goto error_completion;
+ }
+
+ /* even if it fails we will still reset */
+ ret = wait_for_completion_timeout(&mhi_chan->completion,
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+ if (!ret || mhi_chan->ccs != MHI_EV_CC_SUCCESS)
+ MHI_ERR("Failed to receive cmd completion, still resetting\n");
+
+error_completion:
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+error_invalid_state:
+ if (!mhi_chan->offload_ch) {
+ mhi_reset_chan(mhi_cntrl, mhi_chan);
+ mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan);
+ }
+ MHI_LOG("chan:%d successfully resetted\n", mhi_chan->chan);
+ mutex_unlock(&mhi_chan->mutex);
+}
+
+int mhi_debugfs_mhi_states_show(struct seq_file *m, void *d)
+{
+ struct mhi_controller *mhi_cntrl = m->private;
+
+ seq_printf(m, "pm_state:%s dev_state:%s EE:%s M0:%u M2:%u M3:%u ",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ TO_MHI_EXEC_STR(mhi_cntrl->ee),
+ mhi_cntrl->M0, mhi_cntrl->M2, mhi_cntrl->M3);
+ seq_printf(m, "wake:%d dev_wake:%u alloc_size:%u\n",
+ mhi_cntrl->wake_set,
+ atomic_read(&mhi_cntrl->dev_wake),
+ atomic_read(&mhi_cntrl->alloc_size));
+
+ return 0;
+}
+
+static int __mhi_debugfs_mhi_event_show_ring(struct seq_file *m, int index,
+ struct mhi_event_ctxt *er_ctxt,
+ struct mhi_event *mhi_event)
+{
+ struct mhi_ring *ring = &mhi_event->ring;
+
+ if (mhi_event->offload_ev) {
+ seq_printf(m, "Index:%d offload event ring\n", index);
+ return 0;
+ }
+
+ seq_printf(m, "Index:%d modc:%d modt:%d base:0x%0llx len:0x%llx",
+ index, er_ctxt->intmodc, er_ctxt->intmodt,
+ er_ctxt->rbase, er_ctxt->rlen);
+ seq_printf(m, " rp:0x%llx wp:0x%llx local_rp:0x%llx db:0x%llx\n",
+ er_ctxt->rp, er_ctxt->wp,
+ (unsigned long long)mhi_to_physical(ring, ring->rptr),
+ (unsigned long long)mhi_event->db_cfg.db_val);
+
+ return 0;
+}
+
+int mhi_debugfs_mhi_event_show(struct seq_file *m, void *d)
+{
+ struct mhi_controller *mhi_cntrl = m->private;
+ struct mhi_event *mhi_event;
+ struct mhi_event_ctxt *er_ctxt;
+
+ int i;
+
+ er_ctxt = mhi_cntrl->mhi_ctxt->er_ctxt;
+ mhi_event = mhi_cntrl->mhi_event;
+ for (i = 0; i < mhi_cntrl->total_ev_rings;
+ i++, er_ctxt++, mhi_event++)
+ __mhi_debugfs_mhi_event_show_ring(m, i, er_ctxt, mhi_event);
+
+ return 0;
+}
+
+static int __mhi_debugfs_mhi_chan_show_ring(struct seq_file *m, int index,
+ struct mhi_chan_ctxt *chan_ctxt,
+ struct mhi_chan *mhi_chan)
+{
+ struct mhi_ring *ring = &mhi_chan->tre_ring;
+
+ if (mhi_chan->offload_ch) {
+ seq_printf(m, "%s(%u) offload channel\n",
+ mhi_chan->name, mhi_chan->chan);
+ } else if (mhi_chan->mhi_dev) {
+ seq_printf(m, "%s(%u) state:0x%x brstmode:0x%x pllcfg:0x%x ",
+ mhi_chan->name, mhi_chan->chan,
+ chan_ctxt->chstate, chan_ctxt->brstmode,
+ chan_ctxt->pollcfg);
+ seq_printf(m, "type:0x%x erindex:%u ",
+ chan_ctxt->chtype, chan_ctxt->erindex);
+ seq_printf(m, "base:0x%llx len:0x%llx wp:0x%llx ",
+ chan_ctxt->rbase, chan_ctxt->rlen,
+ chan_ctxt->wp);
+ seq_printf(m, "local_rp:0x%llx local_wp:0x%llx db:0x%llx\n",
+ (unsigned long long)mhi_to_physical(ring, ring->rptr),
+ (unsigned long long)mhi_to_physical(ring, ring->wptr),
+ (unsigned long long)mhi_chan->db_cfg.db_val);
+ }
+
+ return 0;
+}
+
+int mhi_debugfs_mhi_chan_show(struct seq_file *m, void *d)
+{
+ struct mhi_controller *mhi_cntrl = m->private;
+ struct mhi_chan *mhi_chan;
+ struct mhi_chan_ctxt *chan_ctxt;
+ int i;
+
+ mhi_chan = mhi_cntrl->mhi_chan;
+ chan_ctxt = mhi_cntrl->mhi_ctxt->chan_ctxt;
+ for (i = 0; i < mhi_cntrl->max_chan; i++, chan_ctxt++, mhi_chan++)
+ __mhi_debugfs_mhi_chan_show_ring(m, i, chan_ctxt, mhi_chan);
+
+ return 0;
+}
+
+/* move channel to start state */
+int mhi_prepare_for_transfer(struct mhi_device *mhi_dev)
+{
+ int ret, dir;
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_chan *mhi_chan;
+
+ for (dir = 0; dir < 2; dir++) {
+ mhi_chan = dir ? mhi_dev->dl_chan : mhi_dev->ul_chan;
+
+ if (!mhi_chan)
+ continue;
+
+ ret = __mhi_prepare_channel(mhi_cntrl, mhi_chan);
+ if (ret) {
+ MHI_ERR("Error moving chan %s,%d to START state\n",
+ mhi_chan->name, mhi_chan->chan);
+ goto error_open_chan;
+ }
+ }
+
+ return 0;
+
+error_open_chan:
+ for (--dir; dir >= 0; dir--) {
+ mhi_chan = dir ? mhi_dev->dl_chan : mhi_dev->ul_chan;
+
+ if (!mhi_chan)
+ continue;
+
+ __mhi_unprepare_channel(mhi_cntrl, mhi_chan);
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL(mhi_prepare_for_transfer);
+
+void mhi_unprepare_from_transfer(struct mhi_device *mhi_dev)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_chan *mhi_chan;
+ int dir;
+
+ for (dir = 0; dir < 2; dir++) {
+ mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan;
+
+ if (!mhi_chan)
+ continue;
+
+ __mhi_unprepare_channel(mhi_cntrl, mhi_chan);
+ }
+}
+EXPORT_SYMBOL(mhi_unprepare_from_transfer);
+
+int mhi_get_no_free_descriptors(struct mhi_device *mhi_dev,
+ enum dma_data_direction dir)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ?
+ mhi_dev->ul_chan : mhi_dev->dl_chan;
+ struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
+
+ return get_nr_avail_ring_elements(mhi_cntrl, tre_ring);
+}
+EXPORT_SYMBOL(mhi_get_no_free_descriptors);
+
+static int __mhi_bdf_to_controller(struct device *dev, const void *tmp)
+{
+ struct mhi_device *mhi_dev = to_mhi_device(dev);
+ const struct mhi_device *match = tmp;
+
+ /* return any none-zero value if match */
+ if (mhi_dev->dev_type == MHI_CONTROLLER_TYPE &&
+ mhi_dev->domain == match->domain && mhi_dev->bus == match->bus &&
+ mhi_dev->slot == match->slot && mhi_dev->dev_id == match->dev_id)
+ return 1;
+
+ return 0;
+}
+
+struct mhi_controller *mhi_bdf_to_controller(u32 domain,
+ u32 bus, u32 slot, u32 dev_id)
+{
+ struct mhi_device tmp, *mhi_dev;
+ struct device *dev;
+
+ tmp.domain = domain;
+ tmp.bus = bus;
+ tmp.slot = slot;
+ tmp.dev_id = dev_id;
+
+ dev = bus_find_device(&mhi_bus_type, NULL, &tmp,
+ __mhi_bdf_to_controller);
+ if (!dev)
+ return NULL;
+
+ mhi_dev = to_mhi_device(dev);
+
+ return mhi_dev->mhi_cntrl;
+}
+EXPORT_SYMBOL(mhi_bdf_to_controller);
+
+int mhi_poll(struct mhi_device *mhi_dev, u32 budget)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_chan *mhi_chan = mhi_dev->dl_chan;
+ struct mhi_event *mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index];
+ int ret;
+
+ spin_lock_bh(&mhi_event->lock);
+ ret = mhi_event->process_event(mhi_cntrl, mhi_event, budget);
+ spin_unlock_bh(&mhi_event->lock);
+
+ return ret;
+}
+EXPORT_SYMBOL(mhi_poll);
+
+int mhi_get_remote_time_sync(struct mhi_device *mhi_dev,
+ u64 *t_host, u64 *t_dev)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_timesync *mhi_tsync = mhi_cntrl->mhi_tsync;
+ int ret;
+
+ /* not all devices support time feature */
+ if (!mhi_tsync)
+ return -EIO;
+
+ /* bring to M0 state */
+ ret = __mhi_device_get_sync(mhi_cntrl);
+ if (ret)
+ return ret;
+
+ mutex_lock(&mhi_tsync->lpm_mutex);
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) {
+ MHI_ERR("MHI is not in active state, pm_state:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ ret = -EIO;
+ goto error_invalid_state;
+ }
+
+ /* disable link level low power modes */
+ ret = mhi_cntrl->lpm_disable(mhi_cntrl, mhi_cntrl->priv_data);
+ if (ret)
+ goto error_invalid_state;
+
+ /*
+ * time critical code to fetch device times,
+ * delay between these two steps should be
+ * deterministic as possible.
+ */
+ preempt_disable();
+ local_irq_disable();
+
+ *t_host = mhi_cntrl->time_get(mhi_cntrl, mhi_cntrl->priv_data);
+#ifdef CONFIG_ARM
+ *t_dev = readq_relaxed_no_log(mhi_tsync->time_reg);
+#else
+ *t_dev = readq_relaxed(mhi_tsync->time_reg);
+#endif
+
+ local_irq_enable();
+ preempt_enable();
+
+ mhi_cntrl->lpm_enable(mhi_cntrl, mhi_cntrl->priv_data);
+
+error_invalid_state:
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ mutex_unlock(&mhi_tsync->lpm_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL(mhi_get_remote_time_sync);
+
+/**
+ * mhi_get_remote_time - Get external modem time relative to host time
+ * Trigger event to capture modem time, also capture host time so client
+ * can do a relative drift comparision.
+ * Recommended only tsync device calls this method and do not call this
+ * from atomic context
+ * @mhi_dev: Device associated with the channels
+ * @sequence:unique sequence id track event
+ * @cb_func: callback function to call back
+ */
+int mhi_get_remote_time(struct mhi_device *mhi_dev,
+ u32 sequence,
+ void (*cb_func)(struct mhi_device *mhi_dev,
+ u32 sequence,
+ u64 local_time, u64 remote_time))
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ struct mhi_timesync *mhi_tsync = mhi_cntrl->mhi_tsync;
+ struct tsync_node *tsync_node;
+ int ret;
+
+ /* not all devices support time feature */
+ if (!mhi_tsync)
+ return -EIO;
+
+ /* tsync db can only be rung in M0 state */
+ ret = __mhi_device_get_sync(mhi_cntrl);
+ if (ret)
+ return ret;
+
+ /*
+ * technically we can use GFP_KERNEL, but wants to avoid
+ * # of times scheduling out
+ */
+ tsync_node = kzalloc(sizeof(*tsync_node), GFP_ATOMIC);
+ if (!tsync_node) {
+ ret = -ENOMEM;
+ goto error_no_mem;
+ }
+
+ tsync_node->sequence = sequence;
+ tsync_node->cb_func = cb_func;
+ tsync_node->mhi_dev = mhi_dev;
+
+ /* disable link level low power modes */
+ mhi_cntrl->lpm_disable(mhi_cntrl, mhi_cntrl->priv_data);
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) {
+ MHI_ERR("MHI is not in active state, pm_state:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ ret = -EIO;
+ goto error_invalid_state;
+ }
+
+ spin_lock_irq(&mhi_tsync->lock);
+ list_add_tail(&tsync_node->node, &mhi_tsync->head);
+ spin_unlock_irq(&mhi_tsync->lock);
+
+ /*
+ * time critical code, delay between these two steps should be
+ * deterministic as possible.
+ */
+ preempt_disable();
+ local_irq_disable();
+
+ tsync_node->local_time =
+ mhi_cntrl->time_get(mhi_cntrl, mhi_cntrl->priv_data);
+#ifdef CONFIG_ARM
+ writel_relaxed_no_log(tsync_node->sequence, mhi_tsync->db);
+#else
+ writel_relaxed(tsync_node->sequence, mhi_tsync->db);
+#endif
+ /* write must go thru immediately */
+ wmb();
+
+ local_irq_enable();
+ preempt_enable();
+
+ ret = 0;
+
+error_invalid_state:
+ if (ret)
+ kfree(tsync_node);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->lpm_enable(mhi_cntrl, mhi_cntrl->priv_data);
+
+error_no_mem:
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return ret;
+}
+EXPORT_SYMBOL(mhi_get_remote_time);
+
+void mhi_debug_reg_dump(struct mhi_controller *mhi_cntrl)
+{
+ enum mhi_dev_state state;
+ enum mhi_ee ee;
+ int i, ret;
+ u32 val = 0;
+ void __iomem *mhi_base = mhi_cntrl->regs;
+ void __iomem *bhi_base = mhi_cntrl->bhi;
+ void __iomem *bhie_base = mhi_cntrl->bhie;
+ void __iomem *wake_db = mhi_cntrl->wake_db;
+ struct {
+ const char *name;
+ int offset;
+ void *base;
+ } debug_reg[] = {
+ { "MHI_CNTRL", MHICTRL, mhi_base },
+ { "MHI_STATUS", MHISTATUS, mhi_base },
+ { "MHI_WAKE_DB", 0, wake_db },
+ { "BHI_EXECENV", BHI_EXECENV, bhi_base },
+ { "BHI_STATUS", BHI_STATUS, bhi_base },
+ { "BHI_ERRCODE", BHI_ERRCODE, bhi_base },
+ { "BHI_ERRDBG1", BHI_ERRDBG1, bhi_base },
+ { "BHI_ERRDBG2", BHI_ERRDBG2, bhi_base },
+ { "BHI_ERRDBG3", BHI_ERRDBG3, bhi_base },
+ { "BHIE_TXVEC_DB", BHIE_TXVECDB_OFFS, bhie_base },
+ { "BHIE_TXVEC_STATUS", BHIE_TXVECSTATUS_OFFS, bhie_base },
+ { "BHIE_RXVEC_DB", BHIE_RXVECDB_OFFS, bhie_base },
+ { "BHIE_RXVEC_STATUS", BHIE_RXVECSTATUS_OFFS, bhie_base },
+ { NULL },
+ };
+
+ MHI_LOG("host pm_state:%s dev_state:%s ee:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ TO_MHI_EXEC_STR(mhi_cntrl->ee));
+
+ state = mhi_get_mhi_state(mhi_cntrl);
+ ee = mhi_get_exec_env(mhi_cntrl);
+
+ MHI_LOG("device ee:%s dev_state:%s\n", TO_MHI_EXEC_STR(ee),
+ TO_MHI_STATE_STR(state));
+
+ for (i = 0; debug_reg[i].name; i++) {
+ ret = mhi_read_reg(mhi_cntrl, debug_reg[i].base,
+ debug_reg[i].offset, &val);
+ MHI_LOG("reg:%s val:0x%x, ret:%d\n", debug_reg[i].name, val,
+ ret);
+ }
+}
+EXPORT_SYMBOL(mhi_debug_reg_dump);
diff --git a/drivers/bus/mhi/core/mhi_pm.c b/drivers/bus/mhi/core/mhi_pm.c
new file mode 100644
index 000000000000..1c9b5cca2459
--- /dev/null
+++ b/drivers/bus/mhi/core/mhi_pm.c
@@ -0,0 +1,1158 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. */
+
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/of.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/wait.h>
+#include <linux/mhi.h>
+#include "mhi_internal.h"
+
+/*
+ * Not all MHI states transitions are sync transitions. Linkdown, SSR, and
+ * shutdown can happen anytime asynchronously. This function will transition to
+ * new state only if we're allowed to transitions.
+ *
+ * Priority increase as we go down, example while in any states from L0, start
+ * state from L1, L2, or L3 can be set. Notable exception to this rule is state
+ * DISABLE. From DISABLE state we can transition to only POR or state. Also
+ * for example while in L2 state, user cannot jump back to L1 or L0 states.
+ * Valid transitions:
+ * L0: DISABLE <--> POR
+ * POR <--> POR
+ * POR -> M0 -> M2 --> M0
+ * POR -> FW_DL_ERR
+ * FW_DL_ERR <--> FW_DL_ERR
+ * M0 -> FW_DL_ERR
+ * M0 -> M3_ENTER -> M3 -> M3_EXIT --> M0
+ * L1: SYS_ERR_DETECT -> SYS_ERR_PROCESS --> POR
+ * L2: SHUTDOWN_PROCESS -> DISABLE
+ * L3: LD_ERR_FATAL_DETECT <--> LD_ERR_FATAL_DETECT
+ * LD_ERR_FATAL_DETECT -> SHUTDOWN_PROCESS
+ */
+static const struct mhi_pm_transitions mhi_state_transitions[] = {
+ /* L0 States */
+ {
+ MHI_PM_DISABLE,
+ MHI_PM_POR
+ },
+ {
+ MHI_PM_POR,
+ MHI_PM_POR | MHI_PM_DISABLE | MHI_PM_M0 |
+ MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS |
+ MHI_PM_LD_ERR_FATAL_DETECT | MHI_PM_FW_DL_ERR
+ },
+ {
+ MHI_PM_M0,
+ MHI_PM_M2 | MHI_PM_M3_ENTER | MHI_PM_SYS_ERR_DETECT |
+ MHI_PM_SHUTDOWN_PROCESS | MHI_PM_LD_ERR_FATAL_DETECT |
+ MHI_PM_FW_DL_ERR
+ },
+ {
+ MHI_PM_M2,
+ MHI_PM_M0 | MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS |
+ MHI_PM_LD_ERR_FATAL_DETECT
+ },
+ {
+ MHI_PM_M3_ENTER,
+ MHI_PM_M3 | MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS |
+ MHI_PM_LD_ERR_FATAL_DETECT
+ },
+ {
+ MHI_PM_M3,
+ MHI_PM_M3_EXIT | MHI_PM_SYS_ERR_DETECT |
+ MHI_PM_SHUTDOWN_PROCESS | MHI_PM_LD_ERR_FATAL_DETECT
+ },
+ {
+ MHI_PM_M3_EXIT,
+ MHI_PM_M0 | MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS |
+ MHI_PM_LD_ERR_FATAL_DETECT
+ },
+ {
+ MHI_PM_FW_DL_ERR,
+ MHI_PM_FW_DL_ERR | MHI_PM_SYS_ERR_DETECT |
+ MHI_PM_SHUTDOWN_PROCESS | MHI_PM_LD_ERR_FATAL_DETECT
+ },
+ /* L1 States */
+ {
+ MHI_PM_SYS_ERR_DETECT,
+ MHI_PM_SYS_ERR_PROCESS | MHI_PM_SHUTDOWN_PROCESS |
+ MHI_PM_LD_ERR_FATAL_DETECT
+ },
+ {
+ MHI_PM_SYS_ERR_PROCESS,
+ MHI_PM_POR | MHI_PM_SHUTDOWN_PROCESS |
+ MHI_PM_LD_ERR_FATAL_DETECT
+ },
+ /* L2 States */
+ {
+ MHI_PM_SHUTDOWN_PROCESS,
+ MHI_PM_DISABLE | MHI_PM_LD_ERR_FATAL_DETECT
+ },
+ /* L3 States */
+ {
+ MHI_PM_LD_ERR_FATAL_DETECT,
+ MHI_PM_LD_ERR_FATAL_DETECT | MHI_PM_SHUTDOWN_PROCESS
+ },
+};
+
+enum MHI_PM_STATE __must_check mhi_tryset_pm_state(struct mhi_controller
+ *mhi_cntrl,
+ enum MHI_PM_STATE state)
+{
+ unsigned long cur_state = mhi_cntrl->pm_state;
+ int index = find_last_bit(&cur_state, 32);
+
+ if (unlikely(index >= ARRAY_SIZE(mhi_state_transitions))) {
+ MHI_CRITICAL("cur_state:%s is not a valid pm_state\n",
+ to_mhi_pm_state_str(cur_state));
+ return cur_state;
+ }
+
+ if (unlikely(mhi_state_transitions[index].from_state != cur_state)) {
+ MHI_ERR("index:%u cur_state:%s != actual_state: %s\n",
+ index, to_mhi_pm_state_str(cur_state),
+ to_mhi_pm_state_str
+ (mhi_state_transitions[index].from_state));
+ return cur_state;
+ }
+
+ if (unlikely(!(mhi_state_transitions[index].to_states & state))) {
+ MHI_LOG
+ ("Not allowing pm state transition from:%s to:%s state\n",
+ to_mhi_pm_state_str(cur_state),
+ to_mhi_pm_state_str(state));
+ return cur_state;
+ }
+
+ MHI_VERB("Transition to pm state from:%s to:%s\n",
+ to_mhi_pm_state_str(cur_state), to_mhi_pm_state_str(state));
+
+ mhi_cntrl->pm_state = state;
+ return mhi_cntrl->pm_state;
+}
+
+void mhi_set_mhi_state(struct mhi_controller *mhi_cntrl,
+ enum mhi_dev_state state)
+{
+ if (state == MHI_STATE_RESET) {
+ mhi_write_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL,
+ MHICTRL_RESET_MASK, MHICTRL_RESET_SHIFT, 1);
+ } else {
+ mhi_write_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL,
+ MHICTRL_MHISTATE_MASK,
+ MHICTRL_MHISTATE_SHIFT, state);
+ }
+}
+EXPORT_SYMBOL(mhi_set_mhi_state);
+
+/* set device wake */
+void mhi_assert_dev_wake(struct mhi_controller *mhi_cntrl, bool force)
+{
+ unsigned long flags;
+
+ /* if set, regardless of count set the bit if not set */
+ if (unlikely(force)) {
+ spin_lock_irqsave(&mhi_cntrl->wlock, flags);
+ atomic_inc(&mhi_cntrl->dev_wake);
+ if (MHI_WAKE_DB_FORCE_SET_VALID(mhi_cntrl->pm_state) &&
+ !mhi_cntrl->wake_set) {
+ mhi_write_db(mhi_cntrl, mhi_cntrl->wake_db, 1);
+ mhi_cntrl->wake_set = true;
+ }
+ spin_unlock_irqrestore(&mhi_cntrl->wlock, flags);
+ } else {
+ /* if resources requested already, then increment and exit */
+ if (likely(atomic_add_unless(&mhi_cntrl->dev_wake, 1, 0)))
+ return;
+
+ spin_lock_irqsave(&mhi_cntrl->wlock, flags);
+ if ((atomic_inc_return(&mhi_cntrl->dev_wake) == 1) &&
+ MHI_WAKE_DB_SET_VALID(mhi_cntrl->pm_state) &&
+ !mhi_cntrl->wake_set) {
+ mhi_write_db(mhi_cntrl, mhi_cntrl->wake_db, 1);
+ mhi_cntrl->wake_set = true;
+ }
+ spin_unlock_irqrestore(&mhi_cntrl->wlock, flags);
+ }
+}
+
+/* clear device wake */
+void mhi_deassert_dev_wake(struct mhi_controller *mhi_cntrl, bool override)
+{
+ unsigned long flags;
+
+ MHI_ASSERT(atomic_read(&mhi_cntrl->dev_wake) == 0, "dev_wake == 0");
+
+ /* resources not dropping to 0, decrement and exit */
+ if (likely(atomic_add_unless(&mhi_cntrl->dev_wake, -1, 1)))
+ return;
+
+ spin_lock_irqsave(&mhi_cntrl->wlock, flags);
+ if ((atomic_dec_return(&mhi_cntrl->dev_wake) == 0) &&
+ MHI_WAKE_DB_CLEAR_VALID(mhi_cntrl->pm_state) && !override &&
+ mhi_cntrl->wake_set) {
+ mhi_write_db(mhi_cntrl, mhi_cntrl->wake_db, 0);
+ mhi_cntrl->wake_set = false;
+ }
+ spin_unlock_irqrestore(&mhi_cntrl->wlock, flags);
+}
+
+int mhi_ready_state_transition(struct mhi_controller *mhi_cntrl)
+{
+ void __iomem *base = mhi_cntrl->regs;
+ u32 reset = 1, ready = 0;
+ struct mhi_event *mhi_event;
+ enum MHI_PM_STATE cur_state;
+ int ret, i;
+
+ MHI_LOG("Waiting to enter READY state\n");
+
+ /* wait for RESET to be cleared and READY bit to be set */
+ wait_event_timeout(mhi_cntrl->state_event,
+ MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state) ||
+ mhi_read_reg_field(mhi_cntrl, base, MHICTRL,
+ MHICTRL_RESET_MASK,
+ MHICTRL_RESET_SHIFT, &reset) ||
+ mhi_read_reg_field(mhi_cntrl, base, MHISTATUS,
+ MHISTATUS_READY_MASK,
+ MHISTATUS_READY_SHIFT, &ready) ||
+ (!reset && ready),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ /* device enter into error state */
+ if (MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state))
+ return -EIO;
+
+ /* device did not transition to ready state */
+ if (reset || !ready)
+ return -ETIMEDOUT;
+
+ MHI_LOG("Device in READY State\n");
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_POR);
+ mhi_cntrl->dev_state = MHI_STATE_READY;
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+
+ if (cur_state != MHI_PM_POR) {
+ MHI_ERR("Error moving to state %s from %s\n",
+ to_mhi_pm_state_str(MHI_PM_POR),
+ to_mhi_pm_state_str(cur_state));
+ return -EIO;
+ }
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state))
+ goto error_mmio;
+
+ ret = mhi_init_mmio(mhi_cntrl);
+ if (ret) {
+ MHI_ERR("Error programming mmio registers\n");
+ goto error_mmio;
+ }
+
+ /* add elements to all sw event rings */
+ mhi_event = mhi_cntrl->mhi_event;
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
+ struct mhi_ring *ring = &mhi_event->ring;
+
+ if (mhi_event->offload_ev || mhi_event->hw_ring)
+ continue;
+
+ ring->wptr = ring->base + ring->len - ring->el_size;
+ *ring->ctxt_wp = ring->iommu_base + ring->len - ring->el_size;
+ /* needs to update to all cores */
+ smp_wmb();
+
+ /* ring the db for event rings */
+ spin_lock_irq(&mhi_event->lock);
+ mhi_ring_er_db(mhi_event);
+ spin_unlock_irq(&mhi_event->lock);
+ }
+
+ /* set device into M0 state */
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M0);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return 0;
+
+error_mmio:
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return -EIO;
+}
+
+int mhi_pm_m0_transition(struct mhi_controller *mhi_cntrl)
+{
+ enum MHI_PM_STATE cur_state;
+ struct mhi_chan *mhi_chan;
+ int i;
+
+ MHI_LOG("Entered With State:%s PM_STATE:%s\n",
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ mhi_cntrl->dev_state = MHI_STATE_M0;
+ cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M0);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ if (unlikely(cur_state != MHI_PM_M0)) {
+ MHI_ERR("Failed to transition to state %s from %s\n",
+ to_mhi_pm_state_str(MHI_PM_M0),
+ to_mhi_pm_state_str(cur_state));
+ return -EIO;
+ }
+ mhi_cntrl->M0++;
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_get(mhi_cntrl, false);
+
+ /* ring all event rings and CMD ring only if we're in mission mode */
+ if (MHI_IN_MISSION_MODE(mhi_cntrl->ee)) {
+ struct mhi_event *mhi_event = mhi_cntrl->mhi_event;
+ struct mhi_cmd *mhi_cmd = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING];
+
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
+ if (mhi_event->offload_ev)
+ continue;
+
+ spin_lock_irq(&mhi_event->lock);
+ mhi_ring_er_db(mhi_event);
+ spin_unlock_irq(&mhi_event->lock);
+ }
+
+ /* only ring primary cmd ring */
+ spin_lock_irq(&mhi_cmd->lock);
+ if (mhi_cmd->ring.rptr != mhi_cmd->ring.wptr)
+ mhi_ring_cmd_db(mhi_cntrl, mhi_cmd);
+ spin_unlock_irq(&mhi_cmd->lock);
+ }
+
+ /* ring channel db registers */
+ mhi_chan = mhi_cntrl->mhi_chan;
+ for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) {
+ struct mhi_ring *tre_ring = &mhi_chan->tre_ring;
+
+ write_lock_irq(&mhi_chan->lock);
+ if (mhi_chan->db_cfg.reset_req)
+ mhi_chan->db_cfg.db_mode = true;
+
+ /* only ring DB if ring is not empty */
+ if (tre_ring->base && tre_ring->wptr != tre_ring->rptr)
+ mhi_ring_chan_db(mhi_cntrl, mhi_chan);
+ write_unlock_irq(&mhi_chan->lock);
+ }
+
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ wake_up_all(&mhi_cntrl->state_event);
+ MHI_VERB("Exited\n");
+
+ return 0;
+}
+
+void mhi_pm_m1_transition(struct mhi_controller *mhi_cntrl)
+{
+ enum MHI_PM_STATE state;
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ /* if it fails, means we transition to M3 */
+ state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M2);
+ if (state == MHI_PM_M2) {
+ MHI_VERB("Entered M2 State\n");
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M2);
+ mhi_cntrl->dev_state = MHI_STATE_M2;
+ mhi_cntrl->M2++;
+
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ wake_up_all(&mhi_cntrl->state_event);
+
+ /* transfer pending, exit M2 immediately */
+ if (unlikely(atomic_read(&mhi_cntrl->dev_wake))) {
+ MHI_VERB("Exiting M2 Immediately, count:%d\n",
+ atomic_read(&mhi_cntrl->dev_wake));
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_get(mhi_cntrl, true);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ } else {
+ mhi_cntrl->status_cb(mhi_cntrl, mhi_cntrl->priv_data,
+ MHI_CB_IDLE);
+ }
+ } else {
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ }
+}
+
+int mhi_pm_m3_transition(struct mhi_controller *mhi_cntrl)
+{
+ enum MHI_PM_STATE state;
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ mhi_cntrl->dev_state = MHI_STATE_M3;
+ state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ if (state != MHI_PM_M3) {
+ MHI_ERR("Failed to transition to state %s from %s\n",
+ to_mhi_pm_state_str(MHI_PM_M3),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return -EIO;
+ }
+ wake_up_all(&mhi_cntrl->state_event);
+ mhi_cntrl->M3++;
+
+ MHI_LOG("Entered mhi_state:%s pm_state:%s\n",
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return 0;
+}
+
+static int mhi_pm_mission_mode_transition(struct mhi_controller *mhi_cntrl)
+{
+ int i, ret;
+ struct mhi_event *mhi_event;
+
+ MHI_LOG("Processing Mission Mode Transition\n");
+
+ /* force MHI to be in M0 state before continuing */
+ ret = __mhi_device_get_sync(mhi_cntrl);
+ if (ret)
+ return ret;
+
+ ret = -EIO;
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state))
+ mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ if (!MHI_IN_MISSION_MODE(mhi_cntrl->ee))
+ goto error_mission_mode;
+
+ wake_up_all(&mhi_cntrl->state_event);
+
+ /* add elements to all HW event rings */
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+ goto error_mission_mode;
+
+ mhi_event = mhi_cntrl->mhi_event;
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
+ struct mhi_ring *ring = &mhi_event->ring;
+
+ if (mhi_event->offload_ev || !mhi_event->hw_ring)
+ continue;
+
+ ring->wptr = ring->base + ring->len - ring->el_size;
+ *ring->ctxt_wp = ring->iommu_base + ring->len - ring->el_size;
+ /* all ring updates must get updated immediately */
+ smp_wmb();
+
+ spin_lock_irq(&mhi_event->lock);
+ if (MHI_DB_ACCESS_VALID(mhi_cntrl->pm_state))
+ mhi_ring_er_db(mhi_event);
+ spin_unlock_irq(&mhi_event->lock);
+
+ }
+
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ /* setup support for time sync */
+ mhi_init_timesync(mhi_cntrl);
+
+ MHI_LOG("Adding new devices\n");
+
+ /* add supported devices */
+ mhi_create_devices(mhi_cntrl);
+
+ ret = 0;
+
+ /* Free BHI INTVEC msi */
+ if (mhi_cntrl->msi_allocated == 1)
+ free_irq(mhi_cntrl->irq[0], mhi_cntrl);
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+
+error_mission_mode:
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ MHI_LOG("Exit with ret:%d\n", ret);
+
+ return ret;
+}
+
+/* handles both sys_err and shutdown transitions */
+static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl,
+ enum MHI_PM_STATE transition_state)
+{
+ enum MHI_PM_STATE cur_state, prev_state;
+ struct mhi_event *mhi_event;
+ struct mhi_cmd_ctxt *cmd_ctxt;
+ struct mhi_cmd *mhi_cmd;
+ struct mhi_event_ctxt *er_ctxt;
+ int ret, i;
+
+ MHI_LOG("Enter with from pm_state:%s MHI_STATE:%s to pm_state:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ to_mhi_pm_state_str(transition_state));
+
+ /* We must notify MHI control driver so it can clean up first */
+ if (transition_state == MHI_PM_SYS_ERR_PROCESS)
+ mhi_cntrl->status_cb(mhi_cntrl, mhi_cntrl->priv_data,
+ MHI_CB_SYS_ERROR);
+
+ mutex_lock(&mhi_cntrl->pm_mutex);
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ prev_state = mhi_cntrl->pm_state;
+ cur_state = mhi_tryset_pm_state(mhi_cntrl, transition_state);
+ if (cur_state == transition_state) {
+ mhi_cntrl->ee = MHI_EE_DISABLE_TRANSITION;
+ mhi_cntrl->dev_state = MHI_STATE_RESET;
+ }
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+
+ /* wake up any threads waiting for state transitions */
+ wake_up_all(&mhi_cntrl->state_event);
+
+ /* not handling sys_err, could be middle of shut down */
+ if (cur_state != transition_state) {
+ MHI_LOG("Failed to transition to state:0x%x from:0x%x\n",
+ transition_state, cur_state);
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+ return;
+ }
+
+ /* trigger MHI RESET so device will not access host ddr */
+ if (MHI_REG_ACCESS_VALID(prev_state)) {
+ u32 in_reset = -1;
+ unsigned long timeout = msecs_to_jiffies(mhi_cntrl->timeout_ms);
+
+ MHI_LOG("Trigger device into MHI_RESET\n");
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET);
+
+ /* wait for reset to be cleared */
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_read_reg_field(mhi_cntrl,
+ mhi_cntrl->regs,
+ MHICTRL,
+ MHICTRL_RESET_MASK,
+ MHICTRL_RESET_SHIFT,
+ &in_reset)
+ || !in_reset, timeout);
+ if ((!ret || in_reset) && cur_state == MHI_PM_SYS_ERR_PROCESS) {
+ MHI_CRITICAL("Device failed to exit RESET state\n");
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+ return;
+ }
+
+ /*
+ * device cleares INTVEC as part of RESET processing,
+ * re-program it
+ */
+ mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0);
+ }
+
+ MHI_LOG("Waiting for all pending event ring processing to complete\n");
+ mhi_event = mhi_cntrl->mhi_event;
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) {
+ if (mhi_event->offload_ev)
+ continue;
+ tasklet_kill(&mhi_event->task);
+ }
+
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+
+ MHI_LOG("Reset all active channels and remove mhi devices\n");
+ device_for_each_child(mhi_cntrl->dev, NULL, mhi_destroy_device);
+
+ MHI_LOG("Finish resetting channels\n");
+
+ MHI_LOG("Waiting for all pending threads to complete\n");
+ wake_up_all(&mhi_cntrl->state_event);
+ flush_work(&mhi_cntrl->st_worker);
+ flush_work(&mhi_cntrl->fw_worker);
+
+ mutex_lock(&mhi_cntrl->pm_mutex);
+
+ MHI_ASSERT(atomic_read(&mhi_cntrl->dev_wake), "dev_wake != 0");
+
+ /* reset the ev rings and cmd rings */
+ MHI_LOG("Resetting EV CTXT and CMD CTXT\n");
+ mhi_cmd = mhi_cntrl->mhi_cmd;
+ cmd_ctxt = mhi_cntrl->mhi_ctxt->cmd_ctxt;
+ for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++, cmd_ctxt++) {
+ struct mhi_ring *ring = &mhi_cmd->ring;
+
+ ring->rptr = ring->base;
+ ring->wptr = ring->base;
+ cmd_ctxt->rp = cmd_ctxt->rbase;
+ cmd_ctxt->wp = cmd_ctxt->rbase;
+ }
+
+ mhi_event = mhi_cntrl->mhi_event;
+ er_ctxt = mhi_cntrl->mhi_ctxt->er_ctxt;
+ for (i = 0; i < mhi_cntrl->total_ev_rings; i++,
+ er_ctxt++, mhi_event++) {
+ struct mhi_ring *ring = &mhi_event->ring;
+
+ /* do not touch offload er */
+ if (mhi_event->offload_ev)
+ continue;
+
+ ring->rptr = ring->base;
+ ring->wptr = ring->base;
+ er_ctxt->rp = er_ctxt->rbase;
+ er_ctxt->wp = er_ctxt->rbase;
+ }
+
+ /* remove support for time sync */
+ mhi_destroy_timesync(mhi_cntrl);
+
+ if (cur_state == MHI_PM_SYS_ERR_PROCESS) {
+ mhi_ready_state_transition(mhi_cntrl);
+ } else {
+ /* move to disable state */
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_DISABLE);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ if (unlikely(cur_state != MHI_PM_DISABLE))
+ MHI_ERR("Error moving from pm state:%s to state:%s\n",
+ to_mhi_pm_state_str(cur_state),
+ to_mhi_pm_state_str(MHI_PM_DISABLE));
+ }
+
+ MHI_LOG("Exit with pm_state:%s mhi_state:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state));
+
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+}
+
+int mhi_debugfs_trigger_reset(void *data, u64 val)
+{
+ struct mhi_controller *mhi_cntrl = data;
+ enum MHI_PM_STATE cur_state;
+ int ret;
+
+ MHI_LOG("Trigger MHI Reset\n");
+
+ /* exit lpm first */
+ mhi_cntrl->runtime_get(mhi_cntrl, mhi_cntrl->priv_data);
+ mhi_cntrl->runtime_put(mhi_cntrl, mhi_cntrl->priv_data);
+
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->dev_state == MHI_STATE_M0 ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ MHI_ERR("Did not enter M0 state, cur_state:%s pm_state:%s\n",
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return -EIO;
+ }
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_SYS_ERR_DETECT);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+
+ if (cur_state == MHI_PM_SYS_ERR_DETECT)
+ schedule_work(&mhi_cntrl->syserr_worker);
+
+ return 0;
+}
+
+/* queue a new work item and scheduler work */
+int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl,
+ enum MHI_ST_TRANSITION state)
+{
+ struct state_transition *item = kmalloc(sizeof(*item), GFP_ATOMIC);
+ unsigned long flags;
+
+ if (!item)
+ return -ENOMEM;
+
+ item->state = state;
+ spin_lock_irqsave(&mhi_cntrl->transition_lock, flags);
+ list_add_tail(&item->node, &mhi_cntrl->transition_list);
+ spin_unlock_irqrestore(&mhi_cntrl->transition_lock, flags);
+
+ schedule_work(&mhi_cntrl->st_worker);
+
+ return 0;
+}
+
+void mhi_pm_sys_err_worker(struct work_struct *work)
+{
+ struct mhi_controller *mhi_cntrl = container_of(work,
+ struct mhi_controller,
+ syserr_worker);
+
+ MHI_LOG("Enter with pm_state:%s MHI_STATE:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state));
+
+ mhi_pm_disable_transition(mhi_cntrl, MHI_PM_SYS_ERR_PROCESS);
+}
+
+void mhi_pm_st_worker(struct work_struct *work)
+{
+ struct state_transition *itr, *tmp;
+ LIST_HEAD(head);
+ struct mhi_controller *mhi_cntrl = container_of(work,
+ struct mhi_controller,
+ st_worker);
+ spin_lock_irq(&mhi_cntrl->transition_lock);
+ list_splice_tail_init(&mhi_cntrl->transition_list, &head);
+ spin_unlock_irq(&mhi_cntrl->transition_lock);
+
+ list_for_each_entry_safe(itr, tmp, &head, node) {
+ list_del(&itr->node);
+ MHI_LOG("Transition to state:%s\n",
+ TO_MHI_STATE_TRANS_STR(itr->state));
+
+ switch (itr->state) {
+ case MHI_ST_TRANSITION_PBL:
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state))
+ mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ if (MHI_IN_PBL(mhi_cntrl->ee))
+ wake_up_all(&mhi_cntrl->state_event);
+ break;
+ case MHI_ST_TRANSITION_SBL:
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ mhi_cntrl->ee = MHI_EE_SBL;
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ wake_up_all(&mhi_cntrl->state_event);
+ mhi_create_devices(mhi_cntrl);
+ break;
+ case MHI_ST_TRANSITION_MISSION_MODE:
+ mhi_pm_mission_mode_transition(mhi_cntrl);
+ break;
+ case MHI_ST_TRANSITION_READY:
+ mhi_ready_state_transition(mhi_cntrl);
+ break;
+ default:
+ break;
+ }
+ kfree(itr);
+ }
+}
+
+int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
+{
+ int ret;
+ u32 val;
+ enum mhi_ee current_ee;
+ enum MHI_ST_TRANSITION next_state;
+
+ MHI_LOG("Requested to power on\n");
+
+ /* if (mhi_cntrl->msi_allocated < mhi_cntrl->total_ev_rings)
+ * return -EINVAL;
+ */
+
+ /* set to default wake if not set */
+ if (!mhi_cntrl->wake_get || !mhi_cntrl->wake_put) {
+ mhi_cntrl->wake_get = mhi_assert_dev_wake;
+ mhi_cntrl->wake_put = mhi_deassert_dev_wake;
+ }
+
+ mutex_lock(&mhi_cntrl->pm_mutex);
+ mhi_cntrl->pm_state = MHI_PM_DISABLE;
+
+ if (!mhi_cntrl->pre_init) {
+ /* setup device context */
+ ret = mhi_init_dev_ctxt(mhi_cntrl);
+ if (ret) {
+ MHI_ERR("Error setting dev_context\n");
+ goto error_dev_ctxt;
+ }
+
+ ret = mhi_init_irq_setup(mhi_cntrl);
+ if (ret) {
+ MHI_ERR("Error setting up irq\n");
+ goto error_setup_irq;
+ }
+ }
+
+ /* setup bhi offset & intvec */
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, BHIOFF, &val);
+ if (ret) {
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ MHI_ERR("Error getting bhi offset\n");
+ goto error_bhi_offset;
+ }
+
+ mhi_cntrl->bhi = mhi_cntrl->regs + val;
+
+ /* setup bhie offset */
+ if (mhi_cntrl->fbc_download) {
+ ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, BHIEOFF, &val);
+ if (ret) {
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ MHI_ERR("Error getting bhie offset\n");
+ goto error_bhi_offset;
+ }
+
+ mhi_cntrl->bhie = mhi_cntrl->regs + val;
+ }
+
+ mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0);
+ mhi_cntrl->pm_state = MHI_PM_POR;
+ mhi_cntrl->ee = MHI_EE_MAX;
+ current_ee = mhi_get_exec_env(mhi_cntrl);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+
+ /* confirm device is in valid exec env */
+ if (!MHI_IN_PBL(current_ee) && current_ee != MHI_EE_AMSS) {
+ MHI_ERR("Not a valid ee for power on\n");
+ ret = -EIO;
+ goto error_bhi_offset;
+ }
+
+ /* transition to next state */
+ next_state = MHI_IN_PBL(current_ee) ?
+ MHI_ST_TRANSITION_PBL : MHI_ST_TRANSITION_READY;
+
+ if (next_state == MHI_ST_TRANSITION_PBL)
+ schedule_work(&mhi_cntrl->fw_worker);
+
+ mhi_queue_state_transition(mhi_cntrl, next_state);
+
+ mhi_init_debugfs(mhi_cntrl);
+
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+
+ MHI_LOG("Power on setup success\n");
+
+ return 0;
+
+error_bhi_offset:
+ if (!mhi_cntrl->pre_init)
+ mhi_deinit_free_irq(mhi_cntrl);
+
+error_setup_irq:
+ if (!mhi_cntrl->pre_init)
+ mhi_deinit_dev_ctxt(mhi_cntrl);
+
+error_dev_ctxt:
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+
+ return ret;
+}
+EXPORT_SYMBOL(mhi_async_power_up);
+
+void mhi_power_down(struct mhi_controller *mhi_cntrl, bool graceful)
+{
+ enum MHI_PM_STATE cur_state;
+
+ /* if it's not graceful shutdown, force MHI to a linkdown state */
+ if (!graceful) {
+ mutex_lock(&mhi_cntrl->pm_mutex);
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ cur_state = mhi_tryset_pm_state(mhi_cntrl,
+ MHI_PM_LD_ERR_FATAL_DETECT);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ mutex_unlock(&mhi_cntrl->pm_mutex);
+ if (cur_state != MHI_PM_LD_ERR_FATAL_DETECT)
+ MHI_ERR("Failed to move to state:%s from:%s\n",
+ to_mhi_pm_state_str(MHI_PM_LD_ERR_FATAL_DETECT),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ }
+ mhi_pm_disable_transition(mhi_cntrl, MHI_PM_SHUTDOWN_PROCESS);
+
+ mhi_deinit_debugfs(mhi_cntrl);
+
+ if (!mhi_cntrl->pre_init) {
+ /* free all allocated resources */
+ if (mhi_cntrl->fbc_image) {
+ mhi_free_bhie_table(mhi_cntrl, mhi_cntrl->fbc_image);
+ mhi_cntrl->fbc_image = NULL;
+ }
+ mhi_deinit_free_irq(mhi_cntrl);
+ mhi_deinit_dev_ctxt(mhi_cntrl);
+ }
+}
+EXPORT_SYMBOL(mhi_power_down);
+
+int mhi_sync_power_up(struct mhi_controller *mhi_cntrl)
+{
+ int ret = mhi_async_power_up(mhi_cntrl);
+
+ if (ret)
+ return ret;
+
+ wait_event_timeout(mhi_cntrl->state_event,
+ MHI_IN_MISSION_MODE(mhi_cntrl->ee) ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ return (MHI_IN_MISSION_MODE(mhi_cntrl->ee)) ? 0 : -EIO;
+}
+EXPORT_SYMBOL(mhi_sync_power_up);
+
+int mhi_pm_suspend(struct mhi_controller *mhi_cntrl)
+{
+ int ret;
+ enum MHI_PM_STATE new_state;
+ struct mhi_chan *itr, *tmp;
+
+ if (mhi_cntrl->pm_state == MHI_PM_DISABLE)
+ return -EINVAL;
+
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+ return -EIO;
+
+ /* do a quick check to see if any pending data, then exit */
+ if (atomic_read(&mhi_cntrl->dev_wake)) {
+ MHI_VERB("Busy, aborting M3\n");
+ return -EBUSY;
+ }
+
+ /* exit MHI out of M2 state */
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_get(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->dev_state == MHI_STATE_M0 ||
+ mhi_cntrl->dev_state == MHI_STATE_M1 ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ MHI_ERR
+ ("Did not enter M0||M1 state, cur_state:%s pm_state:%s\n",
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ ret = -EIO;
+ goto error_m0_entry;
+ }
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+
+ /* we're asserting wake so count would be @ least 1 */
+ if (atomic_read(&mhi_cntrl->dev_wake) > 1) {
+ MHI_VERB("Busy, aborting M3\n");
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ ret = -EBUSY;
+ goto error_m0_entry;
+ }
+
+ /* anytime after this, we will resume thru runtime pm framework */
+ MHI_LOG("Allowing M3 transition\n");
+ new_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_ENTER);
+ if (new_state != MHI_PM_M3_ENTER) {
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ MHI_ERR("Error setting to pm_state:%s from pm_state:%s\n",
+ to_mhi_pm_state_str(MHI_PM_M3_ENTER),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+
+ ret = -EIO;
+ goto error_m0_entry;
+ }
+
+ /* set dev to M3 and wait for completion */
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M3);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ MHI_LOG("Wait for M3 completion\n");
+
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->dev_state == MHI_STATE_M3 ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ MHI_ERR("Did not enter M3 state, cur_state:%s pm_state:%s\n",
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return -EIO;
+ }
+
+ /* notify any clients we enter lpm */
+ list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) {
+ mutex_lock(&itr->mutex);
+ if (itr->mhi_dev)
+ mhi_notify(itr->mhi_dev, MHI_CB_LPM_ENTER);
+ mutex_unlock(&itr->mutex);
+ }
+
+ return 0;
+
+error_m0_entry:
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ return ret;
+}
+EXPORT_SYMBOL(mhi_pm_suspend);
+
+int mhi_pm_resume(struct mhi_controller *mhi_cntrl)
+{
+ enum MHI_PM_STATE cur_state;
+ int ret;
+ struct mhi_chan *itr, *tmp;
+
+ MHI_LOG("Entered with pm_state:%s dev_state:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state));
+
+ if (mhi_cntrl->pm_state == MHI_PM_DISABLE)
+ return 0;
+
+ if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))
+ return -EIO;
+
+ MHI_ASSERT(mhi_cntrl->pm_state != MHI_PM_M3, "mhi_pm_state != M3");
+
+ /* notify any clients we enter lpm */
+ list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) {
+ mutex_lock(&itr->mutex);
+ if (itr->mhi_dev)
+ mhi_notify(itr->mhi_dev, MHI_CB_LPM_EXIT);
+ mutex_unlock(&itr->mutex);
+ }
+
+ write_lock_irq(&mhi_cntrl->pm_lock);
+ cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_EXIT);
+ if (cur_state != MHI_PM_M3_EXIT) {
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+ MHI_ERR("Error setting to pm_state:%s from pm_state:%s\n",
+ to_mhi_pm_state_str(MHI_PM_M3_EXIT),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ return -EIO;
+ }
+
+ /* set dev to M0 and wait for completion */
+ mhi_cntrl->wake_get(mhi_cntrl, true);
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M0);
+ write_unlock_irq(&mhi_cntrl->pm_lock);
+
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->dev_state == MHI_STATE_M0 ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ MHI_ERR("Did not enter M0 state, cur_state:%s pm_state:%s\n",
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+
+ /*
+ * It's possible device already in error state and we didn't
+ * process it due to low power mode, force a check
+ */
+ mhi_intvec_threaded_handlr(0, mhi_cntrl);
+ return -EIO;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(mhi_pm_resume);
+
+int __mhi_device_get_sync(struct mhi_controller *mhi_cntrl)
+{
+ int ret;
+
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_get(mhi_cntrl, true);
+ if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) {
+ mhi_cntrl->runtime_get(mhi_cntrl, mhi_cntrl->priv_data);
+ mhi_cntrl->runtime_put(mhi_cntrl, mhi_cntrl->priv_data);
+ }
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->pm_state == MHI_PM_M0 ||
+ MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+
+ if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) {
+ MHI_ERR("Did not enter M0 state, cur_state:%s pm_state:%s\n",
+ TO_MHI_STATE_STR(mhi_cntrl->dev_state),
+ to_mhi_pm_state_str(mhi_cntrl->pm_state));
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+ return -EIO;
+ }
+
+ return 0;
+}
+
+void mhi_device_get(struct mhi_device *mhi_dev)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+
+ atomic_inc(&mhi_dev->dev_wake);
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_get(mhi_cntrl, true);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+}
+EXPORT_SYMBOL(mhi_device_get);
+
+int mhi_device_get_sync(struct mhi_device *mhi_dev)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+ int ret;
+
+ ret = __mhi_device_get_sync(mhi_cntrl);
+ if (!ret)
+ atomic_inc(&mhi_dev->dev_wake);
+
+ return ret;
+}
+EXPORT_SYMBOL(mhi_device_get_sync);
+
+void mhi_device_put(struct mhi_device *mhi_dev)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+
+ atomic_dec(&mhi_dev->dev_wake);
+ read_lock_bh(&mhi_cntrl->pm_lock);
+ mhi_cntrl->wake_put(mhi_cntrl, false);
+ read_unlock_bh(&mhi_cntrl->pm_lock);
+}
+EXPORT_SYMBOL(mhi_device_put);
+
+int mhi_force_rddm_mode(struct mhi_controller *mhi_cntrl)
+{
+ int ret;
+
+ MHI_LOG("Enter with pm_state:%s ee:%s\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_EXEC_STR(mhi_cntrl->ee));
+
+ MHI_LOG("Triggering SYS_ERR to force rddm state\n");
+ mhi_set_mhi_state(mhi_cntrl, MHI_STATE_SYS_ERR);
+
+ /* wait for rddm event */
+ MHI_LOG("Waiting for device to enter RDDM state\n");
+ ret = wait_event_timeout(mhi_cntrl->state_event,
+ mhi_cntrl->ee == MHI_EE_RDDM,
+ msecs_to_jiffies(mhi_cntrl->timeout_ms));
+ ret = ret ? 0 : -EIO;
+
+ MHI_LOG("Exiting with pm_state:%s ee:%s ret:%d\n",
+ to_mhi_pm_state_str(mhi_cntrl->pm_state),
+ TO_MHI_EXEC_STR(mhi_cntrl->ee), ret);
+
+ return ret;
+}
+EXPORT_SYMBOL(mhi_force_rddm_mode);
diff --git a/drivers/bus/mhi/devices/Kconfig b/drivers/bus/mhi/devices/Kconfig
new file mode 100644
index 000000000000..c95d950a6a15
--- /dev/null
+++ b/drivers/bus/mhi/devices/Kconfig
@@ -0,0 +1,43 @@
+menu "MHI device support"
+
+config MHI_NETDEV
+ tristate "MHI NETDEV"
+ depends on MHI_BUS
+ help
+ MHI based net device driver for transferring IP traffic
+ between host and modem. By enabling this driver, clients
+ can transfer data using standard network interface. Over
+ the air traffic goes thru mhi netdev interface.
+
+config MHI_UCI
+ tristate "MHI UCI"
+ depends on MHI_BUS
+ help
+ MHI based uci driver is for transferring data between host and
+ modem using standard file operations from user space. Open, read,
+ write, ioctl, and close operations are supported by this driver.
+ Please check mhi_uci_match_table for all supported channels that
+ are exposed to userspace.
+
+config MHI_MBIM
+ bool "MHI MBIM"
+ default y
+ depends on MHI_UCI && MHI_NETDEV
+ help
+ This driver provides support for MHI MBIM (Mobile Broadband
+ Interface Model) devices.
+ T99W175 can not support MBIM/RmNet on the same time, choose Y
+ here for MBIM, N for RmNet.
+
+config MHI_SATELLITE
+ tristate "MHI SATELLITE"
+ depends on MHI_BUS
+ help
+ MHI proxy satellite device driver enables NON-HLOS MHI satellite
+ drivers to communicate with device over PCIe link without host
+ involvement. Host facilitates propagation of events from device
+ to NON-HLOS MHI satellite drivers, channel states, and power
+ management over IPC communication. It helps in HLOS power
+ savings.
+
+endmenu
diff --git a/drivers/bus/mhi/devices/Makefile b/drivers/bus/mhi/devices/Makefile
new file mode 100644
index 000000000000..e720069fd033
--- /dev/null
+++ b/drivers/bus/mhi/devices/Makefile
@@ -0,0 +1,3 @@
+obj-$(CONFIG_MHI_NETDEV) +=mhi_netdev.o
+obj-$(CONFIG_MHI_UCI) +=mhi_uci.o
+obj-$(CONFIG_MHI_SATELLITE) +=mhi_satellite.o
diff --git a/drivers/bus/mhi/devices/mhi_netdev.c b/drivers/bus/mhi/devices/mhi_netdev.c
new file mode 100644
index 000000000000..b61b73bb6fe0
--- /dev/null
+++ b/drivers/bus/mhi/devices/mhi_netdev.c
@@ -0,0 +1,1830 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.*/
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/netdevice.h>
+#include <linux/skbuff.h>
+#include <linux/if_arp.h>
+#include <linux/dma-mapping.h>
+#include <linux/debugfs.h>
+#include <linux/ipc_logging.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/of_device.h>
+#include <linux/rtnetlink.h>
+#include <linux/mhi.h>
+#include <linux/atomic.h>
+#include <linux/mii.h>
+#include <linux/usb.h>
+#include <linux/usb/cdc.h>
+#include <linux/usb/usbnet.h>
+#include <linux/usb/cdc_ncm.h>
+#include <linux/if_vlan.h>
+#include <linux/spinlock.h>
+#include <linux/rculist.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#ifndef CONFIG_MHI_MBIM
+#include <linux/msm_rmnet.h>
+#endif
+
+#define MHI_NETDEV_DRIVER_NAME "mhi_netdev"
+#define WATCHDOG_TIMEOUT (30 * HZ)
+#define IPC_LOG_PAGES (100)
+#define MAX_NETBUF_SIZE (128)
+
+#define CDC_NCM_LOW_MEM_MAX_CNT 10
+
+#ifdef CONFIG_MHI_DEBUG
+
+#define IPC_LOG_LVL (MHI_MSG_LVL_VERBOSE)
+
+#define MHI_ASSERT(cond, msg) do { \
+ if (cond) \
+ panic(msg); \
+} while (0)
+
+#define MSG_VERB(fmt, ...) do { \
+ if (mhi_netdev->msg_lvl <= MHI_MSG_LVL_VERBOSE) \
+ pr_err("[D][%s] " fmt, __func__, ##__VA_ARGS__);\
+ if (mhi_netdev->ipc_log && (mhi_netdev->ipc_log_lvl <= \
+ MHI_MSG_LVL_VERBOSE)) \
+ ipc_log_string(mhi_netdev->ipc_log, "[D][%s] " fmt, \
+ __func__, ##__VA_ARGS__); \
+} while (0)
+
+#else
+
+#define IPC_LOG_LVL (MHI_MSG_LVL_ERROR)
+
+#define MHI_ASSERT(cond, msg) do { \
+ if (cond) { \
+ MSG_ERR(msg); \
+ WARN_ON(cond); \
+ } \
+} while (0)
+
+#define MSG_VERB(fmt, ...)
+
+#endif
+
+#define MSG_LOG(fmt, ...) do { \
+ if (mhi_netdev->msg_lvl <= MHI_MSG_LVL_INFO) \
+ pr_err("[I][%s] " fmt, __func__, ##__VA_ARGS__);\
+ if (mhi_netdev->ipc_log && (mhi_netdev->ipc_log_lvl <= \
+ MHI_MSG_LVL_INFO)) \
+ ipc_log_string(mhi_netdev->ipc_log, "[I][%s] " fmt, \
+ __func__, ##__VA_ARGS__); \
+} while (0)
+
+#define MSG_ERR(fmt, ...) do { \
+ if (mhi_netdev->msg_lvl <= MHI_MSG_LVL_ERROR) \
+ pr_err("[E][%s] " fmt, __func__, ##__VA_ARGS__); \
+ if (mhi_netdev->ipc_log && (mhi_netdev->ipc_log_lvl <= \
+ MHI_MSG_LVL_ERROR)) \
+ ipc_log_string(mhi_netdev->ipc_log, "[E][%s] " fmt, \
+ __func__, ##__VA_ARGS__); \
+} while (0)
+
+struct mhi_net_chain {
+ struct sk_buff *head, *tail; /* chained skb */
+};
+
+struct mhi_netdev {
+ int alias;
+ struct mhi_device *mhi_dev;
+ struct mhi_netdev *rsc_dev; /* rsc linked node */
+ bool is_rsc_dev;
+ int wake;
+
+ u32 mru;
+ u32 order;
+ const char *interface_name;
+ struct napi_struct *napi;
+ struct net_device *ndev;
+ bool ethernet_interface;
+
+ struct mhi_netbuf **netbuf_pool;
+ int pool_size; /* must be power of 2 */
+ int current_index;
+ bool chain_skb;
+ struct mhi_net_chain *chain;
+
+ struct dentry *dentry;
+ enum MHI_DEBUG_LEVEL msg_lvl;
+ enum MHI_DEBUG_LEVEL ipc_log_lvl;
+ void *ipc_log;
+ unsigned long data[5];
+};
+
+struct mhi_netdev_priv {
+ struct mhi_netdev *mhi_netdev;
+};
+
+/* Try not to make this structure bigger than 128 bytes, since this take space
+ * in payload packet.
+ * Example: If MRU = 16K, effective MRU = 16K - sizeof(mhi_netbuf)
+ */
+struct mhi_netbuf {
+ struct mhi_buf mhi_buf; /* this must be first element */
+ void (*unmap)(struct device *dev, dma_addr_t addr, size_t size,
+ enum dma_data_direction dir);
+};
+
+#ifndef CONFIG_OF
+struct mhi_netdev_init {
+ const char chan[MHI_NAME_SIZE];
+ u32 mru;
+ bool no_chain;
+ int alias;
+ const char *interface_name;
+ bool ethernet_interface;
+};
+
+struct mhi_netdev_init mhi_netdev_init_array[] = {
+#ifdef CONFIG_MHI_MBIM
+ { "IP_HW0", 0x8000, 0, 0, "mbim_mhi", 0 },
+#else
+ { "IP_SW0", 0x4000, 1, 0, "rmnet_mhi_sw", 0 },
+ { "IP_HW0", 0x4000, 0, 0, "rmnet_mhi", 0 },
+#endif
+};
+#endif
+
+#define MBIM_SKB_HEADROOM 64
+
+/* alternative VLAN for IP session 0 if not untagged */
+#define MBIM_IPS0_VID 4094
+
+static struct mhi_driver mhi_netdev_driver;
+static void mhi_netdev_create_debugfs(struct mhi_netdev *mhi_netdev);
+
+static int mhi_netdev_xmit(struct sk_buff *skb, struct net_device *dev);
+
+#ifndef CONFIG_MHI_MBIM
+static __be16 mhi_netdev_ip_type_trans(u8 data)
+{
+ __be16 protocol = 0;
+
+ /* determine L3 protocol */
+ switch (data & 0xf0) {
+ case 0x40:
+ protocol = htons(ETH_P_IP);
+ break;
+ case 0x60:
+ protocol = htons(ETH_P_IPV6);
+ break;
+ default:
+ /* default is QMAP */
+ protocol = htons(ETH_P_MAP);
+ break;
+ }
+ return protocol;
+}
+#endif
+
+static struct mhi_netbuf *mhi_netdev_alloc(struct device *dev,
+ gfp_t gfp, unsigned int order)
+{
+ struct page *page;
+ struct mhi_netbuf *netbuf;
+ struct mhi_buf *mhi_buf;
+ void *vaddr;
+
+ page = __dev_alloc_pages(gfp, order);
+ if (!page)
+ return NULL;
+
+ vaddr = page_address(page);
+
+ /* we going to use the end of page to store cached data */
+ netbuf = vaddr + (PAGE_SIZE << order) - sizeof(*netbuf);
+
+ mhi_buf = (struct mhi_buf *)netbuf;
+ mhi_buf->page = page;
+ mhi_buf->buf = vaddr;
+ mhi_buf->len = (void *)netbuf - vaddr;
+ mhi_buf->dma_addr = dma_map_page(dev, page, 0, mhi_buf->len,
+ DMA_FROM_DEVICE);
+ if (dma_mapping_error(dev, mhi_buf->dma_addr)) {
+ __free_pages(mhi_buf->page, order);
+ return NULL;
+ }
+
+ return netbuf;
+}
+
+static void mhi_netdev_unmap_page(struct device *dev,
+ dma_addr_t dma_addr,
+ size_t len, enum dma_data_direction dir)
+{
+ dma_unmap_page(dev, dma_addr, len, dir);
+}
+
+static int mhi_netdev_tmp_alloc(struct mhi_netdev *mhi_netdev, int nr_tre)
+{
+ struct mhi_device *mhi_dev = mhi_netdev->mhi_dev;
+ struct device *dev = mhi_dev->dev.parent;
+ const u32 order = mhi_netdev->order;
+ int i, ret;
+
+ for (i = 0; i < nr_tre; i++) {
+ struct mhi_buf *mhi_buf;
+ struct mhi_netbuf *netbuf = mhi_netdev_alloc(dev, GFP_ATOMIC,
+ order);
+ if (!netbuf)
+ return -ENOMEM;
+
+ mhi_buf = (struct mhi_buf *)netbuf;
+ netbuf->unmap = mhi_netdev_unmap_page;
+
+ ret = mhi_queue_transfer(mhi_dev, DMA_FROM_DEVICE, mhi_buf,
+ mhi_buf->len, MHI_EOT);
+ if (unlikely(ret)) {
+ MSG_ERR("Failed to queue transfer, ret:%d\n", ret);
+ mhi_netdev_unmap_page(dev, mhi_buf->dma_addr,
+ mhi_buf->len, DMA_FROM_DEVICE);
+ __free_pages(mhi_buf->page, order);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static void mhi_netdev_queue(struct mhi_netdev *mhi_netdev)
+{
+ struct mhi_device *mhi_dev = mhi_netdev->mhi_dev;
+ struct device *dev = mhi_dev->dev.parent;
+ struct mhi_netbuf *netbuf;
+ struct mhi_buf *mhi_buf;
+ struct mhi_netbuf **netbuf_pool = mhi_netdev->netbuf_pool;
+ int nr_tre = mhi_get_no_free_descriptors(mhi_dev, DMA_FROM_DEVICE);
+ int i, peak, cur_index, ret;
+ const int pool_size = mhi_netdev->pool_size - 1, max_peak = 4;
+
+ MSG_VERB("Enter free_desc:%d\n", nr_tre);
+
+ if (!nr_tre)
+ return;
+
+ /* try going thru reclaim pool first */
+ for (i = 0; i < nr_tre; i++) {
+ /* peak for the next buffer, we going to peak several times,
+ * and we going to give up if buffers are not yet free
+ */
+ cur_index = mhi_netdev->current_index;
+ netbuf = NULL;
+ for (peak = 0; peak < max_peak; peak++) {
+ struct mhi_netbuf *tmp = netbuf_pool[cur_index];
+
+ mhi_buf = &tmp->mhi_buf;
+
+ cur_index = (cur_index + 1)&pool_size;
+
+ /* page == 1 idle, buffer is free to reclaim */
+ if (atomic_read(&mhi_buf->page->_refcount) == 1) {
+ netbuf = tmp;
+ break;
+ }
+ }
+
+ /* could not find a free buffer */
+ if (!netbuf)
+ break;
+
+ /* increment reference count so when network stack is done
+ * with buffer, the buffer won't be freed
+ */
+ atomic_inc(&mhi_buf->page->_refcount);
+ dma_sync_single_for_device(dev, mhi_buf->dma_addr, mhi_buf->len,
+ DMA_FROM_DEVICE);
+ ret = mhi_queue_transfer(mhi_dev, DMA_FROM_DEVICE, mhi_buf,
+ mhi_buf->len, MHI_EOT);
+ if (unlikely(ret)) {
+ MSG_ERR("Failed to queue buffer, ret:%d\n", ret);
+ netbuf->unmap(dev, mhi_buf->dma_addr, mhi_buf->len,
+ DMA_FROM_DEVICE);
+ atomic_dec(&mhi_buf->page->_refcount);
+ return;
+ }
+ mhi_netdev->current_index = cur_index;
+ }
+
+ /* recyling did not work, buffers are still busy allocate temp pkts */
+ if (i < nr_tre)
+ mhi_netdev_tmp_alloc(mhi_netdev, nr_tre - i);
+}
+
+/* allocating pool of memory */
+static int mhi_netdev_alloc_pool(struct mhi_netdev *mhi_netdev)
+{
+ int i;
+ struct mhi_netbuf *netbuf, **netbuf_pool;
+ struct mhi_buf *mhi_buf;
+ const u32 order = mhi_netdev->order;
+ struct device *dev = mhi_netdev->mhi_dev->dev.parent;
+
+ netbuf_pool = kmalloc_array(mhi_netdev->pool_size, sizeof(*netbuf_pool),
+ GFP_KERNEL);
+ if (!netbuf_pool)
+ return -ENOMEM;
+
+ for (i = 0; i < mhi_netdev->pool_size; i++) {
+ /* allocate paged data */
+ netbuf = mhi_netdev_alloc(dev, GFP_KERNEL, order);
+ if (!netbuf)
+ goto error_alloc_page;
+
+ netbuf->unmap = dma_sync_single_for_cpu;
+ netbuf_pool[i] = netbuf;
+ }
+
+ mhi_netdev->netbuf_pool = netbuf_pool;
+
+ return 0;
+
+error_alloc_page:
+ for (--i; i >= 0; i--) {
+ netbuf = netbuf_pool[i];
+ mhi_buf = &netbuf->mhi_buf;
+ dma_unmap_page(dev, mhi_buf->dma_addr, mhi_buf->len,
+ DMA_FROM_DEVICE);
+ __free_pages(mhi_buf->page, order);
+ }
+
+ kfree(netbuf_pool);
+
+ return -ENOMEM;
+}
+
+static void mhi_netdev_free_pool(struct mhi_netdev *mhi_netdev)
+{
+ int i;
+ struct mhi_netbuf *netbuf, **netbuf_pool = mhi_netdev->netbuf_pool;
+ struct device *dev = mhi_netdev->mhi_dev->dev.parent;
+ struct mhi_buf *mhi_buf;
+
+ for (i = 0; i < mhi_netdev->pool_size; i++) {
+ netbuf = netbuf_pool[i];
+ mhi_buf = &netbuf->mhi_buf;
+ dma_unmap_page(dev, mhi_buf->dma_addr, mhi_buf->len,
+ DMA_FROM_DEVICE);
+ __free_pages(mhi_buf->page, mhi_netdev->order);
+ }
+
+ kfree(mhi_netdev->netbuf_pool);
+ mhi_netdev->netbuf_pool = NULL;
+}
+
+#ifdef CONFIG_MHI_MBIM
+/* verify NTB header and return offset of first NDP, or negative error */
+static int cdc_ncm_mhi_rx_verify_nth16(struct cdc_ncm_ctx *ctx,
+ struct sk_buff *skb_in)
+{
+ struct usb_cdc_ncm_nth16 *nth16;
+ int len;
+ int ret = -EINVAL;
+
+ if (ctx == NULL)
+ goto error;
+
+ if (skb_in->len < (sizeof(struct usb_cdc_ncm_nth16) +
+ sizeof(struct usb_cdc_ncm_ndp16))) {
+ pr_err("frame too short\n");
+ goto error;
+ }
+
+ nth16 = (struct usb_cdc_ncm_nth16 *)skb_in->data;
+
+ if (nth16->dwSignature != cpu_to_le32(USB_CDC_NCM_NTH16_SIGN)) {
+ pr_err("invalid NTH16 signature <%#010x>\n",
+ le32_to_cpu(nth16->dwSignature));
+ goto error;
+ }
+
+ len = le16_to_cpu(nth16->wBlockLength);
+ if (len > ctx->rx_max) {
+ pr_err("unsupported NTB block length %u/%u\n", len,
+ ctx->rx_max);
+ goto error;
+ }
+
+ if ((ctx->rx_seq + 1) != le16_to_cpu(nth16->wSequence) &&
+ (ctx->rx_seq || le16_to_cpu(nth16->wSequence)) &&
+ !((ctx->rx_seq == 0xffff) && !le16_to_cpu(nth16->wSequence))) {
+ pr_err("sequence number glitch prev=%d curr=%d\n",
+ ctx->rx_seq, le16_to_cpu(nth16->wSequence));
+ }
+ ctx->rx_seq = le16_to_cpu(nth16->wSequence);
+
+ ret = le16_to_cpu(nth16->wNdpIndex);
+error:
+ return ret;
+}
+
+/* verify NDP header and return number of datagrams, or negative error */
+static int cdc_ncm_mhi_rx_verify_ndp16(struct sk_buff *skb_in, int ndpoffset)
+{
+ struct usb_cdc_ncm_ndp16 *ndp16;
+ int ret = -EINVAL;
+
+ if ((ndpoffset + sizeof(struct usb_cdc_ncm_ndp16)) > skb_in->len) {
+ pr_err("invalid NDP offset <%u>\n", ndpoffset);
+ goto error;
+ }
+ ndp16 = (struct usb_cdc_ncm_ndp16 *)(skb_in->data + ndpoffset);
+
+ if (le16_to_cpu(ndp16->wLength) < USB_CDC_NCM_NDP16_LENGTH_MIN) {
+ pr_err("invalid DPT16 length <%u>\n",
+ le16_to_cpu(ndp16->wLength));
+ goto error;
+ }
+
+ ret = ((le16_to_cpu(ndp16->wLength) -
+ sizeof(struct usb_cdc_ncm_ndp16)) /
+ sizeof(struct usb_cdc_ncm_dpe16));
+ ret--; /* we process NDP entries except for the last one */
+
+ if ((sizeof(struct usb_cdc_ncm_ndp16) +
+ ret * (sizeof(struct usb_cdc_ncm_dpe16))) > skb_in->len) {
+ pr_err("Invalid nframes = %d\n", ret);
+ ret = -EINVAL;
+ }
+
+error:
+ return ret;
+}
+
+static struct sk_buff *cdc_mbim_mhi_process_dgram(struct mhi_netdev *dev,
+ u8 *buf, size_t len, u16 tci)
+{
+ __be16 proto = htons(ETH_P_802_3);
+ struct sk_buff *skb = NULL;
+
+ if (tci < 256 || tci == MBIM_IPS0_VID) { /* IPS session? */
+ if (len < sizeof(struct iphdr))
+ goto err;
+
+ switch (*buf & 0xf0) {
+ case 0x40:
+ proto = htons(ETH_P_IP);
+ break;
+ case 0x60:
+ /*
+ * if (is_neigh_solicit(buf, len))
+ * do_neigh_solicit(dev, buf, tci);
+ */
+ proto = htons(ETH_P_IPV6);
+ break;
+ default:
+ goto err;
+ }
+ }
+
+ skb = netdev_alloc_skb_ip_align(dev->ndev, len + ETH_HLEN);
+ if (!skb)
+ goto err;
+
+ /* add an ethernet header */
+ skb_put(skb, ETH_HLEN);
+ skb_reset_mac_header(skb);
+ eth_hdr(skb)->h_proto = proto;
+ eth_zero_addr(eth_hdr(skb)->h_source);
+ memcpy(eth_hdr(skb)->h_dest, dev->ndev->dev_addr, ETH_ALEN);
+
+ /* add datagram */
+ skb_put_data(skb, buf, len);
+
+ /* map MBIM session to VLAN */
+ if (tci)
+ __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tci);
+err:
+ return skb;
+}
+
+static int cdc_ncm_mhi_rx_fixup(struct mhi_netdev *dev, struct sk_buff *skb_in)
+{
+ struct sk_buff *skb;
+ struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
+ int len;
+ int nframes;
+ int x;
+ int offset;
+ struct usb_cdc_ncm_ndp16 *ndp16;
+ struct usb_cdc_ncm_dpe16 *dpe16;
+ int ndpoffset;
+ int loopcount = 50; /* arbitrary max preventing infinite loop */
+ u32 payload = 0;
+ u8 *c;
+ u16 tci;
+
+ ndpoffset = cdc_ncm_mhi_rx_verify_nth16(ctx, skb_in);
+ if (ndpoffset < 0)
+ goto error;
+
+next_ndp:
+ nframes = cdc_ncm_mhi_rx_verify_ndp16(skb_in, ndpoffset);
+ if (nframes < 0)
+ goto error;
+
+ ndp16 = (struct usb_cdc_ncm_ndp16 *)(skb_in->data + ndpoffset);
+
+ switch (ndp16->dwSignature & cpu_to_le32(0x00ffffff)) {
+ case cpu_to_le32(USB_CDC_MBIM_NDP16_IPS_SIGN):
+ c = (u8 *)&ndp16->dwSignature;
+ tci = c[3];
+ /* tag IPS<0> packets too if MBIM_IPS0_VID exists */
+ /*
+ * if (!tci && info->flags & FLAG_IPS0_VLAN)
+ * tci = MBIM_IPS0_VID;
+ */
+ break;
+ case cpu_to_le32(USB_CDC_MBIM_NDP16_DSS_SIGN):
+ c = (u8 *)&ndp16->dwSignature;
+ tci = c[3] + 256;
+ break;
+ default:
+ pr_err("unsupported NDP signature <0x%08x>\n",
+ le32_to_cpu(ndp16->dwSignature));
+ goto err_ndp;
+
+ }
+
+ dpe16 = ndp16->dpe16;
+ for (x = 0; x < nframes; x++, dpe16++) {
+ offset = le16_to_cpu(dpe16->wDatagramIndex);
+ len = le16_to_cpu(dpe16->wDatagramLength);
+ /*
+ * CDC NCM ch. 3.7
+ * All entries after first NULL entry are to be ignored
+ */
+ if ((offset == 0) || (len == 0)) {
+ if (!x)
+ goto err_ndp; /* empty NTB */
+ break;
+ }
+
+ /* sanity checking */
+ if (((offset + len) > skb_in->len) || (len > ctx->rx_max)) {
+ pr_err("Invalid frame detected(ignored)\n");
+ pr_err(" Offset[%u]=%u, length=%u, skb=%p\n",
+ x, offset, len, skb_in);
+ if (!x)
+ goto err_ndp;
+ break;
+ }
+
+ skb = cdc_mbim_mhi_process_dgram(dev, skb_in->data + offset,
+ len, tci);
+ if (!skb)
+ goto error;
+
+ if (skb->protocol == 0)
+ skb->protocol = eth_type_trans(skb, dev->ndev);
+
+ memset(skb->cb, 0, sizeof(struct skb_data));
+
+ netif_rx(skb);
+
+ payload += len; /* count payload bytes in this NTB */
+ }
+
+err_ndp:
+ /* are there more NDPs to process? */
+ ndpoffset = le16_to_cpu(ndp16->wNextNdpIndex);
+ if (ndpoffset && loopcount--)
+ goto next_ndp;
+
+ /* update stats */
+ ctx->rx_overhead += skb_in->len - payload;
+ ctx->rx_ntbs++;
+ return 1;
+error:
+ return 0;
+}
+#endif /* CONFIG_MHI_MBIM */
+
+static int mhi_netdev_poll(struct napi_struct *napi, int budget)
+{
+ struct net_device *dev = napi->dev;
+ struct mhi_netdev_priv *mhi_netdev_priv = netdev_priv(dev);
+ struct mhi_netdev *mhi_netdev = mhi_netdev_priv->mhi_netdev;
+ struct mhi_device *mhi_dev = mhi_netdev->mhi_dev;
+ struct mhi_netdev *rsc_dev = mhi_netdev->rsc_dev;
+#ifndef CONFIG_MHI_MBIM
+ struct mhi_net_chain *chain = mhi_netdev->chain;
+#endif
+ int rx_work = 0;
+
+ MSG_VERB("Entered\n");
+
+ rx_work = mhi_poll(mhi_dev, budget);
+
+#ifndef CONFIG_MHI_MBIM
+ /* chained skb, push it to stack */
+ if (chain && chain->head) {
+ netif_receive_skb(chain->head);
+ chain->head = NULL;
+ }
+#endif
+
+ if (rx_work < 0) {
+ MSG_ERR("Error polling ret:%d\n", rx_work);
+ napi_complete(napi);
+ return 0;
+ }
+
+ /* queue new buffers */
+ mhi_netdev_queue(mhi_netdev);
+
+ if (rsc_dev)
+ mhi_netdev_queue(rsc_dev);
+
+ /* complete work if # of packet processed less than allocated budget */
+ if (rx_work < budget)
+ napi_complete(napi);
+
+ MSG_VERB("polled %d pkts\n", rx_work);
+
+ return rx_work;
+}
+
+static int mhi_netdev_open(struct net_device *dev)
+{
+ struct mhi_netdev_priv *mhi_netdev_priv = netdev_priv(dev);
+ struct mhi_netdev *mhi_netdev = mhi_netdev_priv->mhi_netdev;
+ struct mhi_device *mhi_dev = mhi_netdev->mhi_dev;
+
+ MSG_LOG("Opened net dev interface\n");
+
+ /* tx queue may not necessarily be stopped already
+ * so stop the queue if tx path is not enabled
+ */
+ if (!mhi_dev->ul_chan)
+ netif_stop_queue(dev);
+ else
+ netif_start_queue(dev);
+
+ return 0;
+
+}
+
+static int mhi_netdev_change_mtu(struct net_device *dev, int new_mtu)
+{
+ struct mhi_netdev_priv *mhi_netdev_priv = netdev_priv(dev);
+ struct mhi_netdev *mhi_netdev = mhi_netdev_priv->mhi_netdev;
+ struct mhi_device *mhi_dev = mhi_netdev->mhi_dev;
+
+ if (new_mtu < 0 || mhi_dev->mtu < new_mtu)
+ return -EINVAL;
+
+ dev->mtu = new_mtu;
+ return 0;
+}
+
+#ifdef CONFIG_MHI_MBIM
+static void cdc_ncm_align_tail(struct sk_buff *skb, size_t modulus,
+ size_t remainder, size_t max)
+{
+ size_t align = ALIGN(skb->len, modulus) - skb->len + remainder;
+
+ if (skb->len + align > max)
+ align = max - skb->len;
+ if (align && skb_tailroom(skb) >= align)
+ skb_put_zero(skb, align);
+}
+
+/* return a pointer to a valid struct usb_cdc_ncm_ndp16 of type sign,
+ * possibly allocating a new one within skb
+ */
+static struct usb_cdc_ncm_ndp16 *cdc_ncm_ndp(struct cdc_ncm_ctx *ctx,
+ struct sk_buff *skb, __le32 sign,
+ size_t reserve)
+{
+ struct usb_cdc_ncm_ndp16 *ndp16 = NULL;
+ struct usb_cdc_ncm_nth16 *nth16 = (void *)skb->data;
+ size_t ndpoffset = le16_to_cpu(nth16->wNdpIndex);
+
+ /* If NDP should be moved to the end of the NCM package,
+ * we can't follow the NTH16 header as we would normally do.
+ * NDP isn't written to the SKB yet, and the wNdpIndex field
+ * in the header is actually not consistent with reality.
+ * It will be later.
+ */
+ if (ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END) {
+ if (ctx->delayed_ndp16->dwSignature == sign)
+ return ctx->delayed_ndp16;
+
+ /* We can only push a single NDP to the end. Return
+ * NULL to send what we've already got and queue this
+ * skb for later.
+ */
+ else if (ctx->delayed_ndp16->dwSignature)
+ return NULL;
+ }
+
+ /* follow the chain of NDPs, looking for a match */
+ while (ndpoffset) {
+ ndp16 = (struct usb_cdc_ncm_ndp16 *)(skb->data + ndpoffset);
+ if (ndp16->dwSignature == sign)
+ return ndp16;
+ ndpoffset = le16_to_cpu(ndp16->wNextNdpIndex);
+ }
+
+ /* align new NDP */
+ if (!(ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END))
+ cdc_ncm_align_tail(skb, ctx->tx_ndp_modulus, 0,
+ ctx->tx_curr_size);
+
+ /* verify that there is room for the NDP and the datagram (reserve) */
+ if ((ctx->tx_curr_size - skb->len - reserve) < ctx->max_ndp_size)
+ return NULL;
+
+ /* link to it */
+ if (ndp16)
+ ndp16->wNextNdpIndex = cpu_to_le16(skb->len);
+ else
+ nth16->wNdpIndex = cpu_to_le16(skb->len);
+
+ /* push a new empty NDP */
+ if (!(ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END))
+ ndp16 = skb_put_zero(skb, ctx->max_ndp_size);
+ else
+ ndp16 = ctx->delayed_ndp16;
+
+ ndp16->dwSignature = sign;
+ ndp16->wLength = cpu_to_le16(sizeof(struct usb_cdc_ncm_ndp16) +
+ sizeof(struct usb_cdc_ncm_dpe16));
+ return ndp16;
+}
+
+static void cdc_ncm_mhi_txpath_bh(unsigned long param)
+{
+ struct mhi_netdev *dev = (struct mhi_netdev *)param;
+ struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
+
+ spin_lock_bh(&ctx->mtx);
+ if (dev->ndev != NULL) {
+ spin_unlock_bh(&ctx->mtx);
+ netif_tx_lock_bh(dev->ndev);
+ mhi_netdev_xmit(NULL, dev->ndev);
+ netif_tx_unlock_bh(dev->ndev);
+ } else {
+ spin_unlock_bh(&ctx->mtx);
+ }
+}
+
+struct sk_buff *cdc_ncm_mhi_fill_tx_frame(struct mhi_netdev *dev,
+ struct sk_buff *skb, __le32 sign)
+{
+ struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
+ struct usb_cdc_ncm_nth16 *nth16;
+ struct usb_cdc_ncm_ndp16 *ndp16;
+ struct sk_buff *skb_out;
+ u16 n = 0, index, ndplen;
+ u8 ready2send = 0;
+ u32 delayed_ndp_size;
+ size_t padding_count;
+
+ /* When our NDP gets written in cdc_ncm_ndp(), then skb_out->len
+ * gets updated accordingly. Otherwise, we should check here.
+ */
+ delayed_ndp_size = 0;
+
+ /* if there is a remaining skb, it gets priority */
+ if (skb != NULL) {
+ swap(skb, ctx->tx_rem_skb);
+ swap(sign, ctx->tx_rem_sign);
+ } else {
+ ready2send = 1;
+ }
+
+ /* check if we are resuming an OUT skb */
+ skb_out = ctx->tx_curr_skb;
+
+ /* allocate a new OUT skb */
+ if (!skb_out) {
+ if (ctx->tx_low_mem_val == 0) {
+ ctx->tx_curr_size = ctx->tx_max;
+ skb_out = alloc_skb(ctx->tx_curr_size, GFP_ATOMIC);
+ /* If the memory allocation fails we will wait longer
+ * each time before attempting another full size
+ * allocation again to not overload the system
+ * further.
+ */
+ if (skb_out == NULL) {
+ ctx->tx_low_mem_max_cnt =
+ min(ctx->tx_low_mem_max_cnt + 1,
+ (unsigned int)CDC_NCM_LOW_MEM_MAX_CNT);
+ ctx->tx_low_mem_val = ctx->tx_low_mem_max_cnt;
+ }
+ }
+ if (skb_out == NULL) {
+ /* See if a very small allocation is possible.
+ * We will send this packet immediately and hope
+ * that there is more memory available later.
+ */
+ if (skb) {
+ ctx->tx_curr_size = max_t(u32, skb->len,
+ USB_CDC_NCM_NTB_MIN_OUT_SIZE);
+ } else {
+ ctx->tx_curr_size =
+ USB_CDC_NCM_NTB_MIN_OUT_SIZE;
+ }
+ skb_out = alloc_skb(ctx->tx_curr_size, GFP_ATOMIC);
+
+ /* No allocation possible so we will abort */
+ if (skb_out == NULL) {
+ if (skb != NULL) {
+ dev_kfree_skb_any(skb);
+ skb = NULL;
+ }
+ dev->ndev->stats.tx_dropped++;
+ goto exit_no_skb;
+ }
+ ctx->tx_low_mem_val--;
+ }
+ /* fill out the initial 16-bit NTB header */
+ nth16 = skb_put_zero(skb_out,
+ sizeof(struct usb_cdc_ncm_nth16));
+ nth16->dwSignature = cpu_to_le32(USB_CDC_NCM_NTH16_SIGN);
+ nth16->wHeaderLength =
+ cpu_to_le16(sizeof(struct usb_cdc_ncm_nth16));
+ nth16->wSequence = cpu_to_le16(ctx->tx_seq++);
+
+ /* count total number of frames in this NTB */
+ ctx->tx_curr_frame_num = 0;
+
+ /* recent payload counter for this skb_out */
+ ctx->tx_curr_frame_payload = 0;
+ }
+
+ for (n = ctx->tx_curr_frame_num; n < ctx->tx_max_datagrams; n++) {
+ /* send any remaining skb first */
+ if (skb == NULL) {
+ skb = ctx->tx_rem_skb;
+ sign = ctx->tx_rem_sign;
+ ctx->tx_rem_skb = NULL;
+
+ /* check for end of skb */
+ if (skb == NULL)
+ break;
+ }
+
+ /* get the appropriate NDP for this skb */
+ ndp16 = cdc_ncm_ndp(ctx, skb_out, sign,
+ skb->len + ctx->tx_modulus + ctx->tx_remainder);
+
+ /* align beginning of next frame */
+ cdc_ncm_align_tail(skb_out, ctx->tx_modulus, ctx->tx_remainder,
+ ctx->tx_curr_size);
+
+ /* check if we had enough room left for both NDP and frame */
+ if (!ndp16 ||
+ skb_out->len + skb->len + delayed_ndp_size >
+ ctx->tx_curr_size) {
+ if (n == 0) {
+ /* won't fit, MTU problem? */
+ dev_kfree_skb_any(skb);
+ skb = NULL;
+ dev->ndev->stats.tx_dropped++;
+ } else {
+ /* no room for skb - store for later */
+ if (ctx->tx_rem_skb != NULL) {
+ dev_kfree_skb_any(ctx->tx_rem_skb);
+ ctx->tx_rem_skb = NULL;
+ dev->ndev->stats.tx_dropped++;
+ }
+ ctx->tx_rem_skb = skb;
+ ctx->tx_rem_sign = sign;
+ skb = NULL;
+ ready2send = 1;
+ ctx->tx_reason_ntb_full++; /* count reason */
+ }
+ break;
+ }
+
+ /* calculate frame number within this NDP */
+ ndplen = le16_to_cpu(ndp16->wLength);
+ index = (ndplen - sizeof(struct usb_cdc_ncm_ndp16)) /
+ sizeof(struct usb_cdc_ncm_dpe16) - 1;
+
+ /* OK, add this skb */
+ ndp16->dpe16[index].wDatagramLength = cpu_to_le16(skb->len);
+ ndp16->dpe16[index].wDatagramIndex = cpu_to_le16(skb_out->len);
+ ndp16->wLength =
+ cpu_to_le16(ndplen + sizeof(struct usb_cdc_ncm_dpe16));
+ skb_put_data(skb_out, skb->data, skb->len);
+ /* count real tx payload data */
+ ctx->tx_curr_frame_payload += skb->len;
+ dev_kfree_skb_any(skb);
+ skb = NULL;
+
+ /* send now if this NDP is full */
+ if (index >= CDC_NCM_DPT_DATAGRAMS_MAX) {
+ ready2send = 1;
+ break;
+ }
+ }
+
+ /* free up any dangling skb */
+ if (skb != NULL) {
+ dev_kfree_skb_any(skb);
+ skb = NULL;
+ dev->ndev->stats.tx_dropped++;
+ }
+
+ ctx->tx_curr_frame_num = n;
+
+ if ((n == 0) || ((n < ctx->tx_max_datagrams) && (ready2send == 0))) {
+ /* push variables */
+ ctx->tx_curr_skb = skb_out;
+ goto exit_no_skb;
+ }
+
+ /* If requested, put NDP at end of frame. */
+ if (ctx->drvflags & CDC_NCM_FLAG_NDP_TO_END) {
+ nth16 = (struct usb_cdc_ncm_nth16 *)skb_out->data;
+ cdc_ncm_align_tail(skb_out, ctx->tx_ndp_modulus, 0,
+ ctx->tx_curr_size - ctx->max_ndp_size);
+ nth16->wNdpIndex = cpu_to_le16(skb_out->len);
+ skb_put_data(skb_out, ctx->delayed_ndp16, ctx->max_ndp_size);
+
+ /* Zero out delayed NDP -
+ * signature checking will naturally fail.
+ */
+ ndp16 = memset(ctx->delayed_ndp16, 0, ctx->max_ndp_size);
+ }
+
+ /* If collected data size is less or equal ctx->min_tx_pkt
+ * bytes, we send buffers as it is. If we get more data, it
+ * would be more efficient for USB HS mobile device with DMA
+ * engine to receive a full size NTB, than canceling DMA
+ * transfer and receiving a short packet.
+ *
+ * This optimization support is pointless if we end up sending
+ * a ZLP after full sized NTBs.
+ */
+ if (skb_out->len > ctx->min_tx_pkt) {
+ padding_count = ctx->tx_curr_size - skb_out->len;
+ skb_put_zero(skb_out, padding_count);
+ } else if (skb_out->len < ctx->tx_curr_size) {
+ skb_put_u8(skb_out, 0); /* force short packet */
+ }
+
+ /* set final frame length */
+ nth16 = (struct usb_cdc_ncm_nth16 *)skb_out->data;
+ nth16->wBlockLength = cpu_to_le16(skb_out->len);
+
+ /* return skb */
+ ctx->tx_curr_skb = NULL;
+
+ dev->ndev->stats.tx_packets++;
+ dev->ndev->stats.tx_bytes += ctx->tx_curr_frame_payload;
+
+ return skb_out;
+
+exit_no_skb:
+ /*
+ * if (ctx->tx_curr_skb != NULL && n > 0) {
+ * tasklet_schedule(&ctx->bh);
+ * }
+ */
+ return NULL;
+}
+
+static struct sk_buff *cdc_mbim_mhi_tx_fixup(struct mhi_netdev *dev,
+ struct sk_buff *skb, gfp_t flags)
+{
+ struct sk_buff *skb_out;
+ struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)dev->data[0];
+
+ __le32 sign = cpu_to_le32(USB_CDC_MBIM_NDP16_IPS_SIGN);
+ u16 tci = 0;
+ u8 *c;
+
+ if (!ctx)
+ goto error;
+
+ if (skb) {
+
+ if (skb->len <= ETH_HLEN)
+ goto error;
+
+ /* Some applications using e.g. packet sockets will
+ * bypass the VLAN acceleration and create tagged
+ * ethernet frames directly. We primarily look for
+ * the accelerated out-of-band tag, but fall back if
+ * required
+ */
+ skb_reset_mac_header(skb);
+ if (vlan_get_tag(skb, &tci) < 0 && skb->len > VLAN_ETH_HLEN &&
+ __vlan_get_tag(skb, &tci) == 0) {
+ skb_pull(skb, VLAN_ETH_HLEN);
+ }
+
+ /* mapping VLANs to MBIM sessions:
+ * no tag => IPS session <0> if !FLAG_IPS0_VLAN
+ * 1 - 255 => IPS session <vlanid>
+ * 256 - 511 => DSS session <vlanid - 256>
+ * 512 - 4093 => unsupported, drop
+ * 4094 => IPS session <0> if FLAG_IPS0_VLAN
+ */
+
+ switch (tci & 0x0f00) {
+ case 0x0000: /* VLAN ID 0 - 255 */
+ c = (u8 *)&sign;
+ c[3] = tci;
+ break;
+ case 0x0100: /* VLAN ID 256 - 511 */
+ sign = cpu_to_le32(USB_CDC_MBIM_NDP16_DSS_SIGN);
+ c = (u8 *)&sign;
+ c[3] = tci;
+ break;
+ default:
+ pr_err("unsupported tci=0x%04x\n", tci);
+ goto error;
+ }
+ }
+
+ spin_lock_bh(&ctx->mtx);
+ skb_out = cdc_ncm_mhi_fill_tx_frame(dev, skb, sign);
+ spin_unlock_bh(&ctx->mtx);
+
+ return skb_out;
+
+error:
+ if (skb) {
+ dev_kfree_skb_any(skb);
+ skb = NULL;
+ }
+ dev->ndev->stats.tx_dropped++;
+ return NULL;
+}
+
+static void cdc_ncm_mhi_free(struct cdc_ncm_ctx *ctx)
+{
+ if (ctx == NULL)
+ return;
+
+ if (ctx->tx_rem_skb != NULL) {
+ dev_kfree_skb_any(ctx->tx_rem_skb);
+ ctx->tx_rem_skb = NULL;
+ }
+
+ if (ctx->tx_curr_skb != NULL) {
+ dev_kfree_skb_any(ctx->tx_curr_skb);
+ ctx->tx_curr_skb = NULL;
+ }
+ if (ctx->delayed_ndp16 != NULL)
+ kfree(ctx->delayed_ndp16);
+
+ kfree(ctx);
+}
+#endif
+
+static int mhi_netdev_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+ struct mhi_netdev_priv *mhi_netdev_priv = netdev_priv(dev);
+ struct mhi_netdev *mhi_netdev = mhi_netdev_priv->mhi_netdev;
+ struct mhi_device *mhi_dev = mhi_netdev->mhi_dev;
+ int res = 0;
+ int drop_num = mhi_netdev->ndev->stats.tx_dropped;
+ struct sk_buff *skb_in;
+ struct sk_buff *skb_out;
+
+ MSG_VERB("Entered\n");
+
+#ifdef CONFIG_MHI_MBIM
+ skb_in = skb_copy(skb, GFP_ATOMIC);
+ skb_in = cdc_mbim_mhi_tx_fixup(mhi_netdev, skb_in, GFP_ATOMIC);
+ if (mhi_netdev->ndev->stats.tx_dropped > drop_num)
+ goto exit_error;
+
+ if (!skb_in) {
+ skb_out = cdc_mbim_mhi_tx_fixup(mhi_netdev, NULL, GFP_ATOMIC);
+ if (!skb_out)
+ goto exit_error;
+ res = mhi_queue_transfer(mhi_dev, DMA_TO_DEVICE, skb_out,
+ skb_out->len, MHI_EOT);
+
+ if (!res) {
+ kfree_skb(skb);
+ } else {
+exit_error:
+ MSG_VERB("Failed to queue with reason:%d\n", res);
+ netif_stop_queue(dev);
+ kfree_skb(skb_out);
+ res = NETDEV_TX_BUSY;
+ }
+ }
+
+#else
+
+ res = mhi_queue_transfer(mhi_dev, DMA_TO_DEVICE, skb, skb->len,
+ MHI_EOT);
+ if (res) {
+ MSG_VERB("Failed to queue with reason:%d\n", res);
+ netif_stop_queue(dev);
+ res = NETDEV_TX_BUSY;
+ }
+#endif
+
+ MSG_VERB("Exited\n");
+
+ return res;
+}
+
+#ifndef CONFIG_MHI_MBIM
+static int mhi_netdev_ioctl_extended(struct net_device *dev, struct ifreq *ifr)
+{
+ struct rmnet_ioctl_extended_s ext_cmd;
+ int rc = 0;
+ struct mhi_netdev_priv *mhi_netdev_priv = netdev_priv(dev);
+ struct mhi_netdev *mhi_netdev = mhi_netdev_priv->mhi_netdev;
+ struct mhi_device *mhi_dev = mhi_netdev->mhi_dev;
+
+ rc = copy_from_user(&ext_cmd, ifr->ifr_ifru.ifru_data,
+ sizeof(struct rmnet_ioctl_extended_s));
+ if (rc)
+ return rc;
+
+ switch (ext_cmd.extended_ioctl) {
+ case RMNET_IOCTL_GET_SUPPORTED_FEATURES:
+ ext_cmd.u.data = 0;
+ break;
+ case RMNET_IOCTL_GET_DRIVER_NAME:
+ memset(ext_cmd.u.if_name, 0, sizeof(ext_cmd.u.if_name));
+ strncpy(ext_cmd.u.if_name, mhi_netdev->interface_name,
+ sizeof(ext_cmd.u.if_name) - 1);
+ break;
+ case RMNET_IOCTL_SET_SLEEP_STATE:
+ if (ext_cmd.u.data && mhi_netdev->wake) {
+ /* Request to enable LPM */
+ MSG_VERB("Enable MHI LPM");
+ mhi_netdev->wake--;
+ mhi_device_put(mhi_dev);
+ } else if (!ext_cmd.u.data && !mhi_netdev->wake) {
+ /* Request to disable LPM */
+ MSG_VERB("Disable MHI LPM");
+ mhi_netdev->wake++;
+ mhi_device_get(mhi_dev);
+ }
+ break;
+ default:
+ rc = -EINVAL;
+ break;
+ }
+
+ rc = copy_to_user(ifr->ifr_ifru.ifru_data, &ext_cmd,
+ sizeof(struct rmnet_ioctl_extended_s));
+ return rc;
+}
+
+static int mhi_netdev_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
+{
+ int rc = 0;
+ struct rmnet_ioctl_data_s ioctl_data;
+
+ switch (cmd) {
+ case RMNET_IOCTL_SET_LLP_IP: /* set RAWIP protocol */
+ break;
+ case RMNET_IOCTL_GET_LLP: /* get link protocol state */
+ ioctl_data.u.operation_mode = RMNET_MODE_LLP_IP;
+ if (copy_to_user(ifr->ifr_ifru.ifru_data, &ioctl_data,
+ sizeof(struct rmnet_ioctl_data_s)))
+ rc = -EFAULT;
+ break;
+ case RMNET_IOCTL_GET_OPMODE: /* get operation mode */
+ ioctl_data.u.operation_mode = RMNET_MODE_LLP_IP;
+ if (copy_to_user(ifr->ifr_ifru.ifru_data, &ioctl_data,
+ sizeof(struct rmnet_ioctl_data_s)))
+ rc = -EFAULT;
+ break;
+ case RMNET_IOCTL_SET_QOS_ENABLE:
+ rc = -EINVAL;
+ break;
+ case RMNET_IOCTL_SET_QOS_DISABLE:
+ rc = 0;
+ break;
+ case RMNET_IOCTL_OPEN:
+ case RMNET_IOCTL_CLOSE:
+ /* we just ignore them and return success */
+ rc = 0;
+ break;
+ case RMNET_IOCTL_EXTENDED:
+ rc = mhi_netdev_ioctl_extended(dev, ifr);
+ break;
+ default:
+ /* don't fail any IOCTL right now */
+ rc = 0;
+ break;
+ }
+
+ return rc;
+}
+#endif
+
+static const struct net_device_ops mhi_netdev_ops_ip = {
+ .ndo_open = mhi_netdev_open,
+ .ndo_start_xmit = mhi_netdev_xmit,
+#ifndef CONFIG_MHI_MBIM
+ .ndo_do_ioctl = mhi_netdev_ioctl,
+#endif
+ .ndo_change_mtu = mhi_netdev_change_mtu,
+ .ndo_set_mac_address = 0,
+ .ndo_validate_addr = 0,
+};
+
+static void mhi_netdev_setup(struct net_device *dev)
+{
+ dev->netdev_ops = &mhi_netdev_ops_ip;
+ ether_setup(dev);
+
+ /* set this after calling ether_setup */
+ dev->header_ops = 0; /* No header */
+ dev->type = ARPHRD_RAWIP;
+ dev->hard_header_len = 0;
+ dev->addr_len = 0;
+ dev->flags &= ~(IFF_BROADCAST | IFF_MULTICAST);
+ dev->watchdog_timeo = WATCHDOG_TIMEOUT;
+}
+
+/* enable mhi_netdev netdev, call only after grabbing mhi_netdev.mutex */
+#ifdef CONFIG_OF
+static int mhi_netdev_enable_iface(struct mhi_netdev *mhi_netdev)
+#else
+static int mhi_netdev_enable_iface(struct mhi_netdev *mhi_netdev, int index)
+#endif
+{
+ int ret = 0;
+ char ifalias[IFALIASZ];
+ char ifname[IFNAMSIZ];
+ struct mhi_device *mhi_dev = mhi_netdev->mhi_dev;
+ struct mhi_netdev_priv *mhi_netdev_priv;
+#ifdef CONFIG_OF
+ struct device_node *of_node = mhi_dev->dev.of_node;
+
+ mhi_netdev->alias = of_alias_get_id(of_node, "mhi-netdev");
+ if (mhi_netdev->alias < 0)
+ mhi_netdev->alias = 0;
+
+ ret = of_property_read_string(of_node, "mhi,interface-name",
+ &mhi_netdev->interface_name);
+ if (ret)
+ mhi_netdev->interface_name = mhi_netdev_driver.driver.name;
+
+ snprintf(ifalias, sizeof(ifalias), "%s_%04x_%02u.%02u.%02u_%u",
+ mhi_netdev->interface_name, mhi_dev->dev_id, mhi_dev->domain,
+ mhi_dev->bus, mhi_dev->slot, mhi_netdev->alias);
+
+ snprintf(ifname, sizeof(ifname), "%s%%d", mhi_netdev->interface_name);
+
+ mhi_netdev->ethernet_interface = of_property_read_bool(of_node,
+ "mhi,ethernet-interface");
+#else
+ mhi_netdev->alias = mhi_netdev_init_array[index].alias;
+
+ mhi_netdev->interface_name =
+ mhi_netdev_init_array[index].interface_name;
+
+ snprintf(ifalias, sizeof(ifalias), "%s_%04x_%02u.%02u.%02u_%u",
+ mhi_netdev->interface_name, mhi_dev->dev_id, mhi_dev->domain,
+ mhi_dev->bus, mhi_dev->slot, mhi_netdev->alias);
+
+ snprintf(ifname, sizeof(ifname), "%s%%d", mhi_netdev->interface_name);
+
+ mhi_netdev->ethernet_interface =
+ mhi_netdev_init_array[index].ethernet_interface;
+#endif
+ rtnl_lock();
+ mhi_netdev->ndev = alloc_netdev(sizeof(*mhi_netdev_priv),
+ ifname, NET_NAME_PREDICTABLE,
+ mhi_netdev_setup);
+ if (!mhi_netdev->ndev) {
+ rtnl_unlock();
+ return -ENOMEM;
+ }
+
+ mhi_netdev->ndev->mtu = mhi_dev->mtu;
+ SET_NETDEV_DEV(mhi_netdev->ndev, &mhi_dev->dev);
+ dev_set_alias(mhi_netdev->ndev, ifalias, strlen(ifalias));
+ mhi_netdev_priv = netdev_priv(mhi_netdev->ndev);
+ mhi_netdev_priv->mhi_netdev = mhi_netdev;
+ rtnl_unlock();
+
+ mhi_netdev->napi = devm_kzalloc(&mhi_dev->dev,
+ sizeof(*mhi_netdev->napi), GFP_KERNEL);
+ if (!mhi_netdev->napi) {
+ ret = -ENOMEM;
+ goto napi_alloc_fail;
+ }
+
+ netif_napi_add(mhi_netdev->ndev, mhi_netdev->napi,
+ mhi_netdev_poll, NAPI_POLL_WEIGHT);
+ ret = register_netdev(mhi_netdev->ndev);
+ if (ret) {
+ MSG_ERR("Network device registration failed\n");
+ goto net_dev_reg_fail;
+ }
+
+ napi_enable(mhi_netdev->napi);
+
+ MSG_LOG("Exited.\n");
+
+ return 0;
+
+net_dev_reg_fail:
+ netif_napi_del(mhi_netdev->napi);
+
+napi_alloc_fail:
+ free_netdev(mhi_netdev->ndev);
+ mhi_netdev->ndev = NULL;
+
+ return ret;
+}
+
+static void mhi_netdev_xfer_ul_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result)
+{
+ struct mhi_netdev *mhi_netdev = mhi_device_get_devdata(mhi_dev);
+ struct sk_buff *skb = mhi_result->buf_addr;
+ struct net_device *ndev = mhi_netdev->ndev;
+
+#ifndef CONFIG_MHI_MBIM
+ ndev->stats.tx_packets++;
+ ndev->stats.tx_bytes += skb->len;
+#endif
+
+ dev_kfree_skb(skb);
+
+ if (netif_queue_stopped(ndev))
+ netif_wake_queue(ndev);
+}
+
+#ifndef CONFIG_MHI_MBIM
+static void mhi_netdev_push_skb(struct mhi_netdev *mhi_netdev,
+ struct mhi_buf *mhi_buf,
+ struct mhi_result *mhi_result)
+{
+ struct sk_buff *skb;
+
+ skb = alloc_skb(0, GFP_ATOMIC);
+ if (!skb) {
+ __free_pages(mhi_buf->page, mhi_netdev->order);
+ return;
+ }
+
+ if (!mhi_netdev->ethernet_interface) {
+ skb_add_rx_frag(skb, 0, mhi_buf->page, 0,
+ mhi_result->bytes_xferd, mhi_netdev->mru);
+ skb->dev = mhi_netdev->ndev;
+ skb->protocol = mhi_netdev_ip_type_trans(*(u8 *)mhi_buf->buf);
+ } else {
+ skb_add_rx_frag(skb, 0, mhi_buf->page, ETH_HLEN,
+ mhi_result->bytes_xferd - ETH_HLEN,
+ mhi_netdev->mru);
+ skb->dev = mhi_netdev->ndev;
+ skb->protocol =
+ mhi_netdev_ip_type_trans(((u8 *)mhi_buf->buf)[ETH_HLEN]);
+ }
+ netif_receive_skb(skb);
+}
+#endif
+
+static void mhi_netdev_xfer_dl_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result)
+{
+ struct mhi_netdev *mhi_netdev = mhi_device_get_devdata(mhi_dev);
+ struct mhi_netbuf *netbuf = mhi_result->buf_addr;
+ struct mhi_buf *mhi_buf = &netbuf->mhi_buf;
+ struct sk_buff *skb;
+ struct net_device *ndev = mhi_netdev->ndev;
+ struct device *dev = mhi_dev->dev.parent;
+#ifndef CONFIG_MHI_MBIM
+ struct mhi_net_chain *chain = mhi_netdev->chain;
+#endif
+
+ netbuf->unmap(dev, mhi_buf->dma_addr, mhi_buf->len, DMA_FROM_DEVICE);
+
+ /* modem is down, drop the buffer */
+ if (mhi_result->transaction_status == -ENOTCONN) {
+ __free_pages(mhi_buf->page, mhi_netdev->order);
+ return;
+ }
+
+ ndev->stats.rx_packets++;
+ ndev->stats.rx_bytes += mhi_result->bytes_xferd;
+
+#ifdef CONFIG_MHI_MBIM
+
+ skb =
+ alloc_skb(MBIM_SKB_HEADROOM + mhi_result->bytes_xferd, GFP_ATOMIC);
+ if (!skb) {
+ __free_pages(mhi_buf->page, mhi_netdev->order);
+ return;
+ }
+
+ skb_reserve(skb, MBIM_SKB_HEADROOM);
+ skb_put_data(skb, mhi_buf->buf, mhi_result->bytes_xferd);
+ skb->dev = ndev;
+ cdc_ncm_mhi_rx_fixup(mhi_netdev, skb);
+ dev_kfree_skb_any(skb);
+ skb = NULL;
+ __free_pages(mhi_buf->page, mhi_netdev->order);
+
+#else
+
+ if (unlikely(!chain)) {
+ mhi_netdev_push_skb(mhi_netdev, mhi_buf, mhi_result);
+ return;
+ }
+
+ /* we support chaining */
+ skb = alloc_skb(0, GFP_ATOMIC);
+ if (likely(skb)) {
+ if (!mhi_netdev->ethernet_interface) {
+ skb_add_rx_frag(skb, 0, mhi_buf->page, 0,
+ mhi_result->bytes_xferd,
+ mhi_netdev->mru);
+ } else {
+ skb_add_rx_frag(skb, 0, mhi_buf->page, ETH_HLEN,
+ mhi_result->bytes_xferd - ETH_HLEN,
+ mhi_netdev->mru);
+ }
+
+ /* this is first on list */
+ if (!chain->head) {
+ skb->dev = ndev;
+ if (!mhi_netdev->ethernet_interface) {
+ skb->protocol =
+ mhi_netdev_ip_type_trans(
+ *(u8 *)mhi_buf->buf);
+ } else {
+ skb->protocol =
+ mhi_netdev_ip_type_trans(
+ ((u8 *)mhi_buf->buf)[ETH_HLEN]);
+ }
+ chain->head = skb;
+ } else {
+ skb_shinfo(chain->tail)->frag_list = skb;
+ }
+
+ chain->tail = skb;
+ } else {
+ __free_pages(mhi_buf->page, mhi_netdev->order);
+ }
+#endif
+
+}
+
+static void mhi_netdev_status_cb(struct mhi_device *mhi_dev, enum MHI_CB mhi_cb)
+{
+ struct mhi_netdev *mhi_netdev = mhi_device_get_devdata(mhi_dev);
+
+ if (mhi_cb != MHI_CB_PENDING_DATA)
+ return;
+
+ napi_schedule(mhi_netdev->napi);
+}
+
+#ifdef CONFIG_DEBUG_FS
+
+struct dentry *dentry;
+
+static void mhi_netdev_create_debugfs(struct mhi_netdev *mhi_netdev)
+{
+ char node_name[32];
+ struct mhi_device *mhi_dev = mhi_netdev->mhi_dev;
+
+ /* Both tx & rx client handle contain same device info */
+ snprintf(node_name, sizeof(node_name), "%s_%04x_%02u.%02u.%02u_%u",
+ mhi_netdev->interface_name, mhi_dev->dev_id, mhi_dev->domain,
+ mhi_dev->bus, mhi_dev->slot, mhi_netdev->alias);
+
+ if (IS_ERR_OR_NULL(dentry))
+ return;
+
+ mhi_netdev->dentry = debugfs_create_dir(node_name, dentry);
+ if (IS_ERR_OR_NULL(mhi_netdev->dentry))
+ return;
+}
+
+static void mhi_netdev_create_debugfs_dir(void)
+{
+ dentry = debugfs_create_dir(MHI_NETDEV_DRIVER_NAME, 0);
+}
+
+#else
+
+static void mhi_netdev_create_debugfs(struct mhi_netdev_private *mhi_netdev)
+{
+}
+
+static void mhi_netdev_create_debugfs_dir(void)
+{
+}
+
+#endif
+
+static void mhi_netdev_remove(struct mhi_device *mhi_dev)
+{
+ struct mhi_netdev *mhi_netdev = mhi_device_get_devdata(mhi_dev);
+#ifdef CONFIG_MHI_MBIM
+ struct cdc_ncm_ctx *ctx = (struct cdc_ncm_ctx *)mhi_netdev->data[0];
+#endif
+
+ MSG_LOG("Remove notification received\n");
+
+ /* rsc parent takes cares of the cleanup */
+ if (mhi_netdev->is_rsc_dev) {
+ mhi_netdev_free_pool(mhi_netdev);
+ return;
+ }
+
+ netif_stop_queue(mhi_netdev->ndev);
+ napi_disable(mhi_netdev->napi);
+ unregister_netdev(mhi_netdev->ndev);
+ netif_napi_del(mhi_netdev->napi);
+ free_netdev(mhi_netdev->ndev);
+ mhi_netdev_free_pool(mhi_netdev);
+
+#ifdef CONFIG_MHI_MBIM
+ /* tasklet_kill(&ctx->bh); */
+ cdc_ncm_mhi_free(ctx);
+#endif
+
+ if (!IS_ERR_OR_NULL(mhi_netdev->dentry))
+ debugfs_remove_recursive(mhi_netdev->dentry);
+}
+
+#ifdef CONFIG_OF
+static int mhi_netdev_match(struct device *dev, void *data)
+{
+ /* if phandle dt == device dt, we found a match */
+ return (dev->of_node == data);
+}
+
+static void mhi_netdev_clone_dev(struct mhi_netdev *mhi_netdev,
+ struct mhi_netdev *parent)
+{
+ mhi_netdev->ndev = parent->ndev;
+ mhi_netdev->napi = parent->napi;
+ mhi_netdev->ipc_log = parent->ipc_log;
+ mhi_netdev->msg_lvl = parent->msg_lvl;
+ mhi_netdev->ipc_log_lvl = parent->ipc_log_lvl;
+ mhi_netdev->is_rsc_dev = true;
+ mhi_netdev->chain = parent->chain;
+}
+#endif
+
+static int mhi_netdev_probe(struct mhi_device *mhi_dev,
+ const struct mhi_device_id *id)
+{
+ int ret;
+ struct mhi_netdev *mhi_netdev, *p_netdev = NULL;
+ int nr_tre;
+ char node_name[32];
+ bool no_chain;
+#ifdef CONFIG_MHI_MBIM
+ struct cdc_ncm_ctx *ctx;
+#endif
+#ifdef CONFIG_OF
+ struct device_node *of_node = mhi_dev->dev.of_node;
+ struct device_node *phandle;
+
+ if (!of_node)
+ return -ENODEV;
+#else
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(mhi_netdev_init_array); i++) {
+ if (!strcmp(mhi_netdev_init_array[i].chan, id->chan))
+ break;
+ }
+
+ if (i == ARRAY_SIZE(mhi_netdev_init_array))
+ return -ENODEV;
+#endif
+
+ mhi_netdev = devm_kzalloc(&mhi_dev->dev, sizeof(*mhi_netdev),
+ GFP_KERNEL);
+ if (!mhi_netdev)
+ return -ENOMEM;
+
+ mhi_netdev->mhi_dev = mhi_dev;
+ mhi_device_set_devdata(mhi_dev, mhi_netdev);
+
+#ifdef CONFIG_OF
+ ret = of_property_read_u32(of_node, "mhi,mru", &mhi_netdev->mru);
+ if (ret)
+ return -ENODEV;
+
+ /* MRU must be multiplication of page size */
+ mhi_netdev->order = __ilog2_u32(mhi_netdev->mru / PAGE_SIZE);
+ if ((PAGE_SIZE << mhi_netdev->order) < mhi_netdev->mru)
+ return -EINVAL;
+
+ /* check if this device shared by a parent device */
+ phandle = of_parse_phandle(of_node, "mhi,rsc-parent", 0);
+ if (phandle) {
+ struct device *dev;
+ struct mhi_device *pdev;
+ /* find the parent device */
+ dev = driver_find_device(mhi_dev->dev.driver, NULL, phandle,
+ mhi_netdev_match);
+ if (!dev)
+ return -ENODEV;
+
+ /* this device is shared with parent device. so we won't be
+ * creating a new network interface. Clone parent
+ * information to child node
+ */
+ pdev = to_mhi_device(dev);
+ p_netdev = mhi_device_get_devdata(pdev);
+ mhi_netdev_clone_dev(mhi_netdev, p_netdev);
+ put_device(dev);
+ } else {
+ mhi_netdev->msg_lvl = MHI_MSG_LVL_ERROR;
+ no_chain = of_property_read_bool(of_node,
+ "mhi,disable-chain-skb");
+ if (!no_chain) {
+ mhi_netdev->chain = devm_kzalloc(&mhi_dev->dev,
+ sizeof(*mhi_netdev->chain),
+ GFP_KERNEL);
+ if (!mhi_netdev->chain)
+ return -ENOMEM;
+ }
+
+ ret = mhi_netdev_enable_iface(mhi_netdev);
+ if (ret)
+ return ret;
+
+ /* create ipc log buffer */
+ snprintf(node_name, sizeof(node_name),
+ "%s_%04x_%02u.%02u.%02u_%u",
+ mhi_netdev->interface_name, mhi_dev->dev_id,
+ mhi_dev->domain, mhi_dev->bus, mhi_dev->slot,
+ mhi_netdev->alias);
+ mhi_netdev->ipc_log = ipc_log_context_create(IPC_LOG_PAGES,
+ node_name, 0);
+ mhi_netdev->ipc_log_lvl = IPC_LOG_LVL;
+
+ mhi_netdev_create_debugfs(mhi_netdev);
+ }
+#else
+ mhi_netdev->mru = mhi_netdev_init_array[i].mru;
+
+ /* MRU must be multiplication of page size */
+ mhi_netdev->order = __ilog2_u32(mhi_netdev->mru / PAGE_SIZE);
+ if ((PAGE_SIZE << mhi_netdev->order) < mhi_netdev->mru)
+ return -EINVAL;
+
+ mhi_netdev->msg_lvl = MHI_MSG_LVL_ERROR;
+ no_chain = mhi_netdev_init_array[i].no_chain;
+ if (!no_chain) {
+ mhi_netdev->chain = devm_kzalloc(&mhi_dev->dev,
+ sizeof(*mhi_netdev->chain),
+ GFP_KERNEL);
+ if (!mhi_netdev->chain)
+ return -ENOMEM;
+ }
+
+ ret = mhi_netdev_enable_iface(mhi_netdev, i);
+ if (ret)
+ return ret;
+
+ /* create ipc log buffer */
+ snprintf(node_name, sizeof(node_name),
+ "%s_%04x_%02u.%02u.%02u_%u",
+ mhi_netdev->interface_name, mhi_dev->dev_id,
+ mhi_dev->domain, mhi_dev->bus, mhi_dev->slot,
+ mhi_netdev->alias);
+ mhi_netdev->ipc_log = ipc_log_context_create(IPC_LOG_PAGES,
+ node_name, 0);
+ mhi_netdev->ipc_log_lvl = IPC_LOG_LVL;
+
+ mhi_netdev_create_debugfs(mhi_netdev);
+#endif
+
+#ifdef CONFIG_MHI_MBIM
+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return -ENOMEM;
+
+ ctx->bh.data = (unsigned long)mhi_netdev;
+ ctx->bh.func = cdc_ncm_mhi_txpath_bh;
+
+ spin_lock_init(&ctx->mtx);
+
+ /* store ctx pointer in device data field */
+ mhi_netdev->data[0] = (unsigned long)ctx;
+
+ ctx->rx_max = 31744;
+ ctx->tx_max = 16384;
+ ctx->tx_remainder = 0;
+ ctx->tx_modulus = 4;
+ ctx->tx_ndp_modulus = 4;
+ /* devices prior to NCM Errata shall set this field to zero */
+ ctx->tx_max_datagrams = 16;
+
+ /* set up maximum NDP size */
+ ctx->max_ndp_size = sizeof(struct usb_cdc_ncm_ndp16) +
+ (ctx->tx_max_datagrams + 1) * sizeof(struct usb_cdc_ncm_dpe16);
+#endif
+
+ /* move mhi channels to start state */
+ ret = mhi_prepare_for_transfer(mhi_dev);
+ if (ret) {
+ MSG_ERR("Failed to start channels ret %d\n", ret);
+ goto error_start;
+ }
+
+ /* setup pool size ~2x ring length */
+ nr_tre = mhi_get_no_free_descriptors(mhi_dev, DMA_FROM_DEVICE);
+ mhi_netdev->pool_size = 1 << __ilog2_u32(nr_tre);
+ if (nr_tre > mhi_netdev->pool_size)
+ mhi_netdev->pool_size <<= 1;
+ mhi_netdev->pool_size <<= 1;
+
+ /* allocate memory pool */
+ ret = mhi_netdev_alloc_pool(mhi_netdev);
+ if (ret)
+ goto error_start;
+
+ /* link child node with parent node if it's children dev */
+ if (p_netdev)
+ p_netdev->rsc_dev = mhi_netdev;
+
+ /* now we have a pool of buffers allocated, queue to hardware
+ * by triggering a napi_poll
+ */
+ napi_schedule(mhi_netdev->napi);
+
+ return 0;
+
+error_start:
+#ifdef CONFIG_OF
+ if (phandle)
+ return ret;
+#endif
+
+ netif_stop_queue(mhi_netdev->ndev);
+ napi_disable(mhi_netdev->napi);
+ unregister_netdev(mhi_netdev->ndev);
+ netif_napi_del(mhi_netdev->napi);
+ free_netdev(mhi_netdev->ndev);
+#ifdef CONFIG_MHI_MBIM
+ cdc_ncm_mhi_free(ctx);
+#endif
+
+ return ret;
+}
+
+static const struct mhi_device_id mhi_netdev_match_table[] = {
+ { .chan = "IP_HW0" },
+ { .chan = "IP_HW_ADPL" },
+ { .chan = "IP_HW0_RSC" },
+ { .chan = "IP_SW0" },
+ { },
+};
+
+static struct mhi_driver mhi_netdev_driver = {
+ .id_table = mhi_netdev_match_table,
+ .probe = mhi_netdev_probe,
+ .remove = mhi_netdev_remove,
+ .ul_xfer_cb = mhi_netdev_xfer_ul_cb,
+ .dl_xfer_cb = mhi_netdev_xfer_dl_cb,
+ .status_cb = mhi_netdev_status_cb,
+ .driver = {
+ .name = "mhi_netdev",
+ .owner = THIS_MODULE,
+ }
+};
+
+static int __init mhi_netdev_init(void)
+{
+ if (get_mhi_pci_status())
+ return -ENODEV;
+
+ BUILD_BUG_ON(sizeof(struct mhi_netbuf) > MAX_NETBUF_SIZE);
+ mhi_netdev_create_debugfs_dir();
+
+ return mhi_driver_register(&mhi_netdev_driver);
+}
+
+module_init(mhi_netdev_init);
+
+static void __exit mhi_netdev_exit(void)
+{
+ mhi_driver_unregister(&mhi_netdev_driver);
+ debugfs_remove(dentry);
+}
+
+module_exit(mhi_netdev_exit);
+
+MODULE_AUTHOR("Qualcomm Corporation");
+MODULE_DESCRIPTION("Qualcomm Modem Host Interface Bus Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/bus/mhi/devices/mhi_satellite.c b/drivers/bus/mhi/devices/mhi_satellite.c
new file mode 100644
index 000000000000..a42608c0708a
--- /dev/null
+++ b/drivers/bus/mhi/devices/mhi_satellite.c
@@ -0,0 +1,1155 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2019, The Linux Foundation. All rights reserved.*/
+
+#include <linux/debugfs.h>
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/dma-mapping.h>
+#include <linux/errno.h>
+#include <linux/ipc_logging.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/rpmsg.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/wait.h>
+#include <linux/mhi.h>
+
+#define MHI_SAT_DRIVER_NAME "mhi_satellite"
+
+static bool mhi_sat_defer_init = true; /* set by default */
+
+/* logging macros */
+#define IPC_LOG_PAGES (10)
+#define IPC_LOG_LVL (MHI_MSG_LVL_INFO)
+#define KLOG_LVL (MHI_MSG_LVL_ERROR)
+
+#define MHI_SUBSYS_LOG(fmt, ...) do { \
+ if (!subsys) \
+ break; \
+ if (mhi_sat_driver.klog_lvl <= MHI_MSG_LVL_INFO) \
+ pr_info("[I][%s][%s] " fmt, __func__, subsys->name, \
+ ##__VA_ARGS__);\
+ if (subsys->ipc_log && mhi_sat_driver.ipc_log_lvl <= \
+ MHI_MSG_LVL_INFO) \
+ ipc_log_string(subsys->ipc_log, "[I][%s] " fmt, __func__, \
+ ##__VA_ARGS__); \
+} while (0)
+
+#define MHI_SAT_LOG(fmt, ...) do { \
+ if (!subsys || !sat_cntrl) \
+ break; \
+ if (mhi_sat_driver.klog_lvl <= MHI_MSG_LVL_INFO) \
+ pr_info("[I][%s][%s][%x] " fmt, __func__, subsys->name, \
+ sat_cntrl->dev_id, ##__VA_ARGS__);\
+ if (subsys->ipc_log && mhi_sat_driver.ipc_log_lvl <= \
+ MHI_MSG_LVL_INFO) \
+ ipc_log_string(subsys->ipc_log, "[I][%s][%x] " fmt, __func__, \
+ sat_cntrl->dev_id, ##__VA_ARGS__); \
+} while (0)
+
+#define MHI_SAT_ERR(fmt, ...) do { \
+ if (!subsys || !sat_cntrl) \
+ break; \
+ if (mhi_sat_driver.klog_lvl <= MHI_MSG_LVL_ERROR) \
+ pr_err("[E][%s][%s][%x] " fmt, __func__, subsys->name, \
+ sat_cntrl->dev_id, ##__VA_ARGS__); \
+ if (subsys->ipc_log && mhi_sat_driver.ipc_log_lvl <= \
+ MHI_MSG_LVL_ERROR) \
+ ipc_log_string(subsys->ipc_log, "[E][%s][%x] " fmt, __func__, \
+ sat_cntrl->dev_id, ##__VA_ARGS__); \
+} while (0)
+
+#define MHI_SAT_ASSERT(cond, msg) do { \
+ if (cond) \
+ panic(msg); \
+} while (0)
+
+/* mhi sys error command */
+#define MHI_TRE_CMD_SYS_ERR_PTR (0)
+#define MHI_TRE_CMD_SYS_ERR_D0 (0)
+#define MHI_TRE_CMD_SYS_ERR_D1 (MHI_PKT_TYPE_SYS_ERR_CMD << 16)
+
+/* mhi state change event */
+#define MHI_TRE_EVT_MHI_STATE_PTR (0)
+#define MHI_TRE_EVT_MHI_STATE_D0(state) (state << 24)
+#define MHI_TRE_EVT_MHI_STATE_D1 (MHI_PKT_TYPE_STATE_CHANGE_EVENT << 16)
+
+/* mhi exec env change event */
+#define MHI_TRE_EVT_EE_PTR (0)
+#define MHI_TRE_EVT_EE_D0(ee) (ee << 24)
+#define MHI_TRE_EVT_EE_D1 (MHI_PKT_TYPE_EE_EVENT << 16)
+
+/* mhi config event */
+#define MHI_TRE_EVT_CFG_PTR(base_addr) (base_addr)
+#define MHI_TRE_EVT_CFG_D0(er_base, num) ((er_base << 16) | (num & 0xFFFF))
+#define MHI_TRE_EVT_CFG_D1 (MHI_PKT_TYPE_CFG_EVENT << 16)
+
+/* command completion event */
+#define MHI_TRE_EVT_CMD_COMPLETION_PTR(ptr) (ptr)
+#define MHI_TRE_EVT_CMD_COMPLETION_D0(code) (code << 24)
+#define MHI_TRE_EVT_CMD_COMPLETION_D1 (MHI_PKT_TYPE_CMD_COMPLETION_EVENT << 16)
+
+/* packet parser macros */
+#define MHI_TRE_GET_PTR(tre) ((tre)->ptr)
+#define MHI_TRE_GET_SIZE(tre) ((tre)->dword[0])
+#define MHI_TRE_GET_CCS(tre) (((tre)->dword[0] >> 24) & 0xFF)
+#define MHI_TRE_GET_ID(tre) (((tre)->dword[1] >> 24) & 0xFF)
+#define MHI_TRE_GET_TYPE(tre) (((tre)->dword[1] >> 16) & 0xFF)
+#define MHI_TRE_IS_ER_CTXT_TYPE(tre) (((tre)->dword[1]) & 0x1)
+
+/* creates unique device ID based on connection topology */
+#define MHI_SAT_CREATE_DEVICE_ID(dev, domain, bus, slot) \
+ ((dev & 0xFFFF) << 16 | (domain & 0xF) << 12 | (bus & 0xFF) << 4 | \
+ (slot & 0xF))
+
+/* mhi core definitions */
+#define MHI_CTXT_TYPE_GENERIC (0xA)
+
+struct __packed mhi_generic_ctxt {
+ u32 reserved0;
+ u32 type;
+ u32 reserved1;
+ u64 ctxt_base;
+ u64 ctxt_size;
+ u64 reserved[2];
+};
+
+enum mhi_pkt_type {
+ MHI_PKT_TYPE_INVALID = 0x0,
+ MHI_PKT_TYPE_RESET_CHAN_CMD = 0x10,
+ MHI_PKT_TYPE_STOP_CHAN_CMD = 0x11,
+ MHI_PKT_TYPE_START_CHAN_CMD = 0x12,
+ MHI_PKT_TYPE_STATE_CHANGE_EVENT = 0x20,
+ MHI_PKT_TYPE_CMD_COMPLETION_EVENT = 0x21,
+ MHI_PKT_TYPE_EE_EVENT = 0x40,
+ MHI_PKT_TYPE_CTXT_UPDATE_CMD = 0x64,
+ MHI_PKT_TYPE_IOMMU_MAP_CMD = 0x65,
+ MHI_PKT_TYPE_CFG_EVENT = 0x6E,
+ MHI_PKT_TYPE_SYS_ERR_CMD = 0xFF,
+};
+
+enum mhi_cmd_type {
+ MHI_CMD_TYPE_RESET = 0x10,
+ MHI_CMD_TYPE_STOP = 0x11,
+ MHI_CMD_TYPE_START = 0x12,
+};
+
+/* mhi event completion codes */
+enum mhi_ev_ccs {
+ MHI_EV_CC_INVALID = 0x0,
+ MHI_EV_CC_SUCCESS = 0x1,
+ MHI_EV_CC_BAD_TRE = 0x11,
+};
+
+/* satellite subsystem definitions */
+enum subsys_id {
+ SUBSYS_ADSP,
+ SUBSYS_CDSP,
+ SUBSYS_SLPI,
+ SUBSYS_MODEM,
+ SUBSYS_MAX,
+};
+
+static const char *const subsys_names[SUBSYS_MAX] = {
+ [SUBSYS_ADSP] = "adsp",
+ [SUBSYS_CDSP] = "cdsp",
+ [SUBSYS_SLPI] = "slpi",
+ [SUBSYS_MODEM] = "modem",
+};
+
+struct mhi_sat_subsys {
+ const char *name;
+
+ struct rpmsg_device *rpdev; /* rpmsg device */
+
+ /*
+ * acquire either mutex or spinlock to walk controller list
+ * acquire both when modifying list
+ */
+ struct list_head cntrl_list; /* controllers list */
+ struct mutex cntrl_mutex; /* mutex for controllers list */
+ spinlock_t cntrl_lock; /* lock for controllers list */
+
+ void *ipc_log;
+};
+
+/* satellite IPC definitions */
+#define SAT_MAJOR_VERSION (1)
+#define SAT_MINOR_VERSION (0)
+#define SAT_RESERVED_SEQ_NUM (0xFFFF)
+#define SAT_MSG_SIZE(n) (sizeof(struct sat_header) + \
+ (n * sizeof(struct sat_tre)))
+#define SAT_TRE_SIZE(msg_size) (msg_size - sizeof(struct sat_header))
+#define SAT_TRE_OFFSET(msg) (msg + sizeof(struct sat_header))
+#define SAT_TRE_NUM_PKTS(payload_size) ((payload_size) / sizeof(struct sat_tre))
+
+/* satellite IPC msg type */
+enum sat_msg_id {
+ SAT_MSG_ID_ACK = 0xA,
+ SAT_MSG_ID_CMD = 0xC,
+ SAT_MSG_ID_EVT = 0xE,
+};
+
+/* satellite IPC context type */
+enum sat_ctxt_type {
+ SAT_CTXT_TYPE_CHAN = 0x0,
+ SAT_CTXT_TYPE_EVENT = 0x1,
+ SAT_CTXT_TYPE_MAX,
+};
+
+/* satellite IPC context string */
+#define TO_SAT_CTXT_TYPE_STR(type) (type >= SAT_CTXT_TYPE_MAX ? "INVALID" : \
+ sat_ctxt_str[type])
+
+const char *const sat_ctxt_str[SAT_CTXT_TYPE_MAX] = {
+ [SAT_CTXT_TYPE_CHAN] = "CCA",
+ [SAT_CTXT_TYPE_EVENT] = "ECA",
+};
+
+/* satellite IPC transfer ring element */
+struct __packed sat_tre {
+ u64 ptr;
+ u32 dword[2];
+};
+
+/* satellite IPC header */
+struct __packed sat_header {
+ u16 major_ver;
+ u16 minor_ver;
+ u16 msg_id;
+ u16 seq;
+ u16 reply_seq;
+ u16 payload_size;
+ u32 dev_id;
+ u8 reserved[8];
+};
+
+/* satellite driver definitions */
+struct mhi_sat_packet {
+ struct list_head node;
+
+ struct mhi_sat_cntrl *cntrl; /* satellite controller reference */
+ void *msg; /* incoming message */
+};
+
+struct mhi_sat_cntrl {
+ struct list_head node;
+
+ struct mhi_controller *mhi_cntrl; /* device MHI controller */
+ struct mhi_sat_subsys *subsys;
+
+ struct list_head dev_list;
+ struct list_head addr_map_list; /* IOMMU mapped addresses list */
+ struct mutex list_mutex; /* mutex for devices and address map lists */
+
+ struct list_head packet_list;
+ spinlock_t pkt_lock; /* lock to walk/modify received packets list */
+
+ struct work_struct connect_work; /* subsystem connection worker */
+ struct work_struct process_work; /* incoming packets processor */
+
+ /* mhi core/controller configurations */
+ u32 dev_id; /* unique ID with BDF as per connection topology */
+ int er_base; /* event rings base index */
+ int er_max; /* event rings max index */
+ int num_er; /* total number of event rings */
+
+ /* satellite controller function counts */
+ int num_devices; /* mhi devices current count */
+ int max_devices; /* count of maximum devices for subsys/controller */
+ u16 seq; /* internal s/n number for all outgoing packets */
+ bool active; /* flag set if hello packet/MHI_CFG event was sent */
+
+ /* command completion variables */
+ u16 last_cmd_seq; /* s/n number of last sent command packet */
+ enum mhi_ev_ccs last_cmd_ccs; /* last command completion event */
+ struct completion completion; /* command completion event wait */
+ struct mutex cmd_wait_mutex; /* command completion wait mutex */
+};
+
+struct mhi_sat_device {
+ struct list_head node;
+
+ struct mhi_device *mhi_dev; /* mhi device pointer */
+ struct mhi_sat_cntrl *cntrl; /* parent controller */
+
+ bool chan_started;
+};
+
+struct mhi_sat_driver {
+ enum MHI_DEBUG_LEVEL ipc_log_lvl; /* IPC log level */
+ enum MHI_DEBUG_LEVEL klog_lvl; /* klog/dmesg levels */
+
+ struct mhi_sat_subsys *subsys; /* pointer to subsystem array */
+ unsigned int num_subsys;
+
+ struct dentry *dentry; /* debugfs directory */
+ bool deferred_init_done; /* flag for deferred init protection */
+};
+
+static struct mhi_sat_driver mhi_sat_driver;
+
+static struct mhi_sat_subsys *find_subsys_by_name(const char *name)
+{
+ int i;
+ struct mhi_sat_subsys *subsys = mhi_sat_driver.subsys;
+
+ for (i = 0; i < mhi_sat_driver.num_subsys; i++, subsys++) {
+ if (!strcmp(name, subsys->name))
+ return subsys;
+ }
+
+ return NULL;
+}
+
+static struct mhi_sat_cntrl *find_sat_cntrl_by_id(struct mhi_sat_subsys *subsys,
+ u32 dev_id)
+{
+ struct mhi_sat_cntrl *sat_cntrl;
+ unsigned long flags;
+
+ spin_lock_irqsave(&subsys->cntrl_lock, flags);
+ list_for_each_entry(sat_cntrl, &subsys->cntrl_list, node) {
+ if (sat_cntrl->dev_id == dev_id) {
+ spin_unlock_irqrestore(&subsys->cntrl_lock, flags);
+ return sat_cntrl;
+ }
+ }
+ spin_unlock_irqrestore(&subsys->cntrl_lock, flags);
+
+ return NULL;
+}
+
+static struct mhi_sat_device *find_sat_dev_by_id(
+ struct mhi_sat_cntrl *sat_cntrl, int id,
+ enum sat_ctxt_type evt)
+{
+ struct mhi_sat_device *sat_dev;
+ int compare_id;
+
+ mutex_lock(&sat_cntrl->list_mutex);
+ list_for_each_entry(sat_dev, &sat_cntrl->dev_list, node) {
+ compare_id = (evt == SAT_CTXT_TYPE_EVENT) ?
+ sat_dev->mhi_dev->dl_event_id :
+ sat_dev->mhi_dev->dl_chan_id;
+
+ if (compare_id == id) {
+ mutex_unlock(&sat_cntrl->list_mutex);
+ return sat_dev;
+ }
+ }
+ mutex_unlock(&sat_cntrl->list_mutex);
+
+ return NULL;
+}
+
+static bool mhi_sat_isvalid_header(struct sat_header *hdr, int len)
+{
+ /* validate payload size */
+ if (len >= sizeof(*hdr) && (len != hdr->payload_size + sizeof(*hdr)))
+ return false;
+
+ /* validate SAT IPC version */
+ if (hdr->major_ver != SAT_MAJOR_VERSION &&
+ hdr->minor_ver != SAT_MINOR_VERSION)
+ return false;
+
+ /* validate msg ID */
+ if (hdr->msg_id != SAT_MSG_ID_CMD && hdr->msg_id != SAT_MSG_ID_EVT)
+ return false;
+
+ return true;
+}
+
+static int mhi_sat_wait_cmd_completion(struct mhi_sat_cntrl *sat_cntrl)
+{
+ struct mhi_sat_subsys *subsys = sat_cntrl->subsys;
+ int ret;
+
+ reinit_completion(&sat_cntrl->completion);
+
+ MHI_SAT_LOG("Wait for command completion\n");
+ ret = wait_for_completion_timeout(&sat_cntrl->completion,
+ msecs_to_jiffies(sat_cntrl->mhi_cntrl->timeout_ms));
+ if (!ret || sat_cntrl->last_cmd_ccs != MHI_EV_CC_SUCCESS) {
+ MHI_SAT_ERR("Command completion failure:seq:%u:ret:%d:ccs:%d\n",
+ sat_cntrl->last_cmd_seq, ret,
+ sat_cntrl->last_cmd_ccs);
+ return -EIO;
+ }
+
+ MHI_SAT_LOG("Command completion successful for seq:%u\n",
+ sat_cntrl->last_cmd_seq);
+
+ return 0;
+}
+
+static int mhi_sat_send_msg(struct mhi_sat_cntrl *sat_cntrl,
+ enum sat_msg_id type, u16 reply_seq,
+ void *msg, u16 msg_size)
+{
+ struct mhi_sat_subsys *subsys = sat_cntrl->subsys;
+ struct sat_header *hdr = msg;
+
+ /* create sequence number for controller */
+ sat_cntrl->seq++;
+ if (sat_cntrl->seq == SAT_RESERVED_SEQ_NUM)
+ sat_cntrl->seq = 0;
+
+ /* populate header */
+ hdr->major_ver = SAT_MAJOR_VERSION;
+ hdr->minor_ver = SAT_MINOR_VERSION;
+ hdr->msg_id = type;
+ hdr->seq = sat_cntrl->seq;
+ hdr->reply_seq = reply_seq;
+ hdr->payload_size = SAT_TRE_SIZE(msg_size);
+ hdr->dev_id = sat_cntrl->dev_id;
+
+ /* save last sent command sequence number for completion event */
+ if (type == SAT_MSG_ID_CMD)
+ sat_cntrl->last_cmd_seq = sat_cntrl->seq;
+
+ return rpmsg_send(subsys->rpdev->ept, msg, msg_size);
+}
+
+static void mhi_sat_process_cmds(struct mhi_sat_cntrl *sat_cntrl,
+ struct sat_header *hdr, struct sat_tre *pkt)
+{
+ struct mhi_sat_subsys *subsys = sat_cntrl->subsys;
+ int num_pkts = SAT_TRE_NUM_PKTS(hdr->payload_size), i;
+
+ for (i = 0; i < num_pkts; i++, pkt++) {
+ enum mhi_ev_ccs code = MHI_EV_CC_INVALID;
+
+ switch (MHI_TRE_GET_TYPE(pkt)) {
+ case MHI_PKT_TYPE_IOMMU_MAP_CMD:
+ {
+ struct mhi_buf *buf;
+ struct mhi_controller *mhi_cntrl = sat_cntrl->mhi_cntrl;
+ dma_addr_t iova = DMA_ERROR_CODE;
+
+ buf = kmalloc(sizeof(*buf), GFP_ATOMIC);
+ if (!buf)
+ goto iommu_map_cmd_completion;
+
+ buf->phys_addr = MHI_TRE_GET_PTR(pkt);
+ buf->len = MHI_TRE_GET_SIZE(pkt);
+
+ iova = dma_map_resource(mhi_cntrl->dev,
+ buf->phys_addr, buf->len,
+ DMA_BIDIRECTIONAL, 0);
+ if (dma_mapping_error(mhi_cntrl->dev, iova)) {
+ kfree(buf);
+ goto iommu_map_cmd_completion;
+ }
+
+ buf->dma_addr = iova;
+
+ mutex_lock(&sat_cntrl->list_mutex);
+ list_add_tail(&buf->node, &sat_cntrl->addr_map_list);
+ mutex_unlock(&sat_cntrl->list_mutex);
+
+ code = MHI_EV_CC_SUCCESS;
+
+iommu_map_cmd_completion:
+ MHI_SAT_LOG("IOMMU MAP 0x%llx CMD processing %s\n",
+ MHI_TRE_GET_PTR(pkt),
+ (code == MHI_EV_CC_SUCCESS) ?
+ "successful" : "failed");
+
+ pkt->ptr = MHI_TRE_EVT_CMD_COMPLETION_PTR(iova);
+ pkt->dword[0] = MHI_TRE_EVT_CMD_COMPLETION_D0(code);
+ pkt->dword[1] = MHI_TRE_EVT_CMD_COMPLETION_D1;
+ break;
+ }
+ case MHI_PKT_TYPE_CTXT_UPDATE_CMD:
+ {
+ u64 ctxt_ptr = MHI_TRE_GET_PTR(pkt);
+ u64 ctxt_size = MHI_TRE_GET_SIZE(pkt);
+ int id = MHI_TRE_GET_ID(pkt);
+ enum sat_ctxt_type evt = MHI_TRE_IS_ER_CTXT_TYPE(pkt);
+ struct mhi_generic_ctxt gen_ctxt;
+ struct mhi_buf buf;
+ struct mhi_sat_device *sat_dev =
+ find_sat_dev_by_id(sat_cntrl, id, evt);
+ int ret;
+
+ MHI_SAT_ASSERT(!sat_dev,
+ "No device with given chan/evt ID");
+
+ memset(&gen_ctxt, 0, sizeof(gen_ctxt));
+ memset(&buf, 0, sizeof(buf));
+
+ gen_ctxt.type = MHI_CTXT_TYPE_GENERIC;
+ gen_ctxt.ctxt_base = ctxt_ptr;
+ gen_ctxt.ctxt_size = ctxt_size;
+
+ buf.buf = &gen_ctxt;
+ buf.len = sizeof(gen_ctxt);
+ buf.name = TO_SAT_CTXT_TYPE_STR(evt);
+
+ ret = mhi_device_configure(sat_dev->mhi_dev,
+ DMA_BIDIRECTIONAL, &buf, 1);
+ if (!ret)
+ code = MHI_EV_CC_SUCCESS;
+
+ MHI_SAT_LOG("CTXT UPDATE CMD %s:%d processing %s\n",
+ buf.name, id,
+ (code == MHI_EV_CC_SUCCESS)
+ ? "successful" : "failed");
+
+ pkt->ptr = MHI_TRE_EVT_CMD_COMPLETION_PTR(0);
+ pkt->dword[0] = MHI_TRE_EVT_CMD_COMPLETION_D0(code);
+ pkt->dword[1] = MHI_TRE_EVT_CMD_COMPLETION_D1;
+ break;
+ }
+ case MHI_PKT_TYPE_START_CHAN_CMD:
+ {
+ int id = MHI_TRE_GET_ID(pkt);
+ struct mhi_sat_device *sat_dev =
+ find_sat_dev_by_id(sat_cntrl, id,
+ SAT_CTXT_TYPE_CHAN);
+ int ret;
+
+ MHI_SAT_ASSERT(!sat_dev,
+ "No device with given channel ID\n");
+
+ MHI_SAT_ASSERT(sat_dev->chan_started,
+ "Channel already started!");
+
+ ret = mhi_prepare_for_transfer(sat_dev->mhi_dev);
+ if (!ret) {
+ sat_dev->chan_started = true;
+ code = MHI_EV_CC_SUCCESS;
+ }
+
+ MHI_SAT_LOG("START CHANNEL %d CMD processing %s\n", id,
+ (code == MHI_EV_CC_SUCCESS)
+ ? "successful" : "failure");
+
+ pkt->ptr = MHI_TRE_EVT_CMD_COMPLETION_PTR(0);
+ pkt->dword[0] = MHI_TRE_EVT_CMD_COMPLETION_D0(code);
+ pkt->dword[1] = MHI_TRE_EVT_CMD_COMPLETION_D1;
+ break;
+ }
+ case MHI_PKT_TYPE_RESET_CHAN_CMD:
+ {
+ int id = MHI_TRE_GET_ID(pkt);
+ struct mhi_sat_device *sat_dev =
+ find_sat_dev_by_id(sat_cntrl, id,
+ SAT_CTXT_TYPE_CHAN);
+
+ MHI_SAT_ASSERT(!sat_dev,
+ "No device with given channel ID\n");
+
+ MHI_SAT_ASSERT(!sat_dev->chan_started,
+ "Resetting unstarted channel!");
+
+ mhi_unprepare_from_transfer(sat_dev->mhi_dev);
+ sat_dev->chan_started = false;
+
+ MHI_SAT_LOG
+ ("RESET CHANNEL %d CMD processing successful\n",
+ id);
+
+ pkt->ptr = MHI_TRE_EVT_CMD_COMPLETION_PTR(0);
+ pkt->dword[0] =
+ MHI_TRE_EVT_CMD_COMPLETION_D0(MHI_EV_CC_SUCCESS);
+ pkt->dword[1] = MHI_TRE_EVT_CMD_COMPLETION_D1;
+ break;
+ }
+ default:
+ MHI_SAT_ASSERT(1, "Unhandled command!");
+ break;
+ }
+ }
+}
+
+static void mhi_sat_process_worker(struct work_struct *work)
+{
+ struct mhi_sat_cntrl *sat_cntrl =
+ container_of(work, struct mhi_sat_cntrl, process_work);
+ struct mhi_sat_subsys *subsys = sat_cntrl->subsys;
+ struct mhi_sat_packet *packet, *tmp;
+ struct sat_header *hdr;
+ struct sat_tre *pkt;
+ LIST_HEAD(head);
+
+ MHI_SAT_LOG("Entered\n");
+
+ spin_lock_irq(&sat_cntrl->pkt_lock);
+ list_splice_tail_init(&sat_cntrl->packet_list, &head);
+ spin_unlock_irq(&sat_cntrl->pkt_lock);
+
+ list_for_each_entry_safe(packet, tmp, &head, node) {
+ hdr = packet->msg;
+ pkt = SAT_TRE_OFFSET(packet->msg);
+
+ list_del(&packet->node);
+
+ mhi_sat_process_cmds(sat_cntrl, hdr, pkt);
+
+ /* send response event(s) */
+ mhi_sat_send_msg(sat_cntrl, SAT_MSG_ID_EVT, hdr->seq,
+ packet->msg,
+ SAT_MSG_SIZE(SAT_TRE_NUM_PKTS
+ (hdr->payload_size)));
+
+ kfree(packet);
+ }
+
+ MHI_SAT_LOG("Exited\n");
+}
+
+static void mhi_sat_connect_worker(struct work_struct *work)
+{
+ struct mhi_sat_cntrl *sat_cntrl = container_of(work,
+ struct mhi_sat_cntrl,
+ connect_work);
+ struct mhi_sat_subsys *subsys = sat_cntrl->subsys;
+ struct sat_tre *pkt;
+ void *msg;
+ int ret;
+
+ if (!subsys->rpdev || sat_cntrl->max_devices != sat_cntrl->num_devices
+ || sat_cntrl->active)
+ return;
+
+ MHI_SAT_LOG("Entered\n");
+
+ msg = kmalloc(SAT_MSG_SIZE(3), GFP_ATOMIC);
+ if (!msg)
+ return;
+
+ sat_cntrl->active = true;
+
+ pkt = SAT_TRE_OFFSET(msg);
+
+ /* prepare #1 MHI_CFG HELLO event */
+ pkt->ptr = MHI_TRE_EVT_CFG_PTR(sat_cntrl->mhi_cntrl->base_addr);
+ pkt->dword[0] = MHI_TRE_EVT_CFG_D0(sat_cntrl->er_base,
+ sat_cntrl->num_er);
+ pkt->dword[1] = MHI_TRE_EVT_CFG_D1;
+ pkt++;
+
+ /* prepare M0 event */
+ pkt->ptr = MHI_TRE_EVT_MHI_STATE_PTR;
+ pkt->dword[0] = MHI_TRE_EVT_MHI_STATE_D0(MHI_STATE_M0);
+ pkt->dword[1] = MHI_TRE_EVT_MHI_STATE_D1;
+ pkt++;
+
+ /* prepare AMSS event */
+ pkt->ptr = MHI_TRE_EVT_EE_PTR;
+ pkt->dword[0] = MHI_TRE_EVT_EE_D0(MHI_EE_AMSS);
+ pkt->dword[1] = MHI_TRE_EVT_EE_D1;
+
+ ret = mhi_sat_send_msg(sat_cntrl, SAT_MSG_ID_EVT, SAT_RESERVED_SEQ_NUM,
+ msg, SAT_MSG_SIZE(3));
+ kfree(msg);
+ if (ret) {
+ MHI_SAT_ERR("Failed to send hello packet:%d\n", ret);
+ sat_cntrl->active = false;
+ return;
+ }
+
+ MHI_SAT_LOG("Device 0x%x sent hello packet\n", sat_cntrl->dev_id);
+}
+
+static void mhi_sat_process_events(struct mhi_sat_cntrl *sat_cntrl,
+ struct sat_header *hdr, struct sat_tre *pkt)
+{
+ int num_pkts = SAT_TRE_NUM_PKTS(hdr->payload_size);
+ int i;
+
+ for (i = 0; i < num_pkts; i++, pkt++) {
+ if (MHI_TRE_GET_TYPE(pkt) ==
+ MHI_PKT_TYPE_CMD_COMPLETION_EVENT) {
+ if (hdr->reply_seq != sat_cntrl->last_cmd_seq)
+ continue;
+
+ sat_cntrl->last_cmd_ccs = MHI_TRE_GET_CCS(pkt);
+ complete(&sat_cntrl->completion);
+ }
+ }
+}
+
+static int mhi_sat_rpmsg_cb(struct rpmsg_device *rpdev, void *data, int len,
+ void *priv, u32 src)
+{
+ struct mhi_sat_subsys *subsys = dev_get_drvdata(&rpdev->dev);
+ struct sat_header *hdr = data;
+ struct sat_tre *pkt = SAT_TRE_OFFSET(data);
+ struct mhi_sat_cntrl *sat_cntrl;
+ struct mhi_sat_packet *packet;
+
+ MHI_SAT_ASSERT(!mhi_sat_isvalid_header(hdr, len), "Invalid header!\n");
+
+ /* find controller packet was sent for */
+ sat_cntrl = find_sat_cntrl_by_id(subsys, hdr->dev_id);
+
+ MHI_SAT_ASSERT(!sat_cntrl, "Packet for unknown device!\n");
+
+ /* handle events directly regardless of controller active state */
+ if (hdr->msg_id == SAT_MSG_ID_EVT) {
+ mhi_sat_process_events(sat_cntrl, hdr, pkt);
+ return 0;
+ }
+
+ /* Inactive controller cannot process incoming commands */
+ if (unlikely(!sat_cntrl->active)) {
+ MHI_SAT_ERR("Message for inactive controller!\n");
+ return 0;
+ }
+
+ /* offload commands to process worker */
+ packet = kmalloc(sizeof(*packet) + len, GFP_ATOMIC);
+ if (!packet)
+ return 0;
+
+ packet->cntrl = sat_cntrl;
+ packet->msg = packet + 1;
+ memcpy(packet->msg, data, len);
+
+ spin_lock_irq(&sat_cntrl->pkt_lock);
+ list_add_tail(&packet->node, &sat_cntrl->packet_list);
+ spin_unlock_irq(&sat_cntrl->pkt_lock);
+
+ schedule_work(&sat_cntrl->process_work);
+
+ return 0;
+}
+
+static void mhi_sat_rpmsg_remove(struct rpmsg_device *rpdev)
+{
+ struct mhi_sat_subsys *subsys = dev_get_drvdata(&rpdev->dev);
+ struct mhi_sat_cntrl *sat_cntrl;
+ struct mhi_sat_device *sat_dev;
+ struct mhi_buf *buf, *tmp;
+
+ MHI_SUBSYS_LOG("Enter\n");
+
+ /* unprepare each controller/device from transfer */
+ mutex_lock(&subsys->cntrl_mutex);
+ list_for_each_entry(sat_cntrl, &subsys->cntrl_list, node) {
+ sat_cntrl->active = false;
+
+ flush_work(&sat_cntrl->connect_work);
+ flush_work(&sat_cntrl->process_work);
+
+ mutex_lock(&sat_cntrl->list_mutex);
+ list_for_each_entry(sat_dev, &sat_cntrl->dev_list, node) {
+ if (sat_dev->chan_started) {
+ mhi_unprepare_from_transfer(sat_dev->mhi_dev);
+ sat_dev->chan_started = false;
+ }
+ }
+
+ list_for_each_entry_safe(buf, tmp, &sat_cntrl->addr_map_list,
+ node) {
+ dma_unmap_resource(sat_cntrl->mhi_cntrl->dev,
+ buf->dma_addr, buf->len,
+ DMA_BIDIRECTIONAL, 0);
+ list_del(&buf->node);
+ kfree(buf);
+ }
+ mutex_unlock(&sat_cntrl->list_mutex);
+
+ MHI_SAT_LOG("Removed RPMSG link\n");
+ }
+ mutex_unlock(&subsys->cntrl_mutex);
+
+ subsys->rpdev = NULL;
+}
+
+static int mhi_sat_rpmsg_probe(struct rpmsg_device *rpdev)
+{
+ struct mhi_sat_subsys *subsys;
+ struct mhi_sat_cntrl *sat_cntrl;
+ const char *subsys_name;
+ int ret;
+
+ ret = of_property_read_string(rpdev->dev.parent->of_node, "label",
+ &subsys_name);
+ if (ret)
+ return ret;
+
+ /* find which subsystem has probed */
+ subsys = find_subsys_by_name(subsys_name);
+ if (!subsys)
+ return -EINVAL;
+
+ MHI_SUBSYS_LOG("Received RPMSG probe\n");
+
+ dev_set_drvdata(&rpdev->dev, subsys);
+
+ subsys->rpdev = rpdev;
+
+ /* schedule work for each controller as GLINK has connected */
+ spin_lock_irq(&subsys->cntrl_lock);
+ list_for_each_entry(sat_cntrl, &subsys->cntrl_list, node)
+ schedule_work(&sat_cntrl->connect_work);
+ spin_unlock_irq(&subsys->cntrl_lock);
+
+ return 0;
+}
+
+static struct rpmsg_device_id mhi_sat_rpmsg_match_table[] = {
+ { .name = "mhi_sat" },
+ { },
+};
+
+static struct rpmsg_driver mhi_sat_rpmsg_driver = {
+ .id_table = mhi_sat_rpmsg_match_table,
+ .probe = mhi_sat_rpmsg_probe,
+ .remove = mhi_sat_rpmsg_remove,
+ .callback = mhi_sat_rpmsg_cb,
+ .drv = {
+ .name = "mhi,sat_rpmsg",
+ },
+};
+
+static void mhi_sat_dev_status_cb(struct mhi_device *mhi_dev,
+ enum MHI_CB mhi_cb)
+{
+}
+
+static void mhi_sat_dev_remove(struct mhi_device *mhi_dev)
+{
+ struct mhi_sat_device *sat_dev = mhi_device_get_devdata(mhi_dev);
+ struct mhi_sat_cntrl *sat_cntrl = sat_dev->cntrl;
+ struct mhi_sat_subsys *subsys = sat_cntrl->subsys;
+ struct mhi_buf *buf, *tmp;
+ struct sat_tre *pkt;
+ void *msg;
+ int ret;
+
+ /* remove device node from probed list */
+ mutex_lock(&sat_cntrl->list_mutex);
+ list_del(&sat_dev->node);
+ mutex_unlock(&sat_cntrl->list_mutex);
+
+ sat_cntrl->num_devices--;
+
+ /* prepare SYS_ERR command if first device is being removed */
+ if (sat_cntrl->active) {
+ sat_cntrl->active = false;
+
+ /* flush all pending work */
+ flush_work(&sat_cntrl->connect_work);
+ flush_work(&sat_cntrl->process_work);
+
+ msg = kmalloc(SAT_MSG_SIZE(1), GFP_KERNEL);
+
+ MHI_SAT_ASSERT(!msg, "Unable to malloc for SYS_ERR message!\n");
+
+ pkt = SAT_TRE_OFFSET(msg);
+ pkt->ptr = MHI_TRE_CMD_SYS_ERR_PTR;
+ pkt->dword[0] = MHI_TRE_CMD_SYS_ERR_D0;
+ pkt->dword[1] = MHI_TRE_CMD_SYS_ERR_D1;
+
+ /* acquire cmd_wait_mutex before sending command */
+ mutex_lock(&sat_cntrl->cmd_wait_mutex);
+
+ ret = mhi_sat_send_msg(sat_cntrl, SAT_MSG_ID_CMD,
+ SAT_RESERVED_SEQ_NUM, msg,
+ SAT_MSG_SIZE(1));
+ kfree(msg);
+ if (ret) {
+ MHI_SAT_ERR("Failed to notify SYS_ERR\n");
+ mutex_unlock(&sat_cntrl->cmd_wait_mutex);
+ goto exit_sys_err_send;
+ }
+
+ MHI_SAT_LOG("SYS_ERR command sent\n");
+
+ /* blocking call to wait for command completion event */
+ mhi_sat_wait_cmd_completion(sat_cntrl);
+
+ mutex_unlock(&sat_cntrl->cmd_wait_mutex);
+ }
+
+exit_sys_err_send:
+ /* exit if some devices are still present */
+ if (sat_cntrl->num_devices)
+ return;
+
+ /* remove address mappings */
+ mutex_lock(&sat_cntrl->list_mutex);
+ list_for_each_entry_safe(buf, tmp, &sat_cntrl->addr_map_list, node) {
+ dma_unmap_resource(sat_cntrl->mhi_cntrl->dev, buf->dma_addr,
+ buf->len, DMA_BIDIRECTIONAL, 0);
+ list_del(&buf->node);
+ kfree(buf);
+ }
+ mutex_unlock(&sat_cntrl->list_mutex);
+
+ /* remove controller */
+ mutex_lock(&subsys->cntrl_mutex);
+ spin_lock_irq(&subsys->cntrl_lock);
+ list_del(&sat_cntrl->node);
+ spin_unlock_irq(&subsys->cntrl_lock);
+ mutex_unlock(&subsys->cntrl_mutex);
+
+ mutex_destroy(&sat_cntrl->cmd_wait_mutex);
+ mutex_destroy(&sat_cntrl->list_mutex);
+ MHI_SAT_LOG("Satellite controller node removed\n");
+ kfree(sat_cntrl);
+}
+
+static int mhi_sat_dev_probe(struct mhi_device *mhi_dev,
+ const struct mhi_device_id *id)
+{
+ struct mhi_sat_device *sat_dev;
+ struct mhi_sat_cntrl *sat_cntrl;
+ struct device_node *of_node = mhi_dev->dev.of_node;
+ struct mhi_sat_subsys *subsys = &mhi_sat_driver.subsys[id->driver_data];
+ u32 dev_id = MHI_SAT_CREATE_DEVICE_ID(mhi_dev->dev_id, mhi_dev->domain,
+ mhi_dev->bus, mhi_dev->slot);
+ int ret;
+
+ /* find controller with unique device ID based on topology */
+ sat_cntrl = find_sat_cntrl_by_id(subsys, dev_id);
+ if (!sat_cntrl) {
+ sat_cntrl = kzalloc(sizeof(*sat_cntrl), GFP_KERNEL);
+ if (!sat_cntrl)
+ return -ENOMEM;
+
+ /*
+ * max_devices will be read from device tree node. Set it to
+ * -1 before it is populated to avoid false positive when
+ * RPMSG probe schedules connect worker but no device has
+ * probed in which case num_devices and max_devices are both
+ * zero.
+ */
+ sat_cntrl->max_devices = -1;
+ sat_cntrl->dev_id = dev_id;
+ sat_cntrl->er_base = mhi_dev->dl_event_id;
+ sat_cntrl->mhi_cntrl = mhi_dev->mhi_cntrl;
+ sat_cntrl->last_cmd_seq = SAT_RESERVED_SEQ_NUM;
+ sat_cntrl->subsys = subsys;
+ init_completion(&sat_cntrl->completion);
+ mutex_init(&sat_cntrl->list_mutex);
+ mutex_init(&sat_cntrl->cmd_wait_mutex);
+ spin_lock_init(&sat_cntrl->pkt_lock);
+ INIT_WORK(&sat_cntrl->connect_work, mhi_sat_connect_worker);
+ INIT_WORK(&sat_cntrl->process_work, mhi_sat_process_worker);
+ INIT_LIST_HEAD(&sat_cntrl->dev_list);
+ INIT_LIST_HEAD(&sat_cntrl->addr_map_list);
+ INIT_LIST_HEAD(&sat_cntrl->packet_list);
+
+ mutex_lock(&subsys->cntrl_mutex);
+ spin_lock_irq(&subsys->cntrl_lock);
+ list_add(&sat_cntrl->node, &subsys->cntrl_list);
+ spin_unlock_irq(&subsys->cntrl_lock);
+ mutex_unlock(&subsys->cntrl_mutex);
+
+ MHI_SAT_LOG("Controller allocated for 0x%x\n", dev_id);
+ }
+
+ /* set maximum devices for subsystem from device tree */
+ if (of_node) {
+ ret = of_property_read_u32(of_node, "mhi,max-devices",
+ &sat_cntrl->max_devices);
+ if (ret) {
+ MHI_SAT_ERR("Could not find max-devices in DT node\n");
+ return -EINVAL;
+ }
+ }
+
+ /* get event ring base and max indexes */
+ sat_cntrl->er_base = min(sat_cntrl->er_base, mhi_dev->dl_event_id);
+ sat_cntrl->er_max = max(sat_cntrl->er_base, mhi_dev->dl_event_id);
+
+ sat_dev = devm_kzalloc(&mhi_dev->dev, sizeof(*sat_dev), GFP_KERNEL);
+ if (!sat_dev)
+ return -ENOMEM;
+
+ sat_dev->mhi_dev = mhi_dev;
+ sat_dev->cntrl = sat_cntrl;
+
+ mutex_lock(&sat_cntrl->list_mutex);
+ list_add(&sat_dev->node, &sat_cntrl->dev_list);
+ mutex_unlock(&sat_cntrl->list_mutex);
+
+ mhi_device_set_devdata(mhi_dev, sat_dev);
+
+ sat_cntrl->num_devices++;
+
+ /* schedule connect worker if all devices for controller have probed */
+ if (sat_cntrl->num_devices == sat_cntrl->max_devices) {
+ /* number of event rings is 1 more than difference in IDs */
+ sat_cntrl->num_er =
+ (sat_cntrl->er_max - sat_cntrl->er_base) + 1;
+ MHI_SAT_LOG("All satellite channels probed!\n");
+ schedule_work(&sat_cntrl->connect_work);
+ }
+
+ return 0;
+}
+
+/* .driver_data stores subsys id */
+static const struct mhi_device_id mhi_sat_dev_match_table[] = {
+ /* ADSP */
+ { .chan = "ADSP_0", .driver_data = SUBSYS_ADSP },
+ { .chan = "ADSP_1", .driver_data = SUBSYS_ADSP },
+ { .chan = "ADSP_2", .driver_data = SUBSYS_ADSP },
+ { .chan = "ADSP_3", .driver_data = SUBSYS_ADSP },
+ { .chan = "ADSP_4", .driver_data = SUBSYS_ADSP },
+ { .chan = "ADSP_5", .driver_data = SUBSYS_ADSP },
+ { .chan = "ADSP_6", .driver_data = SUBSYS_ADSP },
+ { .chan = "ADSP_7", .driver_data = SUBSYS_ADSP },
+ { .chan = "ADSP_8", .driver_data = SUBSYS_ADSP },
+ { .chan = "ADSP_9", .driver_data = SUBSYS_ADSP },
+ /* CDSP */
+ { .chan = "CDSP_0", .driver_data = SUBSYS_CDSP },
+ { .chan = "CDSP_1", .driver_data = SUBSYS_CDSP },
+ { .chan = "CDSP_2", .driver_data = SUBSYS_CDSP },
+ { .chan = "CDSP_3", .driver_data = SUBSYS_CDSP },
+ { .chan = "CDSP_4", .driver_data = SUBSYS_CDSP },
+ { .chan = "CDSP_5", .driver_data = SUBSYS_CDSP },
+ { .chan = "CDSP_6", .driver_data = SUBSYS_CDSP },
+ { .chan = "CDSP_7", .driver_data = SUBSYS_CDSP },
+ { .chan = "CDSP_8", .driver_data = SUBSYS_CDSP },
+ { .chan = "CDSP_9", .driver_data = SUBSYS_CDSP },
+ /* SLPI */
+ { .chan = "SLPI_0", .driver_data = SUBSYS_SLPI },
+ { .chan = "SLPI_1", .driver_data = SUBSYS_SLPI },
+ { .chan = "SLPI_2", .driver_data = SUBSYS_SLPI },
+ { .chan = "SLPI_3", .driver_data = SUBSYS_SLPI },
+ { .chan = "SLPI_4", .driver_data = SUBSYS_SLPI },
+ { .chan = "SLPI_5", .driver_data = SUBSYS_SLPI },
+ { .chan = "SLPI_6", .driver_data = SUBSYS_SLPI },
+ { .chan = "SLPI_7", .driver_data = SUBSYS_SLPI },
+ { .chan = "SLPI_8", .driver_data = SUBSYS_SLPI },
+ { .chan = "SLPI_9", .driver_data = SUBSYS_SLPI },
+ /* MODEM */
+ { .chan = "MODEM_0", .driver_data = SUBSYS_MODEM },
+ { .chan = "MODEM_1", .driver_data = SUBSYS_MODEM },
+ { .chan = "MODEM_2", .driver_data = SUBSYS_MODEM },
+ { .chan = "MODEM_3", .driver_data = SUBSYS_MODEM },
+ { .chan = "MODEM_4", .driver_data = SUBSYS_MODEM },
+ { .chan = "MODEM_5", .driver_data = SUBSYS_MODEM },
+ { .chan = "MODEM_6", .driver_data = SUBSYS_MODEM },
+ { .chan = "MODEM_7", .driver_data = SUBSYS_MODEM },
+ { .chan = "MODEM_8", .driver_data = SUBSYS_MODEM },
+ { .chan = "MODEM_9", .driver_data = SUBSYS_MODEM },
+ { },
+};
+
+static struct mhi_driver mhi_sat_dev_driver = {
+ .id_table = mhi_sat_dev_match_table,
+ .probe = mhi_sat_dev_probe,
+ .remove = mhi_sat_dev_remove,
+ .status_cb = mhi_sat_dev_status_cb,
+ .driver = {
+ .name = MHI_SAT_DRIVER_NAME,
+ .owner = THIS_MODULE,
+ },
+};
+
+int mhi_sat_trigger_init(void *data, u64 val)
+{
+ struct mhi_sat_subsys *subsys;
+ int i, ret;
+
+ if (mhi_sat_driver.deferred_init_done)
+ return -EIO;
+
+ ret = register_rpmsg_driver(&mhi_sat_rpmsg_driver);
+ if (ret)
+ goto error_sat_trigger_init;
+
+ ret = mhi_driver_register(&mhi_sat_dev_driver);
+ if (ret)
+ goto error_sat_trigger_register;
+
+ mhi_sat_driver.deferred_init_done = true;
+
+ return 0;
+
+error_sat_trigger_register:
+ unregister_rpmsg_driver(&mhi_sat_rpmsg_driver);
+
+error_sat_trigger_init:
+ subsys = mhi_sat_driver.subsys;
+ for (i = 0; i < mhi_sat_driver.num_subsys; i++, subsys++) {
+ ipc_log_context_destroy(subsys->ipc_log);
+ mutex_destroy(&subsys->cntrl_mutex);
+ }
+ kfree(mhi_sat_driver.subsys);
+ mhi_sat_driver.subsys = NULL;
+
+ return ret;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(mhi_sat_debugfs_fops, NULL,
+ mhi_sat_trigger_init, "%llu\n");
+
+static int mhi_sat_init(void)
+{
+ struct mhi_sat_subsys *subsys;
+ int i, ret;
+
+ subsys = kcalloc(SUBSYS_MAX, sizeof(*subsys), GFP_KERNEL);
+ if (!subsys)
+ return -ENOMEM;
+
+ mhi_sat_driver.subsys = subsys;
+ mhi_sat_driver.num_subsys = SUBSYS_MAX;
+ mhi_sat_driver.klog_lvl = KLOG_LVL;
+ mhi_sat_driver.ipc_log_lvl = IPC_LOG_LVL;
+
+ for (i = 0; i < mhi_sat_driver.num_subsys; i++, subsys++) {
+ char log[32];
+
+ subsys->name = subsys_names[i];
+ mutex_init(&subsys->cntrl_mutex);
+ spin_lock_init(&subsys->cntrl_lock);
+ INIT_LIST_HEAD(&subsys->cntrl_list);
+ scnprintf(log, sizeof(log), "mhi_sat_%s", subsys->name);
+ subsys->ipc_log = ipc_log_context_create(IPC_LOG_PAGES, log, 0);
+ }
+
+ /* create debugfs entry if defer_init is enabled */
+ if (mhi_sat_defer_init) {
+ mhi_sat_driver.dentry = debugfs_create_dir("mhi_sat", NULL);
+ if (IS_ERR_OR_NULL(mhi_sat_driver.dentry)) {
+ ret = -ENODEV;
+ goto error_sat_init;
+ }
+
+ debugfs_create_file("debug", 0444, mhi_sat_driver.dentry, NULL,
+ &mhi_sat_debugfs_fops);
+
+ return 0;
+ }
+
+ ret = register_rpmsg_driver(&mhi_sat_rpmsg_driver);
+ if (ret)
+ goto error_sat_init;
+
+ ret = mhi_driver_register(&mhi_sat_dev_driver);
+ if (ret)
+ goto error_sat_register;
+
+ return 0;
+
+error_sat_register:
+ unregister_rpmsg_driver(&mhi_sat_rpmsg_driver);
+
+error_sat_init:
+ subsys = mhi_sat_driver.subsys;
+ for (i = 0; i < mhi_sat_driver.num_subsys; i++, subsys++) {
+ ipc_log_context_destroy(subsys->ipc_log);
+ mutex_destroy(&subsys->cntrl_mutex);
+ }
+ kfree(mhi_sat_driver.subsys);
+ mhi_sat_driver.subsys = NULL;
+
+ return ret;
+}
+
+module_init(mhi_sat_init);
diff --git a/drivers/bus/mhi/devices/mhi_uci.c b/drivers/bus/mhi/devices/mhi_uci.c
new file mode 100644
index 000000000000..66bd96839837
--- /dev/null
+++ b/drivers/bus/mhi/devices/mhi_uci.c
@@ -0,0 +1,802 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.*/
+
+#include <linux/cdev.h>
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/errno.h>
+#include <linux/fs.h>
+#include <linux/ipc_logging.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/wait.h>
+#include <linux/uaccess.h>
+#include <linux/mhi.h>
+#include <linux/kthread.h>
+#include <linux/delay.h>
+#include <linux/fs.h>
+#include <asm/segment.h>
+#include <linux/buffer_head.h>
+
+#define DEVICE_NAME "mhi"
+#define MHI_UCI_DRIVER_NAME "mhi_uci"
+
+struct uci_chan {
+ wait_queue_head_t wq;
+ spinlock_t lock;
+ struct list_head pending; /* user space waiting to read */
+ struct uci_buf *cur_buf; /* current buffer user space reading */
+ size_t rx_size;
+};
+
+struct uci_buf {
+ void *data;
+ size_t len;
+ struct list_head node;
+};
+
+struct uci_dev {
+ struct list_head node;
+ dev_t devt;
+ struct device *dev;
+ struct mhi_device *mhi_dev;
+ const char *chan;
+ struct mutex mutex; /* sync open and close */
+ struct uci_chan ul_chan;
+ struct uci_chan dl_chan;
+ size_t mtu;
+ int ref_count;
+ bool enabled;
+ void *ipc_log;
+};
+
+struct mhi_uci_drv {
+ struct list_head head;
+ struct mutex lock;
+ struct class *class;
+ int major;
+ dev_t dev_t;
+};
+
+static struct task_struct *tsk[110];
+
+enum MHI_DEBUG_LEVEL msg_lvl = MHI_MSG_LVL_ERROR;
+
+#ifdef CONFIG_MHI_DEBUG
+
+#define IPC_LOG_LVL (MHI_MSG_LVL_VERBOSE)
+#define MHI_UCI_IPC_LOG_PAGES (25)
+
+#else
+
+#define IPC_LOG_LVL (MHI_MSG_LVL_ERROR)
+#define MHI_UCI_IPC_LOG_PAGES (1)
+
+#endif
+
+#ifdef CONFIG_MHI_DEBUG
+
+#define MSG_VERB(fmt, ...) do { \
+ if (msg_lvl <= MHI_MSG_LVL_VERBOSE) \
+ pr_err("[D][%s] " fmt, __func__, ##__VA_ARGS__); \
+ if (uci_dev->ipc_log && (IPC_LOG_LVL <= MHI_MSG_LVL_VERBOSE)) \
+ ipc_log_string(uci_dev->ipc_log, "[D][%s] " fmt, \
+ __func__, ##__VA_ARGS__); \
+ } while (0)
+
+#else
+
+#define MSG_VERB(fmt, ...)
+
+#endif
+
+#define MSG_LOG(fmt, ...) do { \
+ if (msg_lvl <= MHI_MSG_LVL_INFO) \
+ pr_err("[I][%s] " fmt, __func__, ##__VA_ARGS__); \
+ if (uci_dev->ipc_log && (IPC_LOG_LVL <= MHI_MSG_LVL_INFO)) \
+ ipc_log_string(uci_dev->ipc_log, "[I][%s] " fmt, \
+ __func__, ##__VA_ARGS__); \
+ } while (0)
+
+#define MSG_ERR(fmt, ...) do { \
+ if (msg_lvl <= MHI_MSG_LVL_ERROR) \
+ pr_err("[E][%s] " fmt, __func__, ##__VA_ARGS__); \
+ if (uci_dev->ipc_log && (IPC_LOG_LVL <= MHI_MSG_LVL_ERROR)) \
+ ipc_log_string(uci_dev->ipc_log, "[E][%s] " fmt, \
+ __func__, ##__VA_ARGS__); \
+ } while (0)
+
+#define MAX_UCI_DEVICES (64)
+
+static DECLARE_BITMAP(uci_minors, MAX_UCI_DEVICES);
+static struct mhi_uci_drv mhi_uci_drv;
+
+struct file *file_open(const char *path, int flags, int rights)
+{
+ struct file *filp = NULL;
+ mm_segment_t oldfs;
+ int err = 0;
+
+ oldfs = get_fs();
+ set_fs(KERNEL_DS);
+ filp = filp_open(path, flags, rights);
+ set_fs(oldfs);
+ if (IS_ERR(filp)) {
+ err = PTR_ERR(filp);
+ return NULL;
+ }
+ return filp;
+}
+
+void file_close(struct file *file)
+{
+ filp_close(file, NULL);
+}
+
+static int mhi_queue_inbound(struct uci_dev *uci_dev)
+{
+ struct mhi_device *mhi_dev = uci_dev->mhi_dev;
+ int nr_trbs = mhi_get_no_free_descriptors(mhi_dev, DMA_FROM_DEVICE);
+ size_t mtu = uci_dev->mtu;
+ void *buf;
+ struct uci_buf *uci_buf;
+ int ret = -EIO, i;
+
+ for (i = 0; i < nr_trbs; i++) {
+ buf = kmalloc(mtu + sizeof(*uci_buf), GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ uci_buf = buf + mtu;
+ uci_buf->data = buf;
+
+ MSG_VERB("Allocated buf %d of %d size %zu\n", i, nr_trbs, mtu);
+
+ ret = mhi_queue_transfer(mhi_dev, DMA_FROM_DEVICE, buf, mtu,
+ MHI_EOT);
+ if (ret) {
+ kfree(buf);
+ MSG_ERR("Failed to queue buffer %d\n", i);
+ return ret;
+ }
+ }
+
+ return ret;
+}
+
+static long mhi_uci_ioctl(struct file *file,
+ unsigned int cmd, unsigned long arg)
+{
+ struct uci_dev *uci_dev = file->private_data;
+ struct mhi_device *mhi_dev = uci_dev->mhi_dev;
+ long ret = -ERESTARTSYS;
+
+ mutex_lock(&uci_dev->mutex);
+ if (uci_dev->enabled)
+ ret = mhi_ioctl(mhi_dev, cmd, arg);
+ mutex_unlock(&uci_dev->mutex);
+
+ return ret;
+}
+
+static int mhi_uci_release(struct inode *inode, struct file *file)
+{
+ struct uci_dev *uci_dev = file->private_data;
+
+ mutex_lock(&uci_dev->mutex);
+ uci_dev->ref_count--;
+ if (!uci_dev->ref_count) {
+ struct uci_buf *itr, *tmp;
+ struct uci_chan *uci_chan;
+
+ MSG_LOG("Last client left, closing node\n");
+
+ if (uci_dev->enabled)
+ mhi_unprepare_from_transfer(uci_dev->mhi_dev);
+
+ /* clean inbound channel */
+ uci_chan = &uci_dev->dl_chan;
+ list_for_each_entry_safe(itr, tmp, &uci_chan->pending, node) {
+ list_del(&itr->node);
+ kfree(itr->data);
+ }
+ if (uci_chan->cur_buf)
+ kfree(uci_chan->cur_buf->data);
+
+ uci_chan->cur_buf = NULL;
+
+ if (!uci_dev->enabled) {
+ MSG_LOG("Node is deleted, freeing dev node\n");
+ mutex_unlock(&uci_dev->mutex);
+ mutex_destroy(&uci_dev->mutex);
+ clear_bit(MINOR(uci_dev->devt), uci_minors);
+ kfree(uci_dev);
+ return 0;
+ }
+ }
+
+ MSG_LOG("exit: ref_count:%d\n", uci_dev->ref_count);
+
+ mutex_unlock(&uci_dev->mutex);
+
+ return 0;
+}
+
+static unsigned int mhi_uci_poll(struct file *file, poll_table *wait)
+{
+ struct uci_dev *uci_dev = file->private_data;
+ struct mhi_device *mhi_dev = uci_dev->mhi_dev;
+ struct uci_chan *uci_chan;
+ unsigned int mask = 0;
+
+ poll_wait(file, &uci_dev->dl_chan.wq, wait);
+ poll_wait(file, &uci_dev->ul_chan.wq, wait);
+
+ uci_chan = &uci_dev->dl_chan;
+ spin_lock_bh(&uci_chan->lock);
+ if (!uci_dev->enabled) {
+ mask = POLLERR;
+ } else if (!list_empty(&uci_chan->pending) || uci_chan->cur_buf) {
+ MSG_VERB("Client can read from node\n");
+ mask |= POLLIN | POLLRDNORM;
+ }
+ spin_unlock_bh(&uci_chan->lock);
+
+ uci_chan = &uci_dev->ul_chan;
+ spin_lock_bh(&uci_chan->lock);
+ if (!uci_dev->enabled) {
+ mask |= POLLERR;
+ } else if (mhi_get_no_free_descriptors(mhi_dev, DMA_TO_DEVICE) > 0) {
+ MSG_VERB("Client can write to node\n");
+ mask |= POLLOUT | POLLWRNORM;
+ }
+ spin_unlock_bh(&uci_chan->lock);
+
+ MSG_LOG("Client attempted to poll, returning mask 0x%x\n", mask);
+
+ return mask;
+}
+
+static ssize_t mhi_uci_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *offp)
+{
+ struct uci_dev *uci_dev = file->private_data;
+ struct mhi_device *mhi_dev = uci_dev->mhi_dev;
+ struct uci_chan *uci_chan = &uci_dev->ul_chan;
+ size_t bytes_xfered = 0;
+ int ret, nr_avail;
+
+ if (!buf || !count)
+ return -EINVAL;
+
+ /* confirm channel is active */
+ spin_lock_bh(&uci_chan->lock);
+ if (!uci_dev->enabled) {
+ spin_unlock_bh(&uci_chan->lock);
+ return -ERESTARTSYS;
+ }
+
+ MSG_VERB("Enter: to xfer:%zu bytes\n", count);
+
+ while (count) {
+ size_t xfer_size;
+ void *kbuf;
+ enum MHI_FLAGS flags;
+
+ spin_unlock_bh(&uci_chan->lock);
+
+ /* wait for free descriptors */
+ ret = wait_event_interruptible(uci_chan->wq,
+ (!uci_dev->enabled) ||
+ (nr_avail = mhi_get_no_free_descriptors(mhi_dev,
+ DMA_TO_DEVICE)) > 0);
+
+ if (ret == -ERESTARTSYS || !uci_dev->enabled) {
+ MSG_LOG("Exit signal caught for node or not enabled\n");
+ return -ERESTARTSYS;
+ }
+
+ xfer_size = min_t(size_t, count, uci_dev->mtu);
+ kbuf = kmalloc(xfer_size, GFP_KERNEL);
+ if (!kbuf) {
+ MSG_ERR("Failed to allocate memory %zu\n", xfer_size);
+ return -ENOMEM;
+ }
+
+ ret = copy_from_user(kbuf, buf, xfer_size);
+ if (unlikely(ret)) {
+ kfree(kbuf);
+ return ret;
+ }
+
+ spin_lock_bh(&uci_chan->lock);
+
+ /* if ring is full after this force EOT */
+ if (nr_avail > 1 && (count - xfer_size))
+ flags = MHI_CHAIN;
+ else
+ flags = MHI_EOT;
+
+ if (uci_dev->enabled)
+ ret = mhi_queue_transfer(mhi_dev, DMA_TO_DEVICE, kbuf,
+ xfer_size, flags);
+ else
+ ret = -ERESTARTSYS;
+
+ if (ret) {
+ kfree(kbuf);
+ goto sys_interrupt;
+ }
+
+ bytes_xfered += xfer_size;
+ count -= xfer_size;
+ buf += xfer_size;
+ }
+
+ spin_unlock_bh(&uci_chan->lock);
+ MSG_VERB("Exit: Number of bytes xferred:%zu\n", bytes_xfered);
+
+ return bytes_xfered;
+
+sys_interrupt:
+ spin_unlock_bh(&uci_chan->lock);
+
+ return ret;
+}
+
+static ssize_t mhi_uci_read(struct file *file,
+ char __user *buf, size_t count, loff_t *ppos)
+{
+ struct uci_dev *uci_dev = file->private_data;
+ struct mhi_device *mhi_dev = uci_dev->mhi_dev;
+ struct uci_chan *uci_chan = &uci_dev->dl_chan;
+ struct uci_buf *uci_buf;
+ char *ptr;
+ size_t to_copy;
+ int ret = 0;
+
+ if (!buf)
+ return -EINVAL;
+
+ MSG_VERB("Client provided buf len:%zu\n", count);
+
+ /* confirm channel is active */
+ spin_lock_bh(&uci_chan->lock);
+ if (!uci_dev->enabled) {
+ spin_unlock_bh(&uci_chan->lock);
+ return -ERESTARTSYS;
+ }
+
+ /* No data available to read, wait */
+ if (!uci_chan->cur_buf && list_empty(&uci_chan->pending)) {
+ MSG_VERB("No data available to read waiting\n");
+
+ spin_unlock_bh(&uci_chan->lock);
+ ret = wait_event_interruptible(uci_chan->wq,
+ (!uci_dev->enabled ||
+ !list_empty(&uci_chan->pending)));
+ if (ret == -ERESTARTSYS) {
+ MSG_LOG("Exit signal caught for node\n");
+ return -ERESTARTSYS;
+ }
+
+ spin_lock_bh(&uci_chan->lock);
+ if (!uci_dev->enabled) {
+ MSG_LOG("node is disabled\n");
+ ret = -ERESTARTSYS;
+ goto read_error;
+ }
+ }
+
+ /* new read, get the next descriptor from the list */
+ if (!uci_chan->cur_buf) {
+ uci_buf = list_first_entry_or_null(&uci_chan->pending,
+ struct uci_buf, node);
+ if (unlikely(!uci_buf)) {
+ ret = -EIO;
+ goto read_error;
+ }
+
+ list_del(&uci_buf->node);
+ uci_chan->cur_buf = uci_buf;
+ uci_chan->rx_size = uci_buf->len;
+ MSG_VERB("Got pkt of size:%zu\n", uci_chan->rx_size);
+ }
+
+ uci_buf = uci_chan->cur_buf;
+ spin_unlock_bh(&uci_chan->lock);
+
+ /* Copy the buffer to user space */
+ to_copy = min_t(size_t, count, uci_chan->rx_size);
+ ptr = uci_buf->data + (uci_buf->len - uci_chan->rx_size);
+ ret = copy_to_user(buf, ptr, to_copy);
+ if (ret)
+ return ret;
+
+ MSG_VERB("Copied %zu of %zu bytes\n", to_copy, uci_chan->rx_size);
+ uci_chan->rx_size -= to_copy;
+
+ /* we finished with this buffer, queue it back to hardware */
+ if (!uci_chan->rx_size) {
+ spin_lock_bh(&uci_chan->lock);
+ uci_chan->cur_buf = NULL;
+
+ if (uci_dev->enabled)
+ ret = mhi_queue_transfer(mhi_dev, DMA_FROM_DEVICE,
+ uci_buf->data, uci_dev->mtu,
+ MHI_EOT);
+ else
+ ret = -ERESTARTSYS;
+
+ if (ret) {
+ MSG_ERR("Failed to recycle element\n");
+ kfree(uci_buf->data);
+ goto read_error;
+ }
+
+ spin_unlock_bh(&uci_chan->lock);
+ }
+
+ MSG_VERB("Returning %zu bytes\n", to_copy);
+
+ return to_copy;
+
+read_error:
+ spin_unlock_bh(&uci_chan->lock);
+
+ return ret;
+}
+
+static int mhi_uci_open(struct inode *inode, struct file *filp)
+{
+ struct uci_dev *uci_dev = NULL, *tmp_dev;
+ int ret = -EIO;
+ struct uci_buf *buf_itr, *tmp;
+ struct uci_chan *dl_chan;
+
+ mutex_lock(&mhi_uci_drv.lock);
+ list_for_each_entry(tmp_dev, &mhi_uci_drv.head, node) {
+ if (tmp_dev->devt == inode->i_rdev) {
+ uci_dev = tmp_dev;
+ break;
+ }
+ }
+
+ /* could not find a minor node */
+ if (!uci_dev)
+ goto error_exit;
+
+ mutex_lock(&uci_dev->mutex);
+ if (!uci_dev->enabled) {
+ MSG_ERR("Node exist, but not in active state!\n");
+ goto error_open_chan;
+ }
+
+ uci_dev->ref_count++;
+
+ MSG_LOG("Node open, ref counts %u\n", uci_dev->ref_count);
+
+ if (uci_dev->ref_count == 1) {
+ MSG_LOG("Starting channel\n");
+ ret = mhi_prepare_for_transfer(uci_dev->mhi_dev);
+ if (ret) {
+ MSG_ERR("Error starting transfer channels\n");
+ uci_dev->ref_count--;
+ goto error_open_chan;
+ }
+
+ ret = mhi_queue_inbound(uci_dev);
+ if (ret)
+ goto error_rx_queue;
+ }
+
+ filp->private_data = uci_dev;
+ mutex_unlock(&uci_dev->mutex);
+ mutex_unlock(&mhi_uci_drv.lock);
+
+ return 0;
+
+error_rx_queue:
+ dl_chan = &uci_dev->dl_chan;
+ mhi_unprepare_from_transfer(uci_dev->mhi_dev);
+ list_for_each_entry_safe(buf_itr, tmp, &dl_chan->pending, node) {
+ list_del(&buf_itr->node);
+ kfree(buf_itr->data);
+ }
+
+error_open_chan:
+ mutex_unlock(&uci_dev->mutex);
+
+error_exit:
+ mutex_unlock(&mhi_uci_drv.lock);
+
+ return ret;
+}
+
+static const struct file_operations mhidev_fops = {
+ .open = mhi_uci_open,
+ .release = mhi_uci_release,
+ .read = mhi_uci_read,
+ .write = mhi_uci_write,
+ .poll = mhi_uci_poll,
+ .unlocked_ioctl = mhi_uci_ioctl,
+};
+
+static void mhi_uci_remove(struct mhi_device *mhi_dev)
+{
+ struct uci_dev *uci_dev = mhi_device_get_devdata(mhi_dev);
+
+ MSG_LOG("Enter\n");
+
+#ifdef CONFIG_MHI_MBIM
+ if (strcmp(uci_dev->mhi_dev->chan_name, "MBIM")
+ && (!IS_ERR(tsk[uci_dev->mhi_dev->ul_chan_id])))
+ kthread_stop(tsk[uci_dev->mhi_dev->ul_chan_id]);
+#else
+ if (!IS_ERR(tsk[uci_dev->mhi_dev->ul_chan_id]))
+ kthread_stop(tsk[uci_dev->mhi_dev->ul_chan_id]);
+#endif
+
+ mutex_lock(&mhi_uci_drv.lock);
+ mutex_lock(&uci_dev->mutex);
+
+ /* disable the node */
+ spin_lock_irq(&uci_dev->dl_chan.lock);
+ spin_lock_irq(&uci_dev->ul_chan.lock);
+ uci_dev->enabled = false;
+ spin_unlock_irq(&uci_dev->ul_chan.lock);
+ spin_unlock_irq(&uci_dev->dl_chan.lock);
+ wake_up(&uci_dev->dl_chan.wq);
+ wake_up(&uci_dev->ul_chan.wq);
+
+ /* delete the node to prevent new opens */
+ device_destroy(mhi_uci_drv.class, uci_dev->devt);
+ uci_dev->dev = NULL;
+ list_del(&uci_dev->node);
+
+ /* safe to free memory only if all file nodes are closed */
+ if (!uci_dev->ref_count) {
+ mutex_unlock(&uci_dev->mutex);
+ mutex_destroy(&uci_dev->mutex);
+ clear_bit(MINOR(uci_dev->devt), uci_minors);
+ kfree(uci_dev);
+ mutex_unlock(&mhi_uci_drv.lock);
+ return;
+ }
+
+ MSG_LOG("Exit\n");
+ mutex_unlock(&uci_dev->mutex);
+ mutex_unlock(&mhi_uci_drv.lock);
+
+}
+
+static int open_uci_thread(void *data)
+{
+ struct mhi_device *mhi_dev = (struct mhi_device *)data;
+ struct file *uci_filp;
+ char mhi_uci_dev_name[40];
+
+ if (!strcmp(mhi_dev->chan_name, "AT"))
+ snprintf(mhi_uci_dev_name, sizeof(mhi_uci_dev_name),
+ "/dev/at_pcie");
+ else
+ snprintf(mhi_uci_dev_name, sizeof(mhi_uci_dev_name),
+ "/dev/" DEVICE_NAME "_%04x_%02u.%02u.%02u%s%d",
+ mhi_dev->dev_id, mhi_dev->domain, mhi_dev->bus,
+ mhi_dev->slot, "_pipe_", mhi_dev->ul_chan_id);
+
+ do {
+ msleep(300);
+ uci_filp = file_open(mhi_uci_dev_name, O_RDONLY, 0);
+ } while (!uci_filp);
+
+ do {
+ msleep(1000);
+ } while (!kthread_should_stop());
+
+ file_close(uci_filp);
+
+ return 0;
+}
+
+static int mhi_uci_probe(struct mhi_device *mhi_dev,
+ const struct mhi_device_id *id)
+{
+ struct uci_dev *uci_dev;
+ int minor;
+ char node_name[32];
+ int dir;
+
+ uci_dev = kzalloc(sizeof(*uci_dev), GFP_KERNEL);
+ if (!uci_dev)
+ return -ENOMEM;
+
+ mutex_init(&uci_dev->mutex);
+ uci_dev->mhi_dev = mhi_dev;
+
+ minor = find_first_zero_bit(uci_minors, MAX_UCI_DEVICES);
+ if (minor >= MAX_UCI_DEVICES) {
+ kfree(uci_dev);
+ return -ENOSPC;
+ }
+
+ mutex_lock(&uci_dev->mutex);
+ mutex_lock(&mhi_uci_drv.lock);
+
+ uci_dev->devt = MKDEV(mhi_uci_drv.major, minor);
+
+ if (!strcmp(mhi_dev->chan_name, "AT"))
+ uci_dev->dev = device_create(mhi_uci_drv.class, &mhi_dev->dev,
+ uci_dev->devt, uci_dev, "at_pcie");
+#ifdef CONFIG_MHI_MBIM
+ else if (!strcmp(mhi_dev->chan_name, "MBIM"))
+ uci_dev->dev = device_create(mhi_uci_drv.class, &mhi_dev->dev,
+ uci_dev->devt, uci_dev,
+ "cdc-wdm_pcie");
+#endif
+ else
+ uci_dev->dev = device_create(mhi_uci_drv.class, &mhi_dev->dev,
+ uci_dev->devt, uci_dev,
+ DEVICE_NAME
+ "_%04x_%02u.%02u.%02u%s%d",
+ mhi_dev->dev_id, mhi_dev->domain,
+ mhi_dev->bus, mhi_dev->slot,
+ "_pipe_", mhi_dev->ul_chan_id);
+ set_bit(minor, uci_minors);
+
+ /* create debugging buffer */
+ snprintf(node_name, sizeof(node_name), "mhi_uci_%04x_%02u.%02u.%02u_%d",
+ mhi_dev->dev_id, mhi_dev->domain, mhi_dev->bus, mhi_dev->slot,
+ mhi_dev->ul_chan_id);
+ uci_dev->ipc_log = ipc_log_context_create(MHI_UCI_IPC_LOG_PAGES,
+ node_name, 0);
+
+ for (dir = 0; dir < 2; dir++) {
+ struct uci_chan *uci_chan =
+ (dir) ? &uci_dev->ul_chan : &uci_dev->dl_chan;
+ spin_lock_init(&uci_chan->lock);
+ init_waitqueue_head(&uci_chan->wq);
+ INIT_LIST_HEAD(&uci_chan->pending);
+ }
+
+ uci_dev->mtu = min_t(size_t, id->driver_data, mhi_dev->mtu);
+ mhi_device_set_devdata(mhi_dev, uci_dev);
+ uci_dev->enabled = true;
+
+ list_add(&uci_dev->node, &mhi_uci_drv.head);
+ mutex_unlock(&mhi_uci_drv.lock);
+ mutex_unlock(&uci_dev->mutex);
+
+ MSG_LOG("channel:%s successfully probed\n", mhi_dev->chan_name);
+
+#ifdef CONFIG_MHI_MBIM
+ /* No need to open MBIM device */
+ if (!strcmp(mhi_dev->chan_name, "MBIM"))
+ return 0;
+#endif
+
+ tsk[mhi_dev->ul_chan_id] =
+ kthread_run(open_uci_thread, mhi_dev, "MHI_UCI_%s",
+ mhi_dev->chan_name);
+ if (IS_ERR(tsk[mhi_dev->ul_chan_id]))
+ pr_err("create kthread failed!\n");
+
+ return 0;
+};
+
+static void mhi_ul_xfer_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result)
+{
+ struct uci_dev *uci_dev = mhi_device_get_devdata(mhi_dev);
+ struct uci_chan *uci_chan = &uci_dev->ul_chan;
+
+ MSG_VERB("status:%d xfer_len:%zu\n", mhi_result->transaction_status,
+ mhi_result->bytes_xferd);
+
+ kfree(mhi_result->buf_addr);
+ if (!mhi_result->transaction_status)
+ wake_up(&uci_chan->wq);
+}
+
+static void mhi_dl_xfer_cb(struct mhi_device *mhi_dev,
+ struct mhi_result *mhi_result)
+{
+ struct uci_dev *uci_dev = mhi_device_get_devdata(mhi_dev);
+ struct uci_chan *uci_chan = &uci_dev->dl_chan;
+ unsigned long flags;
+ struct uci_buf *buf;
+
+ MSG_VERB("status:%d receive_len:%zu\n", mhi_result->transaction_status,
+ mhi_result->bytes_xferd);
+
+ if (mhi_result->transaction_status == -ENOTCONN) {
+ kfree(mhi_result->buf_addr);
+ return;
+ }
+
+ spin_lock_irqsave(&uci_chan->lock, flags);
+ buf = mhi_result->buf_addr + uci_dev->mtu;
+ buf->data = mhi_result->buf_addr;
+ buf->len = mhi_result->bytes_xferd;
+ list_add_tail(&buf->node, &uci_chan->pending);
+ spin_unlock_irqrestore(&uci_chan->lock, flags);
+
+ if (mhi_dev->dev.power.wakeup)
+ __pm_wakeup_event(mhi_dev->dev.power.wakeup, 0);
+
+ wake_up(&uci_chan->wq);
+}
+
+/* .driver_data stores max mtu */
+static const struct mhi_device_id mhi_uci_match_table[] = {
+ { .chan = "LOOPBACK", .driver_data = 0x1000 },
+ { .chan = "SAHARA", .driver_data = 0x8000 },
+ { .chan = "EFS", .driver_data = 0x1000 },
+ { .chan = "QMI0", .driver_data = 0x1000 },
+ { .chan = "QMI1", .driver_data = 0x1000 },
+ { .chan = "TF", .driver_data = 0x1000 },
+ { .chan = "DUN", .driver_data = 0x1000 },
+ { .chan = "AT", .driver_data = 0x1000 },
+#ifdef CONFIG_MHI_MBIM
+ { .chan = "MBIM", .driver_data = 0x1000 },
+#endif
+ { },
+};
+
+static struct mhi_driver mhi_uci_driver = {
+ .id_table = mhi_uci_match_table,
+ .remove = mhi_uci_remove,
+ .probe = mhi_uci_probe,
+ .ul_xfer_cb = mhi_ul_xfer_cb,
+ .dl_xfer_cb = mhi_dl_xfer_cb,
+ .driver = {
+ .name = MHI_UCI_DRIVER_NAME,
+ .owner = THIS_MODULE,
+ },
+};
+
+static int mhi_uci_init(void)
+{
+ int ret;
+
+ ret = get_mhi_pci_status();
+ if (ret < 0)
+ return ret;
+
+ ret = register_chrdev(0, MHI_UCI_DRIVER_NAME, &mhidev_fops);
+ if (ret < 0)
+ return ret;
+
+ mhi_uci_drv.major = ret;
+ mhi_uci_drv.class = class_create(THIS_MODULE, MHI_UCI_DRIVER_NAME);
+ if (IS_ERR(mhi_uci_drv.class))
+ return -ENODEV;
+
+ mutex_init(&mhi_uci_drv.lock);
+ INIT_LIST_HEAD(&mhi_uci_drv.head);
+
+ ret = mhi_driver_register(&mhi_uci_driver);
+ if (ret)
+ class_destroy(mhi_uci_drv.class);
+
+ return ret;
+}
+
+module_init(mhi_uci_init);
+
+static void __exit mhi_uci_exit(void)
+{
+ unregister_chrdev(0, MHI_UCI_DRIVER_NAME);
+ mhi_driver_unregister(&mhi_uci_driver);
+ class_destroy(mhi_uci_drv.class);
+}
+
+module_exit(mhi_uci_exit);
+
+MODULE_AUTHOR("Qualcomm Corporation");
+MODULE_DESCRIPTION("Qualcomm Modem Host Interface Bus Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/qualcomm/rmnet/Makefile b/drivers/net/ethernet/qualcomm/rmnet/Makefile
index 8252e40bf570..64aec483046a 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/Makefile
+++ b/drivers/net/ethernet/qualcomm/rmnet/Makefile
@@ -1,11 +1,11 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# Makefile for the RMNET module
-#
rmnet-y := rmnet_config.o
rmnet-y += rmnet_vnd.o
rmnet-y += rmnet_handlers.o
rmnet-y += rmnet_map_data.o
rmnet-y += rmnet_map_command.o
+rmnet-y += rmnet_descriptor.o
obj-$(CONFIG_RMNET) += rmnet.o
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
index 06de59521fc4..4eb45f5757c0 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
@@ -1,17 +1,30 @@
// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*
* RMNET configuration engine
+ *
*/
#include <net/sock.h>
#include <linux/module.h>
#include <linux/netlink.h>
#include <linux/netdevice.h>
+#include <linux/hashtable.h>
#include "rmnet_config.h"
#include "rmnet_handlers.h"
#include "rmnet_vnd.h"
#include "rmnet_private.h"
+#include "rmnet_map.h"
+#include "rmnet_descriptor.h"
/* Locking scheme -
* The shared resource which needs to be protected is realdev->rx_handler_data.
@@ -34,15 +47,28 @@
/* Local Definitions and Declarations */
-static const struct nla_policy rmnet_policy[IFLA_RMNET_MAX + 1] = {
- [IFLA_RMNET_MUX_ID] = { .type = NLA_U16 },
- [IFLA_RMNET_FLAGS] = { .len = sizeof(struct ifla_rmnet_flags) },
+enum {
+ IFLA_RMNET_UL_AGG_PARAMS = __IFLA_RMNET_MAX,
+ __IFLA_RMNET_EXT_MAX,
+};
+
+static const struct nla_policy rmnet_policy[__IFLA_RMNET_EXT_MAX] = {
+ [IFLA_RMNET_MUX_ID] = {
+ .type = NLA_U16
+ },
+ [IFLA_RMNET_FLAGS] = {
+ .len = sizeof(struct ifla_rmnet_flags)
+ },
+ [IFLA_RMNET_UL_AGG_PARAMS] = {
+ .len = sizeof(struct rmnet_egress_agg_params)
+ },
};
-static int rmnet_is_real_dev_registered(const struct net_device *real_dev)
+int rmnet_is_real_dev_registered(const struct net_device *real_dev)
{
return rcu_access_pointer(real_dev->rx_handler) == rmnet_rx_handler;
}
+EXPORT_SYMBOL(rmnet_is_real_dev_registered);
/* Needs rtnl lock */
static struct rmnet_port*
@@ -57,10 +83,15 @@ static int rmnet_unregister_real_device(struct net_device *real_dev,
if (port->nr_rmnet_devs)
return -EINVAL;
- netdev_rx_handler_unregister(real_dev);
+ rmnet_map_cmd_exit(port);
+ rmnet_map_tx_aggregate_exit(port);
+
+ rmnet_descriptor_deinit(port);
kfree(port);
+ netdev_rx_handler_unregister(real_dev);
+
/* release reference on real_dev */
dev_put(real_dev);
@@ -88,13 +119,21 @@ static int rmnet_register_real_device(struct net_device *real_dev)
kfree(port);
return -EBUSY;
}
-
/* hold on to real dev for MAP data */
dev_hold(real_dev);
for (entry = 0; entry < RMNET_MAX_LOGICAL_EP; entry++)
INIT_HLIST_HEAD(&port->muxed_ep[entry]);
+ rc = rmnet_descriptor_init(port);
+ if (rc) {
+ rmnet_descriptor_deinit(port);
+ return rc;
+ }
+
+ rmnet_map_tx_aggregate_init(port);
+ rmnet_map_cmd_init(port);
+
netdev_dbg(real_dev, "registered with rmnet\n");
return 0;
}
@@ -171,6 +210,17 @@ static int rmnet_newlink(struct net *src_net, struct net_device *dev,
netdev_dbg(dev, "data format [0x%08X]\n", data_format);
port->data_format = data_format;
+ if (data[IFLA_RMNET_UL_AGG_PARAMS]) {
+ void *agg_params;
+ unsigned long irq_flags;
+
+ agg_params = nla_data(data[IFLA_RMNET_UL_AGG_PARAMS]);
+ spin_lock_irqsave(&port->agg_lock, irq_flags);
+ memcpy(&port->egress_agg_params, agg_params,
+ sizeof(port->egress_agg_params));
+ spin_unlock_irqrestore(&port->agg_lock, irq_flags);
+ }
+
return 0;
err1:
@@ -202,8 +252,10 @@ static void rmnet_dellink(struct net_device *dev, struct list_head *head)
hlist_del_init_rcu(&ep->hlnode);
rmnet_unregister_bridge(dev, port);
rmnet_vnd_dellink(mux_id, port, ep);
+ synchronize_rcu();
kfree(ep);
}
+
rmnet_unregister_real_device(real_dev, port);
unregister_netdevice_queue(dev, head);
@@ -225,7 +277,6 @@ static void rmnet_force_unassociate_device(struct net_device *dev)
port = rmnet_get_port_rtnl(dev);
- rcu_read_lock();
rmnet_unregister_bridge(dev, port);
hash_for_each_safe(port->muxed_ep, bkt_ep, tmp_ep, ep, hlnode) {
@@ -233,10 +284,10 @@ static void rmnet_force_unassociate_device(struct net_device *dev)
rmnet_vnd_dellink(ep->mux_id, port, ep);
hlist_del_init_rcu(&ep->hlnode);
+ synchronize_rcu();
kfree(ep);
}
- rcu_read_unlock();
unregister_netdevice_many(&list);
rmnet_unregister_real_device(real_dev, port);
@@ -252,6 +303,11 @@ static int rmnet_config_notify_cb(struct notifier_block *nb,
switch (event) {
case NETDEV_UNREGISTER:
+ if (dev->rtnl_link_ops == &rmnet_link_ops) {
+ netdev_rx_handler_unregister(dev);
+ break;
+ }
+
netdev_dbg(dev, "Kernel unregister\n");
rmnet_force_unassociate_device(dev);
break;
@@ -270,14 +326,24 @@ static struct notifier_block rmnet_dev_notifier __read_mostly = {
static int rmnet_rtnl_validate(struct nlattr *tb[], struct nlattr *data[],
struct netlink_ext_ack *extack)
{
+ struct rmnet_egress_agg_params *agg_params;
u16 mux_id;
- if (!data || !data[IFLA_RMNET_MUX_ID])
+ if (!data) {
return -EINVAL;
-
- mux_id = nla_get_u16(data[IFLA_RMNET_MUX_ID]);
- if (mux_id > (RMNET_MAX_LOGICAL_EP - 1))
- return -ERANGE;
+ } else {
+ if (data[IFLA_RMNET_MUX_ID]) {
+ mux_id = nla_get_u16(data[IFLA_RMNET_MUX_ID]);
+ if (mux_id > (RMNET_MAX_LOGICAL_EP - 1))
+ return -ERANGE;
+ }
+
+ if (data[IFLA_RMNET_UL_AGG_PARAMS]) {
+ agg_params = nla_data(data[IFLA_RMNET_UL_AGG_PARAMS]);
+ if (agg_params->agg_time < 3000000)
+ return -EINVAL;
+ }
+ }
return 0;
}
@@ -292,13 +358,10 @@ static int rmnet_changelink(struct net_device *dev, struct nlattr *tb[],
struct rmnet_port *port;
u16 mux_id;
- if (!dev)
- return -ENODEV;
-
real_dev = __dev_get_by_index(dev_net(dev),
nla_get_u32(tb[IFLA_LINK]));
- if (!real_dev || !rmnet_is_real_dev_registered(real_dev))
+ if (!real_dev || !dev || !rmnet_is_real_dev_registered(real_dev))
return -ENODEV;
port = rmnet_get_port_rtnl(real_dev);
@@ -323,6 +386,17 @@ static int rmnet_changelink(struct net_device *dev, struct nlattr *tb[],
port->data_format = flags->flags & flags->mask;
}
+ if (data[IFLA_RMNET_UL_AGG_PARAMS]) {
+ void *agg_params;
+ unsigned long irq_flags;
+
+ agg_params = nla_data(data[IFLA_RMNET_UL_AGG_PARAMS]);
+ spin_lock_irqsave(&port->agg_lock, irq_flags);
+ memcpy(&port->egress_agg_params, agg_params,
+ sizeof(port->egress_agg_params));
+ spin_unlock_irqrestore(&port->agg_lock, irq_flags);
+ }
+
return 0;
}
@@ -332,7 +406,9 @@ static size_t rmnet_get_size(const struct net_device *dev)
/* IFLA_RMNET_MUX_ID */
nla_total_size(2) +
/* IFLA_RMNET_FLAGS */
- nla_total_size(sizeof(struct ifla_rmnet_flags));
+ nla_total_size(sizeof(struct ifla_rmnet_flags)) +
+ /* IFLA_RMNET_UL_AGG_PARAMS */
+ nla_total_size(sizeof(struct rmnet_egress_agg_params));
}
static int rmnet_fill_info(struct sk_buff *skb, const struct net_device *dev)
@@ -340,7 +416,7 @@ static int rmnet_fill_info(struct sk_buff *skb, const struct net_device *dev)
struct rmnet_priv *priv = netdev_priv(dev);
struct net_device *real_dev;
struct ifla_rmnet_flags f;
- struct rmnet_port *port;
+ struct rmnet_port *port = NULL;
real_dev = priv->real_dev;
@@ -359,6 +435,13 @@ static int rmnet_fill_info(struct sk_buff *skb, const struct net_device *dev)
if (nla_put(skb, IFLA_RMNET_FLAGS, sizeof(f), &f))
goto nla_put_failure;
+ if (port) {
+ if (nla_put(skb, IFLA_RMNET_UL_AGG_PARAMS,
+ sizeof(port->egress_agg_params),
+ &port->egress_agg_params))
+ goto nla_put_failure;
+ }
+
return 0;
nla_put_failure:
@@ -367,14 +450,14 @@ static int rmnet_fill_info(struct sk_buff *skb, const struct net_device *dev)
struct rtnl_link_ops rmnet_link_ops __read_mostly = {
.kind = "rmnet",
- .maxtype = __IFLA_RMNET_MAX,
+ .maxtype = __IFLA_RMNET_EXT_MAX,
.priv_size = sizeof(struct rmnet_priv),
.setup = rmnet_vnd_setup,
.validate = rmnet_rtnl_validate,
.newlink = rmnet_newlink,
.dellink = rmnet_dellink,
.get_size = rmnet_get_size,
- .changelink = rmnet_changelink,
+ .changelink = rmnet_changelink,
.policy = rmnet_policy,
.fill_info = rmnet_fill_info,
};
@@ -387,6 +470,7 @@ struct rmnet_port *rmnet_get_port(struct net_device *real_dev)
else
return NULL;
}
+EXPORT_SYMBOL(rmnet_get_port);
struct rmnet_endpoint *rmnet_get_endpoint(struct rmnet_port *port, u8 mux_id)
{
@@ -399,10 +483,11 @@ struct rmnet_endpoint *rmnet_get_endpoint(struct rmnet_port *port, u8 mux_id)
return NULL;
}
+EXPORT_SYMBOL(rmnet_get_endpoint);
int rmnet_add_bridge(struct net_device *rmnet_dev,
struct net_device *slave_dev,
- struct netlink_ext_ack *extack)
+ struct netlink_ext_ack *extack)
{
struct rmnet_priv *priv = netdev_priv(rmnet_dev);
struct net_device *real_dev = priv->real_dev;
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
index cd0a6bcbe74a..03caf3399552 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
@@ -1,16 +1,28 @@
/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (c) 2013-2014, 2016-2018 The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2017, 2019 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*
* RMNET Data configuration engine
+ *
*/
#include <linux/skbuff.h>
#include <net/gro_cells.h>
+#include "rmnet_descriptor.h"
#ifndef _RMNET_CONFIG_H_
#define _RMNET_CONFIG_H_
#define RMNET_MAX_LOGICAL_EP 255
+#define RMNET_MAX_VEID 4
struct rmnet_endpoint {
u8 mux_id;
@@ -18,6 +30,27 @@ struct rmnet_endpoint {
struct hlist_node hlnode;
};
+struct rmnet_port_priv_stats {
+ u64 dl_hdr_last_qmap_vers;
+ u64 dl_hdr_last_ep_id;
+ u64 dl_hdr_last_trans_id;
+ u64 dl_hdr_last_seq;
+ u64 dl_hdr_last_bytes;
+ u64 dl_hdr_last_pkts;
+ u64 dl_hdr_last_flows;
+ u64 dl_hdr_count;
+ u64 dl_hdr_total_bytes;
+ u64 dl_hdr_total_pkts;
+ u64 dl_trl_last_seq;
+ u64 dl_trl_count;
+};
+
+struct rmnet_egress_agg_params {
+ u16 agg_size;
+ u16 agg_count;
+ u32 agg_time;
+};
+
/* One instance of this structure is instantiated for each real_dev associated
* with rmnet.
*/
@@ -28,6 +61,31 @@ struct rmnet_port {
u8 rmnet_mode;
struct hlist_head muxed_ep[RMNET_MAX_LOGICAL_EP];
struct net_device *bridge_ep;
+ void *rmnet_perf;
+
+ struct rmnet_egress_agg_params egress_agg_params;
+
+ /* Protect aggregation related elements */
+ spinlock_t agg_lock;
+
+ struct sk_buff *agg_skb;
+ int agg_state;
+ u8 agg_count;
+ struct timespec64 agg_time;
+ struct timespec64 agg_last;
+ struct hrtimer hrtimer;
+ struct work_struct agg_wq;
+
+ /* dl marker elements */
+ struct list_head dl_list;
+ struct rmnet_port_priv_stats stats;
+ int dl_marker_flush;
+
+ /* Descriptor pool */
+ spinlock_t desc_pool_lock;
+ struct rmnet_frag_descriptor_pool *frag_desc_pool;
+ struct sk_buff *chain_head;
+ struct sk_buff *chain_tail;
};
extern struct rtnl_link_ops rmnet_link_ops;
@@ -45,6 +103,31 @@ struct rmnet_pcpu_stats {
struct u64_stats_sync syncp;
};
+struct rmnet_coal_close_stats {
+ u64 non_coal;
+ u64 ip_miss;
+ u64 trans_miss;
+ u64 hw_nl;
+ u64 hw_pkt;
+ u64 hw_byte;
+ u64 hw_time;
+ u64 hw_evict;
+ u64 coal;
+};
+
+struct rmnet_coal_stats {
+ u64 coal_rx;
+ u64 coal_pkts;
+ u64 coal_hdr_nlo_err;
+ u64 coal_hdr_pkt_err;
+ u64 coal_csum_err;
+ u64 coal_reconstruct;
+ u64 coal_ip_invalid;
+ u64 coal_trans_invalid;
+ struct rmnet_coal_close_stats close;
+ u64 coal_veid[RMNET_MAX_VEID];
+};
+
struct rmnet_priv_stats {
u64 csum_ok;
u64 csum_valid_unset;
@@ -55,6 +138,8 @@ struct rmnet_priv_stats {
u64 csum_fragmented_pkt;
u64 csum_skipped;
u64 csum_sw;
+ u64 csum_hw;
+ struct rmnet_coal_stats coal;
};
struct rmnet_priv {
@@ -65,11 +150,32 @@ struct rmnet_priv {
struct rmnet_priv_stats stats;
};
+enum rmnet_dl_marker_prio {
+ RMNET_PERF,
+ RMNET_SHS,
+};
+
+enum rmnet_trace_func {
+ RMNET_MODULE,
+ NW_STACK_MODULE,
+};
+
+enum rmnet_trace_evt {
+ RMNET_DLVR_SKB,
+ RMNET_RCV_FROM_PND,
+ RMNET_TX_UL_PKT,
+ NW_STACK_DEV_Q_XMIT,
+ NW_STACK_NAPI_GRO_FLUSH,
+ NW_STACK_RX,
+ NW_STACK_TX,
+};
+
+int rmnet_is_real_dev_registered(const struct net_device *real_dev);
struct rmnet_port *rmnet_get_port(struct net_device *real_dev);
struct rmnet_endpoint *rmnet_get_endpoint(struct rmnet_port *port, u8 mux_id);
int rmnet_add_bridge(struct net_device *rmnet_dev,
struct net_device *slave_dev,
- struct netlink_ext_ack *extack);
+ struct netlink_ext_ack *extack);
int rmnet_del_bridge(struct net_device *rmnet_dev,
struct net_device *slave_dev);
#endif /* _RMNET_CONFIG_H_ */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_descriptor.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_descriptor.c
new file mode 100644
index 000000000000..fd1ae63b0605
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_descriptor.c
@@ -0,0 +1,1225 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * RMNET Packet Descriptor Framework
+ *
+ */
+
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <net/ipv6.h>
+#include <net/ip6_checksum.h>
+#include "rmnet_config.h"
+#include "rmnet_descriptor.h"
+#include "rmnet_handlers.h"
+#include "rmnet_private.h"
+#include "rmnet_vnd.h"
+
+#define RMNET_FRAG_DESCRIPTOR_POOL_SIZE 64
+#define RMNET_DL_IND_HDR_SIZE (sizeof(struct rmnet_map_dl_ind_hdr) + \
+ sizeof(struct rmnet_map_header) + \
+ sizeof(struct rmnet_map_control_command_header))
+#define RMNET_DL_IND_TRL_SIZE (sizeof(struct rmnet_map_dl_ind_trl) + \
+ sizeof(struct rmnet_map_header) + \
+ sizeof(struct rmnet_map_control_command_header))
+
+typedef void (*rmnet_perf_desc_hook_t)(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port);
+typedef void (*rmnet_perf_chain_hook_t)(void);
+
+struct rmnet_frag_descriptor *
+rmnet_get_frag_descriptor(struct rmnet_port *port)
+{
+ struct rmnet_frag_descriptor_pool *pool = port->frag_desc_pool;
+ struct rmnet_frag_descriptor *frag_desc;
+
+ spin_lock(&port->desc_pool_lock);
+ if (!list_empty(&pool->free_list)) {
+ frag_desc = list_first_entry(&pool->free_list,
+ struct rmnet_frag_descriptor,
+ list);
+ list_del_init(&frag_desc->list);
+ } else {
+ frag_desc = kzalloc(sizeof(*frag_desc), GFP_ATOMIC);
+ if (!frag_desc)
+ goto out;
+
+ INIT_LIST_HEAD(&frag_desc->list);
+ INIT_LIST_HEAD(&frag_desc->sub_frags);
+ pool->pool_size++;
+ }
+
+out:
+ spin_unlock(&port->desc_pool_lock);
+ return frag_desc;
+}
+EXPORT_SYMBOL(rmnet_get_frag_descriptor);
+
+void rmnet_recycle_frag_descriptor(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port)
+{
+ struct rmnet_frag_descriptor_pool *pool = port->frag_desc_pool;
+ struct page *page = skb_frag_page(&frag_desc->frag);
+
+ list_del(&frag_desc->list);
+ if (page)
+ put_page(page);
+
+ memset(frag_desc, 0, sizeof(*frag_desc));
+ INIT_LIST_HEAD(&frag_desc->list);
+ INIT_LIST_HEAD(&frag_desc->sub_frags);
+ spin_lock(&port->desc_pool_lock);
+ list_add_tail(&frag_desc->list, &pool->free_list);
+ spin_unlock(&port->desc_pool_lock);
+}
+EXPORT_SYMBOL(rmnet_recycle_frag_descriptor);
+
+void rmnet_descriptor_add_frag(struct rmnet_port *port, struct list_head *list,
+ struct page *p, u32 bv_page, u32 len)
+{
+ struct rmnet_frag_descriptor *frag_desc;
+
+ frag_desc = rmnet_get_frag_descriptor(port);
+ if (!frag_desc)
+ return;
+
+ rmnet_frag_fill(frag_desc, p, bv_page, len);
+ list_add_tail(&frag_desc->list, list);
+}
+EXPORT_SYMBOL(rmnet_descriptor_add_frag);
+
+int rmnet_frag_ipv6_skip_exthdr(struct rmnet_frag_descriptor *frag_desc,
+ int start, u8 *nexthdrp, __be16 *fragp)
+{
+ u8 nexthdr = *nexthdrp;
+
+ *fragp = 0;
+
+ while (ipv6_ext_hdr(nexthdr)) {
+ struct ipv6_opt_hdr *hp;
+ int hdrlen;
+
+ if (nexthdr == NEXTHDR_NONE)
+ return -EINVAL;
+
+ hp = rmnet_frag_data_ptr(frag_desc) + start;
+
+ if (nexthdr == NEXTHDR_FRAGMENT) {
+ __be16 *fp;
+
+ fp = rmnet_frag_data_ptr(frag_desc) + start +
+ offsetof(struct frag_hdr, frag_off);
+ *fragp = *fp;
+ if (ntohs(*fragp) & ~0x7)
+ break;
+ hdrlen = 8;
+ } else if (nexthdr == NEXTHDR_AUTH) {
+ hdrlen = (hp->hdrlen + 2) << 2;
+ } else {
+ hdrlen = ipv6_optlen(hp);
+ }
+
+ nexthdr = hp->nexthdr;
+ start += hdrlen;
+ }
+
+ *nexthdrp = nexthdr;
+ return start;
+}
+EXPORT_SYMBOL(rmnet_frag_ipv6_skip_exthdr);
+
+static u8 rmnet_frag_do_flow_control(struct rmnet_map_header *qmap,
+ struct rmnet_port *port,
+ int enable)
+{
+ struct rmnet_map_control_command *cmd;
+ struct rmnet_endpoint *ep;
+ struct net_device *vnd;
+ u16 ip_family;
+ u16 fc_seq;
+ u32 qos_id;
+ u8 mux_id;
+ int r;
+
+ mux_id = qmap->mux_id;
+ cmd = (struct rmnet_map_control_command *)
+ ((char *)qmap + sizeof(*qmap));
+
+ if (mux_id >= RMNET_MAX_LOGICAL_EP)
+ return RX_HANDLER_CONSUMED;
+
+ ep = rmnet_get_endpoint(port, mux_id);
+ if (!ep)
+ return RX_HANDLER_CONSUMED;
+
+ vnd = ep->egress_dev;
+
+ ip_family = cmd->flow_control.ip_family;
+ fc_seq = ntohs(cmd->flow_control.flow_control_seq_num);
+ qos_id = ntohl(cmd->flow_control.qos_id);
+
+ /* Ignore the ip family and pass the sequence number for both v4 and v6
+ * sequence. User space does not support creating dedicated flows for
+ * the 2 protocols
+ */
+ r = rmnet_vnd_do_flow_control(vnd, enable);
+ if (r)
+ return RMNET_MAP_COMMAND_UNSUPPORTED;
+ else
+ return RMNET_MAP_COMMAND_ACK;
+}
+
+static void rmnet_frag_send_ack(struct rmnet_map_header *qmap,
+ unsigned char type,
+ struct rmnet_port *port)
+{
+ struct rmnet_map_control_command *cmd;
+ struct net_device *dev = port->dev;
+ struct sk_buff *skb;
+ u16 alloc_len = ntohs(qmap->pkt_len) + sizeof(*qmap);
+
+ skb = alloc_skb(alloc_len, GFP_ATOMIC);
+ if (!skb)
+ return;
+
+ skb->protocol = htons(ETH_P_MAP);
+ skb->dev = dev;
+
+ cmd = rmnet_map_get_cmd_start(skb);
+ cmd->cmd_type = type & 0x03;
+
+ netif_tx_lock(dev);
+ dev->netdev_ops->ndo_start_xmit(skb, dev);
+ netif_tx_unlock(dev);
+}
+
+static void
+rmnet_frag_process_flow_start(struct rmnet_map_control_command_header *cmd,
+ struct rmnet_port *port,
+ u16 cmd_len)
+{
+ struct rmnet_map_dl_ind_hdr *dlhdr;
+ u32 data_format;
+ bool is_dl_mark_v2;
+
+ if (cmd_len + sizeof(struct rmnet_map_header) < RMNET_DL_IND_HDR_SIZE)
+ return;
+
+ data_format = port->data_format;
+ is_dl_mark_v2 = data_format & RMNET_INGRESS_FORMAT_DL_MARKER_V2;
+ dlhdr = (struct rmnet_map_dl_ind_hdr *)((char *)cmd + sizeof(*cmd));
+
+ port->stats.dl_hdr_last_ep_id = cmd->source_id;
+ port->stats.dl_hdr_last_qmap_vers = cmd->reserved;
+ port->stats.dl_hdr_last_trans_id = cmd->transaction_id;
+ port->stats.dl_hdr_last_seq = dlhdr->le.seq;
+ port->stats.dl_hdr_last_bytes = dlhdr->le.bytes;
+ port->stats.dl_hdr_last_pkts = dlhdr->le.pkts;
+ port->stats.dl_hdr_last_flows = dlhdr->le.flows;
+ port->stats.dl_hdr_total_bytes += port->stats.dl_hdr_last_bytes;
+ port->stats.dl_hdr_total_pkts += port->stats.dl_hdr_last_pkts;
+ port->stats.dl_hdr_count++;
+
+ /* If a target is taking frag path, we can assume DL marker v2 is in
+ * play
+ */
+ if (is_dl_mark_v2)
+ rmnet_map_dl_hdr_notify_v2(port, dlhdr, cmd);
+ else
+ rmnet_map_dl_hdr_notify(port, dlhdr);
+}
+
+static void
+rmnet_frag_process_flow_end(struct rmnet_map_control_command_header *cmd,
+ struct rmnet_port *port, u16 cmd_len)
+{
+ struct rmnet_map_dl_ind_trl *dltrl;
+ u32 data_format;
+ bool is_dl_mark_v2;
+
+ if (cmd_len + sizeof(struct rmnet_map_header) < RMNET_DL_IND_TRL_SIZE)
+ return;
+
+ data_format = port->data_format;
+ is_dl_mark_v2 = data_format & RMNET_INGRESS_FORMAT_DL_MARKER_V2;
+ dltrl = (struct rmnet_map_dl_ind_trl *)((char *)cmd + sizeof(*cmd));
+
+ port->stats.dl_trl_last_seq = dltrl->seq_le;
+ port->stats.dl_trl_count++;
+
+ /* If a target is taking frag path, we can assume DL marker v2 is in
+ * play
+ */
+ if (is_dl_mark_v2)
+ rmnet_map_dl_trl_notify_v2(port, dltrl, cmd);
+ else
+ rmnet_map_dl_trl_notify(port, dltrl);
+}
+
+/* Process MAP command frame and send N/ACK message as appropriate. Message cmd
+ * name is decoded here and appropriate handler is called.
+ */
+void rmnet_frag_command(struct rmnet_map_header *qmap, struct rmnet_port *port)
+{
+ struct rmnet_map_control_command *cmd;
+ unsigned char command_name;
+ unsigned char rc = 0;
+
+ cmd = (struct rmnet_map_control_command *)
+ ((char *)qmap + sizeof(*qmap));
+ command_name = cmd->command_name;
+
+ switch (command_name) {
+ case RMNET_MAP_COMMAND_FLOW_ENABLE:
+ rc = rmnet_frag_do_flow_control(qmap, port, 1);
+ break;
+
+ case RMNET_MAP_COMMAND_FLOW_DISABLE:
+ rc = rmnet_frag_do_flow_control(qmap, port, 0);
+ break;
+
+ default:
+ rc = RMNET_MAP_COMMAND_UNSUPPORTED;
+ break;
+ }
+ if (rc == RMNET_MAP_COMMAND_ACK)
+ rmnet_frag_send_ack(qmap, rc, port);
+}
+
+int rmnet_frag_flow_command(struct rmnet_map_header *qmap,
+ struct rmnet_port *port, u16 pkt_len)
+{
+ struct rmnet_map_control_command_header *cmd;
+ unsigned char command_name;
+
+ cmd = (struct rmnet_map_control_command_header *)
+ ((char *)qmap + sizeof(*qmap));
+ command_name = cmd->command_name;
+
+ switch (command_name) {
+ case RMNET_MAP_COMMAND_FLOW_START:
+ rmnet_frag_process_flow_start(cmd, port, pkt_len);
+ break;
+
+ case RMNET_MAP_COMMAND_FLOW_END:
+ rmnet_frag_process_flow_end(cmd, port, pkt_len);
+ break;
+
+ default:
+ return 1;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(rmnet_frag_flow_command);
+
+void rmnet_frag_deaggregate(skb_frag_t *frag, struct rmnet_port *port,
+ struct list_head *list)
+{
+ struct rmnet_map_header *maph;
+ u8 *data = skb_frag_address(frag);
+ u32 offset = 0;
+ u32 packet_len;
+
+ while (offset < skb_frag_size(frag)) {
+ maph = (struct rmnet_map_header *)data;
+ packet_len = ntohs(maph->pkt_len);
+
+ /* Some hardware can send us empty frames. Catch them */
+ if (packet_len == 0)
+ return;
+
+ packet_len += sizeof(*maph);
+
+ if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) {
+ packet_len += sizeof(struct rmnet_map_dl_csum_trailer);
+ } else if (port->data_format &
+ (RMNET_FLAGS_INGRESS_MAP_CKSUMV5 |
+ RMNET_FLAGS_INGRESS_COALESCE) && !maph->cd_bit) {
+ u32 hsize = 0;
+ u8 type;
+
+ type = ((struct rmnet_map_v5_coal_header *)
+ (data + sizeof(*maph)))->header_type;
+ switch (type) {
+ case RMNET_MAP_HEADER_TYPE_COALESCING:
+ hsize = sizeof(struct rmnet_map_v5_coal_header);
+ break;
+ case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD:
+ hsize = sizeof(struct rmnet_map_v5_csum_header);
+ break;
+ }
+
+ packet_len += hsize;
+ }
+
+ if ((int)skb_frag_size(frag) - (int)packet_len < 0)
+ return;
+
+ rmnet_descriptor_add_frag(port, list, skb_frag_page(frag),
+ frag->bv_offset + offset,
+ packet_len);
+
+ offset += packet_len;
+ data += packet_len;
+ }
+}
+
+/* Fill in GSO metadata to allow the SKB to be segmented by the NW stack
+ * if needed (i.e. forwarding, UDP GRO)
+ */
+static void rmnet_frag_gso_stamp(struct sk_buff *skb,
+ struct rmnet_frag_descriptor *frag_desc)
+{
+ struct skb_shared_info *shinfo = skb_shinfo(skb);
+
+ if (frag_desc->trans_proto == IPPROTO_TCP)
+ shinfo->gso_type = (frag_desc->ip_proto == 4) ?
+ SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
+ else
+ shinfo->gso_type = SKB_GSO_UDP_L4;
+
+ shinfo->gso_size = frag_desc->gso_size;
+ shinfo->gso_segs = frag_desc->gso_segs;
+}
+
+/* Set the partial checksum information. Sets the transport checksum tot he
+ * pseudoheader checksum and sets the offload metadata.
+ */
+static void rmnet_frag_partial_csum(struct sk_buff *skb,
+ struct rmnet_frag_descriptor *frag_desc)
+{
+ struct iphdr *iph = (struct iphdr *)skb->data;
+ __sum16 pseudo;
+ u16 pkt_len = skb->len - frag_desc->ip_len;
+
+ if (frag_desc->ip_proto == 4) {
+ iph->tot_len = htons(skb->len);
+ iph->check = 0;
+ iph->check = ip_fast_csum(iph, iph->ihl);
+ pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
+ pkt_len, frag_desc->trans_proto,
+ 0);
+ } else {
+ struct ipv6hdr *ip6h = (struct ipv6hdr *)iph;
+
+ /* Payload length includes any extension headers */
+ ip6h->payload_len = htons(skb->len - sizeof(*ip6h));
+ pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
+ pkt_len, frag_desc->trans_proto, 0);
+ }
+
+ if (frag_desc->trans_proto == IPPROTO_TCP) {
+ struct tcphdr *tp = (struct tcphdr *)
+ ((u8 *)iph + frag_desc->ip_len);
+
+ tp->check = pseudo;
+ skb->csum_offset = offsetof(struct tcphdr, check);
+ } else {
+ struct udphdr *up = (struct udphdr *)
+ ((u8 *)iph + frag_desc->ip_len);
+
+ up->len = htons(pkt_len);
+ up->check = pseudo;
+ skb->csum_offset = offsetof(struct udphdr, check);
+ }
+
+ skb->ip_summed = CHECKSUM_PARTIAL;
+ skb->csum_start = (u8 *)iph + frag_desc->ip_len - skb->head;
+}
+
+/* Allocate and populate an skb to contain the packet represented by the
+ * frag descriptor.
+ */
+static struct sk_buff *rmnet_alloc_skb(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port)
+{
+ struct sk_buff *head_skb, *current_skb, *skb;
+ struct skb_shared_info *shinfo;
+ struct rmnet_frag_descriptor *sub_frag, *tmp;
+
+ /* Use the exact sizes if we know them (i.e. RSB/RSC, rmnet_perf) */
+ if (frag_desc->hdrs_valid) {
+ u16 hdr_len = frag_desc->ip_len + frag_desc->trans_len;
+
+ head_skb = alloc_skb(hdr_len + RMNET_MAP_DESC_HEADROOM,
+ GFP_ATOMIC);
+ if (!head_skb)
+ return NULL;
+
+ skb_reserve(head_skb, RMNET_MAP_DESC_HEADROOM);
+ skb_put_data(head_skb, frag_desc->hdr_ptr, hdr_len);
+ skb_reset_network_header(head_skb);
+
+ if (frag_desc->trans_len)
+ skb_set_transport_header(head_skb, frag_desc->ip_len);
+
+ /* Packets that have no data portion don't need any frags */
+ if (hdr_len == skb_frag_size(&frag_desc->frag))
+ goto skip_frags;
+
+ /* If the headers we added are the start of the page,
+ * we don't want to add them twice
+ */
+ if (frag_desc->hdr_ptr == rmnet_frag_data_ptr(frag_desc)) {
+ if (!rmnet_frag_pull(frag_desc, port, hdr_len)) {
+ kfree_skb(head_skb);
+ return NULL;
+ }
+ }
+ } else {
+ /* Allocate enough space to avoid penalties in the stack
+ * from __pskb_pull_tail()
+ */
+ head_skb = alloc_skb(256 + RMNET_MAP_DESC_HEADROOM,
+ GFP_ATOMIC);
+ if (!head_skb)
+ return NULL;
+
+ skb_reserve(head_skb, RMNET_MAP_DESC_HEADROOM);
+ }
+
+ /* Add main fragment */
+ get_page(skb_frag_page(&frag_desc->frag));
+ skb_add_rx_frag(head_skb, 0, skb_frag_page(&frag_desc->frag),
+ frag_desc->frag.bv_offset,
+ skb_frag_size(&frag_desc->frag),
+ skb_frag_size(&frag_desc->frag));
+
+ shinfo = skb_shinfo(head_skb);
+ current_skb = head_skb;
+
+ /* Add in any frags from rmnet_perf */
+ list_for_each_entry_safe(sub_frag, tmp, &frag_desc->sub_frags, list) {
+ skb_frag_t *frag;
+ u32 frag_size;
+
+ frag = &sub_frag->frag;
+ frag_size = skb_frag_size(frag);
+
+add_frag:
+ if (shinfo->nr_frags < MAX_SKB_FRAGS) {
+ get_page(skb_frag_page(frag));
+ skb_add_rx_frag(current_skb, shinfo->nr_frags,
+ skb_frag_page(frag), frag->bv_offset,
+ frag_size, frag_size);
+ if (current_skb != head_skb) {
+ head_skb->len += frag_size;
+ head_skb->data_len += frag_size;
+ }
+ } else {
+ /* Alloc a new skb and try again */
+ skb = alloc_skb(0, GFP_ATOMIC);
+ if (!skb)
+ break;
+
+ if (current_skb == head_skb)
+ shinfo->frag_list = skb;
+ else
+ current_skb->next = skb;
+
+ current_skb = skb;
+ shinfo = skb_shinfo(current_skb);
+ goto add_frag;
+ }
+
+ rmnet_recycle_frag_descriptor(sub_frag, port);
+ }
+
+skip_frags:
+ head_skb->dev = frag_desc->dev;
+ rmnet_set_skb_proto(head_skb);
+
+ /* Handle any header metadata that needs to be updated after RSB/RSC
+ * segmentation
+ */
+ if (frag_desc->ip_id_set) {
+ struct iphdr *iph;
+
+ iph = (struct iphdr *)rmnet_map_data_ptr(head_skb);
+ csum_replace2(&iph->check, iph->id, frag_desc->ip_id);
+ iph->id = frag_desc->ip_id;
+ }
+
+ if (frag_desc->tcp_seq_set) {
+ struct tcphdr *th;
+
+ th = (struct tcphdr *)
+ (rmnet_map_data_ptr(head_skb) + frag_desc->ip_len);
+ th->seq = frag_desc->tcp_seq;
+ }
+
+ /* Handle csum offloading */
+ if (frag_desc->csum_valid && frag_desc->hdrs_valid) {
+ /* Set the partial checksum information */
+ rmnet_frag_partial_csum(head_skb, frag_desc);
+ } else if (frag_desc->csum_valid) {
+ /* Non-RSB/RSC/perf packet. The current checksum is fine */
+ head_skb->ip_summed = CHECKSUM_UNNECESSARY;
+ } else if (frag_desc->hdrs_valid &&
+ (frag_desc->trans_proto == IPPROTO_TCP ||
+ frag_desc->trans_proto == IPPROTO_UDP)) {
+ /* Unfortunately, we have to fake a bad checksum here, since
+ * the original bad value is lost by the hardware. The only
+ * reliable way to do it is to calculate the actual checksum
+ * and corrupt it.
+ */
+ __sum16 *check;
+ __wsum csum;
+ unsigned int offset = skb_transport_offset(head_skb);
+ __sum16 pseudo;
+
+ /* Calculate pseudo header and update header fields */
+ if (frag_desc->ip_proto == 4) {
+ struct iphdr *iph = ip_hdr(head_skb);
+ __be16 tot_len = htons(head_skb->len);
+
+ csum_replace2(&iph->check, iph->tot_len, tot_len);
+ iph->tot_len = tot_len;
+ pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
+ head_skb->len -
+ frag_desc->ip_len,
+ frag_desc->trans_proto, 0);
+ } else {
+ struct ipv6hdr *ip6h = ipv6_hdr(head_skb);
+
+ ip6h->payload_len = htons(head_skb->len -
+ sizeof(*ip6h));
+ pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
+ head_skb->len -
+ frag_desc->ip_len,
+ frag_desc->trans_proto, 0);
+ }
+
+ if (frag_desc->trans_proto == IPPROTO_TCP) {
+ check = &tcp_hdr(head_skb)->check;
+ } else {
+ udp_hdr(head_skb)->len = htons(head_skb->len -
+ frag_desc->ip_len);
+ check = &udp_hdr(head_skb)->check;
+ }
+
+ *check = pseudo;
+ csum = skb_checksum(head_skb, offset, head_skb->len - offset,
+ 0);
+ /* Add 1 to corrupt. This cannot produce a final value of 0
+ * since csum_fold() can't return a value of 0xFFFF
+ */
+ *check = csum16_add(csum_fold(csum), htons(1));
+ head_skb->ip_summed = CHECKSUM_NONE;
+ }
+
+ /* Handle any rmnet_perf metadata */
+ if (frag_desc->hash) {
+ head_skb->hash = frag_desc->hash;
+ head_skb->sw_hash = 1;
+ }
+
+ if (frag_desc->flush_shs)
+ head_skb->cb[0] = 1;
+
+ /* Handle coalesced packets */
+ if (frag_desc->gso_segs > 1)
+ rmnet_frag_gso_stamp(head_skb, frag_desc);
+
+ return head_skb;
+}
+
+/* Deliver the packets contained within a frag descriptor */
+void rmnet_frag_deliver(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port)
+{
+ struct sk_buff *skb;
+
+ skb = rmnet_alloc_skb(frag_desc, port);
+ if (skb)
+ rmnet_deliver_skb(skb, port);
+ rmnet_recycle_frag_descriptor(frag_desc, port);
+}
+EXPORT_SYMBOL(rmnet_frag_deliver);
+
+static void __rmnet_frag_segment_data(struct rmnet_frag_descriptor *coal_desc,
+ struct rmnet_port *port,
+ struct list_head *list, u8 pkt_id,
+ bool csum_valid)
+{
+ struct rmnet_priv *priv = netdev_priv(coal_desc->dev);
+ struct rmnet_frag_descriptor *new_frag;
+ u8 *hdr_start = rmnet_frag_data_ptr(coal_desc);
+ u32 offset;
+
+ new_frag = rmnet_get_frag_descriptor(port);
+ if (!new_frag)
+ return;
+
+ /* Account for header lengths to access the data start */
+ offset = coal_desc->frag.bv_offset + coal_desc->ip_len +
+ coal_desc->trans_len + coal_desc->data_offset;
+
+ /* Header information and most metadata is the same as the original */
+ memcpy(new_frag, coal_desc, sizeof(*coal_desc));
+ INIT_LIST_HEAD(&new_frag->list);
+ INIT_LIST_HEAD(&new_frag->sub_frags);
+ rmnet_frag_fill(new_frag, skb_frag_page(&coal_desc->frag), offset,
+ coal_desc->gso_size * coal_desc->gso_segs);
+
+ if (coal_desc->trans_proto == IPPROTO_TCP) {
+ struct tcphdr *th;
+
+ th = (struct tcphdr *)(hdr_start + coal_desc->ip_len);
+ new_frag->tcp_seq_set = 1;
+ new_frag->tcp_seq = htonl(ntohl(th->seq) +
+ coal_desc->data_offset);
+ }
+
+ if (coal_desc->ip_proto == 4) {
+ struct iphdr *iph;
+
+ iph = (struct iphdr *)hdr_start;
+ new_frag->ip_id_set = 1;
+ new_frag->ip_id = htons(ntohs(iph->id) + coal_desc->pkt_id);
+ }
+
+ new_frag->hdr_ptr = hdr_start;
+ new_frag->csum_valid = csum_valid;
+ priv->stats.coal.coal_reconstruct++;
+
+ /* Update meta information to move past the data we just segmented */
+ coal_desc->data_offset += coal_desc->gso_size * coal_desc->gso_segs;
+ coal_desc->pkt_id = pkt_id + 1;
+ coal_desc->gso_segs = 0;
+
+ list_add_tail(&new_frag->list, list);
+}
+
+static bool rmnet_frag_validate_csum(struct rmnet_frag_descriptor *frag_desc)
+{
+ u8 *data = rmnet_frag_data_ptr(frag_desc);
+ unsigned int datagram_len;
+ __wsum csum;
+ __sum16 pseudo;
+
+ datagram_len = skb_frag_size(&frag_desc->frag) - frag_desc->ip_len;
+ if (frag_desc->ip_proto == 4) {
+ struct iphdr *iph = (struct iphdr *)data;
+
+ pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
+ datagram_len,
+ frag_desc->trans_proto, 0);
+ } else {
+ struct ipv6hdr *ip6h = (struct ipv6hdr *)data;
+
+ pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
+ datagram_len, frag_desc->trans_proto,
+ 0);
+ }
+
+ csum = csum_partial(data + frag_desc->ip_len, datagram_len,
+ csum_unfold(pseudo));
+ return !csum_fold(csum);
+}
+
+/* Converts the coalesced frame into a list of descriptors.
+ * NLOs containing csum erros will not be included.
+ */
+static void
+rmnet_frag_segment_coal_data(struct rmnet_frag_descriptor *coal_desc,
+ u64 nlo_err_mask, struct rmnet_port *port,
+ struct list_head *list)
+{
+ struct iphdr *iph;
+ struct rmnet_priv *priv = netdev_priv(coal_desc->dev);
+ struct rmnet_map_v5_coal_header *coal_hdr;
+ u16 pkt_len;
+ u8 pkt, total_pkt = 0;
+ u8 nlo;
+ bool gro = coal_desc->dev->features & NETIF_F_GRO_HW;
+
+ /* Pull off the headers we no longer need */
+ if (!rmnet_frag_pull(coal_desc, port, sizeof(struct rmnet_map_header)))
+ return;
+
+ coal_hdr = (struct rmnet_map_v5_coal_header *)
+ rmnet_frag_data_ptr(coal_desc);
+ if (!rmnet_frag_pull(coal_desc, port, sizeof(*coal_hdr)))
+ return;
+
+ iph = (struct iphdr *)rmnet_frag_data_ptr(coal_desc);
+
+ if (iph->version == 4) {
+ coal_desc->ip_proto = 4;
+ coal_desc->ip_len = iph->ihl * 4;
+ coal_desc->trans_proto = iph->protocol;
+
+ /* Don't allow coalescing of any packets with IP options */
+ if (iph->ihl != 5)
+ gro = false;
+ } else if (iph->version == 6) {
+ struct ipv6hdr *ip6h = (struct ipv6hdr *)iph;
+ __be16 frag_off;
+ u8 protocol = ip6h->nexthdr;
+
+ coal_desc->ip_proto = 6;
+ coal_desc->ip_len = rmnet_frag_ipv6_skip_exthdr(coal_desc,
+ sizeof(*ip6h),
+ &protocol,
+ &frag_off);
+ coal_desc->trans_proto = protocol;
+
+ /* If we run into a problem, or this has a fragment header
+ * (which should technically not be possible, if the HW
+ * works as intended...), bail.
+ */
+ if (coal_desc->ip_len < 0 || frag_off) {
+ priv->stats.coal.coal_ip_invalid++;
+ return;
+ } else if (coal_desc->ip_len > sizeof(*ip6h)) {
+ /* Don't allow coalescing of any packets with IPv6
+ * extension headers.
+ */
+ gro = false;
+ }
+ } else {
+ priv->stats.coal.coal_ip_invalid++;
+ return;
+ }
+
+ if (coal_desc->trans_proto == IPPROTO_TCP) {
+ struct tcphdr *th;
+
+ th = (struct tcphdr *)((u8 *)iph + coal_desc->ip_len);
+ coal_desc->trans_len = th->doff * 4;
+ } else if (coal_desc->trans_proto == IPPROTO_UDP) {
+ coal_desc->trans_len = sizeof(struct udphdr);
+ } else {
+ priv->stats.coal.coal_trans_invalid++;
+ return;
+ }
+
+ coal_desc->hdrs_valid = 1;
+
+ if (rmnet_map_v5_csum_buggy(coal_hdr)) {
+ /* Mark the checksum as valid if it checks out */
+ if (rmnet_frag_validate_csum(coal_desc))
+ coal_desc->csum_valid = true;
+
+ coal_desc->hdr_ptr = rmnet_frag_data_ptr(coal_desc);
+ coal_desc->gso_size = ntohs(coal_hdr->nl_pairs[0].pkt_len);
+ coal_desc->gso_size -= coal_desc->ip_len + coal_desc->trans_len;
+ coal_desc->gso_segs = coal_hdr->nl_pairs[0].num_packets;
+ list_add_tail(&coal_desc->list, list);
+ return;
+ }
+
+ /* Fast-forward the case where we have 1 NLO (i.e. 1 packet length),
+ * no checksum errors, and are allowing GRO. We can just reuse this
+ * descriptor unchanged.
+ */
+ if (gro && coal_hdr->num_nlos == 1 && coal_hdr->csum_valid) {
+ coal_desc->csum_valid = true;
+ coal_desc->hdr_ptr = rmnet_frag_data_ptr(coal_desc);
+ coal_desc->gso_size = ntohs(coal_hdr->nl_pairs[0].pkt_len);
+ coal_desc->gso_size -= coal_desc->ip_len + coal_desc->trans_len;
+ coal_desc->gso_segs = coal_hdr->nl_pairs[0].num_packets;
+ list_add_tail(&coal_desc->list, list);
+ return;
+ }
+
+ /* Segment the coalesced descriptor into new packets */
+ for (nlo = 0; nlo < coal_hdr->num_nlos; nlo++) {
+ pkt_len = ntohs(coal_hdr->nl_pairs[nlo].pkt_len);
+ pkt_len -= coal_desc->ip_len + coal_desc->trans_len;
+ coal_desc->gso_size = pkt_len;
+ for (pkt = 0; pkt < coal_hdr->nl_pairs[nlo].num_packets;
+ pkt++, total_pkt++, nlo_err_mask >>= 1) {
+ bool csum_err = nlo_err_mask & 1;
+
+ /* Segment the packet if we're not sending the larger
+ * packet up the stack.
+ */
+ if (!gro) {
+ coal_desc->gso_segs = 1;
+ if (csum_err)
+ priv->stats.coal.coal_csum_err++;
+
+ __rmnet_frag_segment_data(coal_desc, port,
+ list, total_pkt,
+ !csum_err);
+ continue;
+ }
+
+ if (csum_err) {
+ priv->stats.coal.coal_csum_err++;
+
+ /* Segment out the good data */
+ if (coal_desc->gso_segs)
+ __rmnet_frag_segment_data(coal_desc,
+ port,
+ list,
+ total_pkt,
+ true);
+
+ /* Segment out the bad checksum */
+ coal_desc->gso_segs = 1;
+ __rmnet_frag_segment_data(coal_desc, port,
+ list, total_pkt,
+ false);
+ } else {
+ coal_desc->gso_segs++;
+ }
+ }
+
+ /* If we're switching NLOs, we need to send out everything from
+ * the previous one, if we haven't done so. NLOs only switch
+ * when the packet length changes.
+ */
+ if (coal_desc->gso_segs)
+ __rmnet_frag_segment_data(coal_desc, port, list,
+ total_pkt, true);
+ }
+}
+
+/* Record reason for coalescing pipe closure */
+static void rmnet_frag_data_log_close_stats(struct rmnet_priv *priv, u8 type,
+ u8 code)
+{
+ struct rmnet_coal_close_stats *stats = &priv->stats.coal.close;
+
+ switch (type) {
+ case RMNET_MAP_COAL_CLOSE_NON_COAL:
+ stats->non_coal++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_IP_MISS:
+ stats->ip_miss++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_TRANS_MISS:
+ stats->trans_miss++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_HW:
+ switch (code) {
+ case RMNET_MAP_COAL_CLOSE_HW_NL:
+ stats->hw_nl++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_HW_PKT:
+ stats->hw_pkt++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_HW_BYTE:
+ stats->hw_byte++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_HW_TIME:
+ stats->hw_time++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_HW_EVICT:
+ stats->hw_evict++;
+ break;
+ default:
+ break;
+ }
+ break;
+ case RMNET_MAP_COAL_CLOSE_COAL:
+ stats->coal++;
+ break;
+ default:
+ break;
+ }
+}
+
+/* Check if the coalesced header has any incorrect values, in which case, the
+ * entire coalesced frame must be dropped. Then check if there are any
+ * checksum issues
+ */
+static int
+rmnet_frag_data_check_coal_header(struct rmnet_frag_descriptor *frag_desc,
+ u64 *nlo_err_mask)
+{
+ struct rmnet_map_v5_coal_header *coal_hdr;
+ unsigned char *data = rmnet_frag_data_ptr(frag_desc);
+ struct rmnet_priv *priv = netdev_priv(frag_desc->dev);
+ u64 mask = 0;
+ int i;
+ u8 veid, pkts = 0;
+
+ coal_hdr = (struct rmnet_map_v5_coal_header *)
+ (data + sizeof(struct rmnet_map_header));
+ veid = coal_hdr->virtual_channel_id;
+
+ if (coal_hdr->num_nlos == 0 ||
+ coal_hdr->num_nlos > RMNET_MAP_V5_MAX_NLOS) {
+ priv->stats.coal.coal_hdr_nlo_err++;
+ return -EINVAL;
+ }
+
+ for (i = 0; i < RMNET_MAP_V5_MAX_NLOS; i++) {
+ /* If there is a checksum issue, we need to split
+ * up the skb. Rebuild the full csum error field
+ */
+ u8 err = coal_hdr->nl_pairs[i].csum_error_bitmap;
+ u8 pkt = coal_hdr->nl_pairs[i].num_packets;
+
+ mask |= ((u64)err) << (8 * i);
+
+ /* Track total packets in frame */
+ pkts += pkt;
+ if (pkts > RMNET_MAP_V5_MAX_PACKETS) {
+ priv->stats.coal.coal_hdr_pkt_err++;
+ return -EINVAL;
+ }
+ }
+
+ /* Track number of packets we get inside of coalesced frames */
+ priv->stats.coal.coal_pkts += pkts;
+
+ /* Update ethtool stats */
+ rmnet_frag_data_log_close_stats(priv,
+ coal_hdr->close_type,
+ coal_hdr->close_value);
+ if (veid < RMNET_MAX_VEID)
+ priv->stats.coal.coal_veid[veid]++;
+
+ *nlo_err_mask = mask;
+
+ return 0;
+}
+
+/* Process a QMAPv5 packet header */
+int rmnet_frag_process_next_hdr_packet(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port,
+ struct list_head *list,
+ u16 len)
+{
+ struct rmnet_priv *priv = netdev_priv(frag_desc->dev);
+ u64 nlo_err_mask;
+ int rc = 0;
+
+ switch (rmnet_frag_get_next_hdr_type(frag_desc)) {
+ case RMNET_MAP_HEADER_TYPE_COALESCING:
+ priv->stats.coal.coal_rx++;
+ rc = rmnet_frag_data_check_coal_header(frag_desc,
+ &nlo_err_mask);
+ if (rc)
+ return rc;
+
+ rmnet_frag_segment_coal_data(frag_desc, nlo_err_mask, port,
+ list);
+ if (list_first_entry(list, struct rmnet_frag_descriptor,
+ list) != frag_desc)
+ rmnet_recycle_frag_descriptor(frag_desc, port);
+ break;
+ case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD:
+ if (rmnet_frag_get_csum_valid(frag_desc)) {
+ priv->stats.csum_ok++;
+ frag_desc->csum_valid = true;
+ } else {
+ priv->stats.csum_valid_unset++;
+ }
+
+ if (!rmnet_frag_pull(frag_desc, port,
+ sizeof(struct rmnet_map_header) +
+ sizeof(struct rmnet_map_v5_csum_header))) {
+ rc = -EINVAL;
+ break;
+ }
+
+ frag_desc->hdr_ptr = rmnet_frag_data_ptr(frag_desc);
+
+ /* Remove padding only for csum offload packets.
+ * Coalesced packets should never have padding.
+ */
+ if (!rmnet_frag_trim(frag_desc, port, len)) {
+ rc = -EINVAL;
+ break;
+ }
+
+ list_del_init(&frag_desc->list);
+ list_add_tail(&frag_desc->list, list);
+ break;
+ default:
+ rc = -EINVAL;
+ break;
+ }
+
+ return rc;
+}
+
+/* Perf hook handler */
+rmnet_perf_desc_hook_t rmnet_perf_desc_entry __rcu __read_mostly;
+
+static void
+__rmnet_frag_ingress_handler(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port)
+{
+ rmnet_perf_desc_hook_t rmnet_perf_ingress;
+ struct rmnet_map_header *qmap;
+ struct rmnet_endpoint *ep;
+ struct rmnet_frag_descriptor *frag, *tmp;
+ LIST_HEAD(segs);
+ u16 len, pad;
+ u8 mux_id;
+
+ qmap = (struct rmnet_map_header *)skb_frag_address(&frag_desc->frag);
+ mux_id = qmap->mux_id;
+ pad = qmap->pad_len;
+ len = ntohs(qmap->pkt_len) - pad;
+
+ if (qmap->cd_bit) {
+ if (port->data_format & RMNET_INGRESS_FORMAT_DL_MARKER) {
+ rmnet_frag_flow_command(qmap, port, len);
+ goto recycle;
+ }
+
+ if (port->data_format & RMNET_FLAGS_INGRESS_MAP_COMMANDS)
+ rmnet_frag_command(qmap, port);
+
+ goto recycle;
+ }
+
+ if (mux_id >= RMNET_MAX_LOGICAL_EP)
+ goto recycle;
+
+ ep = rmnet_get_endpoint(port, mux_id);
+ if (!ep)
+ goto recycle;
+
+ frag_desc->dev = ep->egress_dev;
+
+ /* Handle QMAPv5 packet */
+ if (qmap->next_hdr &&
+ (port->data_format & (RMNET_FLAGS_INGRESS_COALESCE |
+ RMNET_FLAGS_INGRESS_MAP_CKSUMV5))) {
+ if (rmnet_frag_process_next_hdr_packet(frag_desc, port, &segs,
+ len))
+ goto recycle;
+ } else {
+ /* We only have the main QMAP header to worry about */
+ if (!rmnet_frag_pull(frag_desc, port, sizeof(*qmap)))
+ return;
+
+ frag_desc->hdr_ptr = rmnet_frag_data_ptr(frag_desc);
+
+ if (!rmnet_frag_trim(frag_desc, port, len))
+ return;
+
+ list_add_tail(&frag_desc->list, &segs);
+ }
+
+ rcu_read_lock();
+ rmnet_perf_ingress = rcu_dereference(rmnet_perf_desc_entry);
+ if (rmnet_perf_ingress) {
+ list_for_each_entry_safe(frag, tmp, &segs, list) {
+ list_del_init(&frag->list);
+ rmnet_perf_ingress(frag, port);
+ }
+ rcu_read_unlock();
+ return;
+ }
+ rcu_read_unlock();
+
+ list_for_each_entry_safe(frag, tmp, &segs, list) {
+ list_del_init(&frag->list);
+ rmnet_frag_deliver(frag, port);
+ }
+ return;
+
+recycle:
+ rmnet_recycle_frag_descriptor(frag_desc, port);
+}
+
+/* Notify perf at the end of SKB chain */
+rmnet_perf_chain_hook_t rmnet_perf_chain_end __rcu __read_mostly;
+
+void rmnet_frag_ingress_handler(struct sk_buff *skb,
+ struct rmnet_port *port)
+{
+ rmnet_perf_chain_hook_t rmnet_perf_opt_chain_end;
+ LIST_HEAD(desc_list);
+ int i = 0;
+
+ /* Deaggregation and freeing of HW originating
+ * buffers is done within here
+ */
+ while (skb) {
+ struct sk_buff *skb_frag;
+
+ port->chain_head = NULL;
+ port->chain_tail = NULL;
+
+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+ rmnet_frag_deaggregate(&skb_shinfo(skb)->frags[i], port,
+ &desc_list);
+ if (!list_empty(&desc_list)) {
+ struct rmnet_frag_descriptor *frag_desc, *tmp;
+
+ list_for_each_entry_safe(frag_desc, tmp,
+ &desc_list, list) {
+ list_del_init(&frag_desc->list);
+ __rmnet_frag_ingress_handler(frag_desc,
+ port);
+ }
+ }
+ }
+
+ skb_frag = skb_shinfo(skb)->frag_list;
+ skb_shinfo(skb)->frag_list = NULL;
+ consume_skb(skb);
+ skb = skb_frag;
+ }
+
+ rcu_read_lock();
+ rmnet_perf_opt_chain_end = rcu_dereference(rmnet_perf_chain_end);
+ if (rmnet_perf_opt_chain_end)
+ rmnet_perf_opt_chain_end();
+ rcu_read_unlock();
+}
+
+void rmnet_descriptor_deinit(struct rmnet_port *port)
+{
+ struct rmnet_frag_descriptor_pool *pool;
+ struct rmnet_frag_descriptor *frag_desc, *tmp;
+
+ pool = port->frag_desc_pool;
+
+ list_for_each_entry_safe(frag_desc, tmp, &pool->free_list, list) {
+ kfree(frag_desc);
+ pool->pool_size--;
+ }
+
+ kfree(pool);
+}
+
+int rmnet_descriptor_init(struct rmnet_port *port)
+{
+ struct rmnet_frag_descriptor_pool *pool;
+ int i;
+
+ spin_lock_init(&port->desc_pool_lock);
+ pool = kzalloc(sizeof(*pool), GFP_ATOMIC);
+ if (!pool)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&pool->free_list);
+ port->frag_desc_pool = pool;
+
+ for (i = 0; i < RMNET_FRAG_DESCRIPTOR_POOL_SIZE; i++) {
+ struct rmnet_frag_descriptor *frag_desc;
+
+ frag_desc = kzalloc(sizeof(*frag_desc), GFP_ATOMIC);
+ if (!frag_desc)
+ return -ENOMEM;
+
+ INIT_LIST_HEAD(&frag_desc->list);
+ INIT_LIST_HEAD(&frag_desc->sub_frags);
+ list_add_tail(&frag_desc->list, &pool->free_list);
+ pool->pool_size++;
+ }
+
+ return 0;
+}
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_descriptor.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_descriptor.h
new file mode 100644
index 000000000000..0f170aaef3d6
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_descriptor.h
@@ -0,0 +1,152 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * RMNET Packet Descriptor Framework
+ *
+ */
+
+#ifndef _RMNET_DESCRIPTOR_H_
+#define _RMNET_DESCRIPTOR_H_
+
+#include <linux/netdevice.h>
+#include <linux/list.h>
+#include <linux/skbuff.h>
+#include "rmnet_config.h"
+#include "rmnet_map.h"
+
+struct rmnet_frag_descriptor_pool {
+ struct list_head free_list;
+ u32 pool_size;
+};
+
+struct rmnet_frag_descriptor {
+ struct list_head list;
+ struct list_head sub_frags;
+ skb_frag_t frag;
+ u8 *hdr_ptr;
+ struct net_device *dev;
+ u32 hash;
+ __be32 tcp_seq;
+ __be16 ip_id;
+ u16 data_offset;
+ u16 gso_size;
+ u16 gso_segs;
+ u16 ip_len;
+ u16 trans_len;
+ u8 ip_proto;
+ u8 trans_proto;
+ u8 pkt_id;
+ u8 csum_valid:1,
+ hdrs_valid:1,
+ ip_id_set:1,
+ tcp_seq_set:1,
+ flush_shs:1,
+ reserved:3;
+};
+
+/* Descriptor management */
+struct rmnet_frag_descriptor *
+rmnet_get_frag_descriptor(struct rmnet_port *port);
+void rmnet_recycle_frag_descriptor(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port);
+void rmnet_descriptor_add_frag(struct rmnet_port *port, struct list_head *list,
+ struct page *p, u32 bv_offset, u32 len);
+int rmnet_frag_ipv6_skip_exthdr(struct rmnet_frag_descriptor *frag_desc,
+ int start, u8 *nexthdrp, __be16 *fragp);
+
+/* QMAP command packets */
+void rmnet_frag_command(struct rmnet_map_header *qmap, struct rmnet_port *port);
+int rmnet_frag_flow_command(struct rmnet_map_header *qmap,
+ struct rmnet_port *port, u16 pkt_len);
+
+/* Ingress data handlers */
+void rmnet_frag_deaggregate(skb_frag_t *frag, struct rmnet_port *port,
+ struct list_head *list);
+void rmnet_frag_deliver(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port);
+int rmnet_frag_process_next_hdr_packet(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port,
+ struct list_head *list,
+ u16 len);
+void rmnet_frag_ingress_handler(struct sk_buff *skb,
+ struct rmnet_port *port);
+
+int rmnet_descriptor_init(struct rmnet_port *port);
+void rmnet_descriptor_deinit(struct rmnet_port *port);
+
+static inline void *rmnet_frag_data_ptr(struct rmnet_frag_descriptor *frag_desc)
+{
+ return skb_frag_address(&frag_desc->frag);
+}
+
+static inline void *rmnet_frag_pull(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port,
+ unsigned int size)
+{
+ if (size >= skb_frag_size(&frag_desc->frag)) {
+ pr_info("%s(): Pulling %u bytes from %u byte pkt. Dropping\n",
+ __func__, size, skb_frag_size(&frag_desc->frag));
+ rmnet_recycle_frag_descriptor(frag_desc, port);
+ return NULL;
+ }
+
+ frag_desc->frag.bv_offset += size;
+ skb_frag_size_sub(&frag_desc->frag, size);
+
+ return rmnet_frag_data_ptr(frag_desc);
+}
+
+static inline void *rmnet_frag_trim(struct rmnet_frag_descriptor *frag_desc,
+ struct rmnet_port *port,
+ unsigned int size)
+{
+ if (!size) {
+ pr_info("%s(): Trimming %u byte pkt to 0. Dropping\n",
+ __func__, skb_frag_size(&frag_desc->frag));
+ rmnet_recycle_frag_descriptor(frag_desc, port);
+ return NULL;
+ }
+
+ if (size < skb_frag_size(&frag_desc->frag))
+ skb_frag_size_set(&frag_desc->frag, size);
+
+ return rmnet_frag_data_ptr(frag_desc);
+}
+
+static inline void rmnet_frag_fill(struct rmnet_frag_descriptor *frag_desc,
+ struct page *p, u32 bv_offset, u32 len)
+{
+ get_page(p);
+ __skb_frag_set_page(&frag_desc->frag, p);
+ skb_frag_size_set(&frag_desc->frag, len);
+ frag_desc->frag.bv_offset = bv_offset;
+}
+
+static inline u8
+rmnet_frag_get_next_hdr_type(struct rmnet_frag_descriptor *frag_desc)
+{
+ unsigned char *data = rmnet_frag_data_ptr(frag_desc);
+
+ data += sizeof(struct rmnet_map_header);
+ return ((struct rmnet_map_v5_coal_header *)data)->header_type;
+}
+
+static inline bool
+rmnet_frag_get_csum_valid(struct rmnet_frag_descriptor *frag_desc)
+{
+ unsigned char *data = rmnet_frag_data_ptr(frag_desc);
+
+ data += sizeof(struct rmnet_map_header);
+ return ((struct rmnet_map_v5_csum_header *)data)->csum_valid_required;
+}
+
+#endif /* _RMNET_DESCRIPTOR_H_ */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
index 1b74bc160402..5249a6913cf9 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
@@ -1,27 +1,77 @@
// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*
* RMNET Data ingress/egress handler
+ *
*/
#include <linux/netdevice.h>
#include <linux/netdev_features.h>
#include <linux/if_arp.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
#include <net/sock.h>
+#include <linux/tracepoint.h>
#include "rmnet_private.h"
#include "rmnet_config.h"
#include "rmnet_vnd.h"
#include "rmnet_map.h"
#include "rmnet_handlers.h"
+#include "rmnet_descriptor.h"
#define RMNET_IP_VERSION_4 0x40
#define RMNET_IP_VERSION_6 0x60
+#define CREATE_TRACE_POINTS
+#include "rmnet_trace.h"
+
+EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_low);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_high);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_err);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_wq_low);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_wq_high);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_shs_wq_err);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_perf_low);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_perf_high);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_perf_err);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_low);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_high);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_err);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_freq_update);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_freq_reset);
+EXPORT_TRACEPOINT_SYMBOL(rmnet_freq_boost);
/* Helper Functions */
-static void rmnet_set_skb_proto(struct sk_buff *skb)
+static int rmnet_check_skb_can_gro(struct sk_buff *skb)
+{
+ unsigned char *data = rmnet_map_data_ptr(skb);
+
+ switch (skb->protocol) {
+ case htons(ETH_P_IP):
+ if (((struct iphdr *)data)->protocol == IPPROTO_TCP)
+ return 0;
+ break;
+ case htons(ETH_P_IPV6):
+ if (((struct ipv6hdr *)data)->nexthdr == IPPROTO_TCP)
+ return 0;
+ fallthrough;
+ }
+
+ return -EPROTONOSUPPORT;
+}
+
+void rmnet_set_skb_proto(struct sk_buff *skb)
{
- switch (skb->data[0] & 0xF0) {
+ switch (rmnet_map_data_ptr(skb)[0] & 0xF0) {
case RMNET_IP_VERSION_4:
skb->protocol = htons(ETH_P_IP);
break;
@@ -33,21 +83,130 @@ static void rmnet_set_skb_proto(struct sk_buff *skb)
break;
}
}
+EXPORT_SYMBOL(rmnet_set_skb_proto);
+
+/* Shs hook handler */
+int (*rmnet_shs_skb_entry)(struct sk_buff *skb,
+ struct rmnet_port *port) __rcu __read_mostly;
+EXPORT_SYMBOL(rmnet_shs_skb_entry);
+
+/* Shs hook handler for work queue*/
+int (*rmnet_shs_skb_entry_wq)(struct sk_buff *skb,
+ struct rmnet_port *port) __rcu __read_mostly;
+EXPORT_SYMBOL(rmnet_shs_skb_entry_wq);
/* Generic handler */
-static void
-rmnet_deliver_skb(struct sk_buff *skb)
+void
+rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_port *port)
{
+ int (*rmnet_shs_stamp)(struct sk_buff *skb, struct rmnet_port *port);
struct rmnet_priv *priv = netdev_priv(skb->dev);
+ trace_rmnet_low(RMNET_MODULE, RMNET_DLVR_SKB, 0xDEF, 0xDEF,
+ 0xDEF, 0xDEF, (void *)skb, NULL);
+
skb_reset_transport_header(skb);
skb_reset_network_header(skb);
- rmnet_vnd_rx_fixup(skb, skb->dev);
+ rmnet_vnd_rx_fixup(skb->dev, skb->len);
skb->pkt_type = PACKET_HOST;
skb_set_mac_header(skb, 0);
- gro_cells_receive(&priv->gro_cells, skb);
+
+ rcu_read_lock();
+ rmnet_shs_stamp = rcu_dereference(rmnet_shs_skb_entry);
+ if (rmnet_shs_stamp) {
+ rmnet_shs_stamp(skb, port);
+ rcu_read_unlock();
+ return;
+ }
+ rcu_read_unlock();
+
+ if (port->data_format & RMNET_INGRESS_FORMAT_DL_MARKER) {
+ if (!rmnet_check_skb_can_gro(skb) &&
+ port->dl_marker_flush >= 0) {
+ struct napi_struct *napi = get_current_napi_context();
+
+ napi_gro_receive(napi, skb);
+ port->dl_marker_flush++;
+ } else {
+ netif_receive_skb(skb);
+ }
+ } else {
+ if (!rmnet_check_skb_can_gro(skb))
+ gro_cells_receive(&priv->gro_cells, skb);
+ else
+ netif_receive_skb(skb);
+ }
+}
+EXPORT_SYMBOL(rmnet_deliver_skb);
+
+/* Important to note, port cannot be used here if it has gone stale */
+void
+rmnet_deliver_skb_wq(struct sk_buff *skb, struct rmnet_port *port,
+ enum rmnet_packet_context ctx)
+{
+ int (*rmnet_shs_stamp)(struct sk_buff *skb, struct rmnet_port *port);
+ struct rmnet_priv *priv = netdev_priv(skb->dev);
+
+ trace_rmnet_low(RMNET_MODULE, RMNET_DLVR_SKB, 0xDEF, 0xDEF,
+ 0xDEF, 0xDEF, (void *)skb, NULL);
+ skb_reset_transport_header(skb);
+ skb_reset_network_header(skb);
+ rmnet_vnd_rx_fixup(skb->dev, skb->len);
+
+ skb->pkt_type = PACKET_HOST;
+ skb_set_mac_header(skb, 0);
+
+ /* packets coming from work queue context due to packet flush timer
+ * must go through the special workqueue path in SHS driver
+ */
+ rcu_read_lock();
+ rmnet_shs_stamp = (!ctx) ? rcu_dereference(rmnet_shs_skb_entry) :
+ rcu_dereference(rmnet_shs_skb_entry_wq);
+ if (rmnet_shs_stamp) {
+ rmnet_shs_stamp(skb, port);
+ rcu_read_unlock();
+ return;
+ }
+ rcu_read_unlock();
+
+ if (ctx == RMNET_NET_RX_CTX) {
+ if (port->data_format & RMNET_INGRESS_FORMAT_DL_MARKER) {
+ if (!rmnet_check_skb_can_gro(skb) &&
+ port->dl_marker_flush >= 0) {
+ struct napi_struct *napi =
+ get_current_napi_context();
+ napi_gro_receive(napi, skb);
+ port->dl_marker_flush++;
+ } else {
+ netif_receive_skb(skb);
+ }
+ } else {
+ if (!rmnet_check_skb_can_gro(skb))
+ gro_cells_receive(&priv->gro_cells, skb);
+ else
+ netif_receive_skb(skb);
+ }
+ } else {
+ if ((port->data_format & RMNET_INGRESS_FORMAT_DL_MARKER) &&
+ port->dl_marker_flush >= 0)
+ port->dl_marker_flush++;
+ gro_cells_receive(&priv->gro_cells, skb);
+ }
+}
+EXPORT_SYMBOL(rmnet_deliver_skb_wq);
+
+/* Deliver a list of skbs after undoing coalescing */
+static void rmnet_deliver_skb_list(struct sk_buff_head *head,
+ struct rmnet_port *port)
+{
+ struct sk_buff *skb;
+
+ while ((skb = __skb_dequeue(head))) {
+ rmnet_set_skb_proto(skb);
+ rmnet_deliver_skb(skb, port);
+ }
}
/* MAP handler */
@@ -56,20 +215,31 @@ static void
__rmnet_map_ingress_handler(struct sk_buff *skb,
struct rmnet_port *port)
{
+ struct rmnet_map_header *qmap;
struct rmnet_endpoint *ep;
+ struct sk_buff_head list;
u16 len, pad;
u8 mux_id;
- if (RMNET_MAP_GET_CD_BIT(skb)) {
+ /* We don't need the spinlock since only we touch this */
+ __skb_queue_head_init(&list);
+
+ qmap = (struct rmnet_map_header *)rmnet_map_data_ptr(skb);
+ if (qmap->cd_bit) {
+ if (port->data_format & RMNET_INGRESS_FORMAT_DL_MARKER) {
+ if (!rmnet_map_flow_command(skb, port, false))
+ return;
+ }
+
if (port->data_format & RMNET_FLAGS_INGRESS_MAP_COMMANDS)
return rmnet_map_command(skb, port);
goto free_skb;
}
- mux_id = RMNET_MAP_GET_MUX_ID(skb);
- pad = RMNET_MAP_GET_PAD(skb);
- len = RMNET_MAP_GET_LENGTH(skb) - pad;
+ mux_id = qmap->mux_id;
+ pad = qmap->pad_len;
+ len = ntohs(qmap->pkt_len) - pad;
if (mux_id >= RMNET_MAX_LOGICAL_EP)
goto free_skb;
@@ -80,31 +250,50 @@ __rmnet_map_ingress_handler(struct sk_buff *skb,
skb->dev = ep->egress_dev;
- /* Subtract MAP header */
- skb_pull(skb, sizeof(struct rmnet_map_header));
- rmnet_set_skb_proto(skb);
+ /* Handle QMAPv5 packet */
+ if (qmap->next_hdr &&
+ (port->data_format & (RMNET_FLAGS_INGRESS_COALESCE |
+ RMNET_FLAGS_INGRESS_MAP_CKSUMV5))) {
+ if (rmnet_map_process_next_hdr_packet(skb, &list, len))
+ goto free_skb;
+ } else {
+ /* We only have the main QMAP header to worry about */
+ pskb_pull(skb, sizeof(*qmap));
+
+ rmnet_set_skb_proto(skb);
- if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) {
- if (!rmnet_map_checksum_downlink_packet(skb, len + pad))
- skb->ip_summed = CHECKSUM_UNNECESSARY;
+ if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4) {
+ if (!rmnet_map_checksum_downlink_packet(skb, len + pad))
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ }
+
+ pskb_trim(skb, len);
+
+ /* Push the single packet onto the list */
+ __skb_queue_tail(&list, skb);
}
- skb_trim(skb, len);
- rmnet_deliver_skb(skb);
+ rmnet_deliver_skb_list(&list, port);
return;
free_skb:
kfree_skb(skb);
}
+int (*rmnet_perf_deag_entry)(struct sk_buff *skb,
+ struct rmnet_port *port) __rcu __read_mostly;
+EXPORT_SYMBOL(rmnet_perf_deag_entry);
+
static void
rmnet_map_ingress_handler(struct sk_buff *skb,
struct rmnet_port *port)
{
struct sk_buff *skbn;
+ int (*rmnet_perf_core_deaggregate)(struct sk_buff *skb,
+ struct rmnet_port *port);
if (skb->dev->type == ARPHRD_ETHER) {
- if (pskb_expand_head(skb, ETH_HLEN, 0, GFP_ATOMIC)) {
+ if (pskb_expand_head(skb, ETH_HLEN, 0, GFP_KERNEL)) {
kfree_skb(skb);
return;
}
@@ -112,13 +301,47 @@ rmnet_map_ingress_handler(struct sk_buff *skb,
skb_push(skb, ETH_HLEN);
}
- if (port->data_format & RMNET_FLAGS_INGRESS_DEAGGREGATION) {
- while ((skbn = rmnet_map_deaggregate(skb, port)) != NULL)
+ if (port->data_format & (RMNET_FLAGS_INGRESS_COALESCE |
+ RMNET_FLAGS_INGRESS_MAP_CKSUMV5)) {
+ if (skb_is_nonlinear(skb)) {
+ rmnet_frag_ingress_handler(skb, port);
+ return;
+ }
+ }
+
+ /* No aggregation. Pass the frame on as is */
+ if (!(port->data_format & RMNET_FLAGS_INGRESS_DEAGGREGATION)) {
+ __rmnet_map_ingress_handler(skb, port);
+ return;
+ }
+
+ /* Pass off handling to rmnet_perf module, if present */
+ rcu_read_lock();
+ rmnet_perf_core_deaggregate = rcu_dereference(rmnet_perf_deag_entry);
+ if (rmnet_perf_core_deaggregate) {
+ rmnet_perf_core_deaggregate(skb, port);
+ rcu_read_unlock();
+ return;
+ }
+ rcu_read_unlock();
+
+ /* Deaggregation and freeing of HW originating
+ * buffers is done within here
+ */
+ while (skb) {
+ struct sk_buff *skb_frag = skb_shinfo(skb)->frag_list;
+
+ skb_shinfo(skb)->frag_list = NULL;
+ while ((skbn = rmnet_map_deaggregate(skb, port)) != NULL) {
__rmnet_map_ingress_handler(skbn, port);
+ if (skbn == skb)
+ goto next_skb;
+ }
+
consume_skb(skb);
- } else {
- __rmnet_map_ingress_handler(skb, port);
+next_skb:
+ skb = skb_frag;
}
}
@@ -126,33 +349,48 @@ static int rmnet_map_egress_handler(struct sk_buff *skb,
struct rmnet_port *port, u8 mux_id,
struct net_device *orig_dev)
{
- int required_headroom, additional_header_len;
+ int required_headroom, additional_header_len, csum_type;
struct rmnet_map_header *map_header;
additional_header_len = 0;
required_headroom = sizeof(struct rmnet_map_header);
+ csum_type = 0;
if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4) {
additional_header_len = sizeof(struct rmnet_map_ul_csum_header);
- required_headroom += additional_header_len;
+ csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV4;
+ } else if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5) {
+ additional_header_len = sizeof(struct rmnet_map_v5_csum_header);
+ csum_type = RMNET_FLAGS_EGRESS_MAP_CKSUMV5;
}
+ required_headroom += additional_header_len;
+
if (skb_headroom(skb) < required_headroom) {
if (pskb_expand_head(skb, required_headroom, 0, GFP_ATOMIC))
return -ENOMEM;
}
- if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV4)
- rmnet_map_checksum_uplink_packet(skb, orig_dev);
+ if (csum_type)
+ rmnet_map_checksum_uplink_packet(skb, orig_dev, csum_type);
- map_header = rmnet_map_add_map_header(skb, additional_header_len, 0);
+ map_header = rmnet_map_add_map_header(skb, additional_header_len, 0,
+ port);
if (!map_header)
return -ENOMEM;
map_header->mux_id = mux_id;
- skb->protocol = htons(ETH_P_MAP);
+ if (port->data_format & RMNET_EGRESS_FORMAT_AGGREGATION) {
+ if (rmnet_map_tx_agg_skip(skb, required_headroom))
+ goto done;
+ rmnet_map_tx_aggregate(skb, port);
+ return -EINPROGRESS;
+ }
+
+done:
+ skb->protocol = htons(ETH_P_MAP);
return 0;
}
@@ -183,9 +421,14 @@ rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb)
if (skb->pkt_type == PACKET_LOOPBACK)
return RX_HANDLER_PASS;
+ trace_rmnet_low(RMNET_MODULE, RMNET_RCV_FROM_PND, 0xDEF,
+ 0xDEF, 0xDEF, 0xDEF, NULL, NULL);
dev = skb->dev;
port = rmnet_get_port(dev);
+ port->chain_head = NULL;
+ port->chain_tail = NULL;
+
switch (port->rmnet_mode) {
case RMNET_EPMODE_VND:
rmnet_map_ingress_handler(skb, port);
@@ -198,6 +441,7 @@ rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb)
done:
return RX_HANDLER_CONSUMED;
}
+EXPORT_SYMBOL(rmnet_rx_handler);
/* Modifies packet as per logical endpoint configuration and egress data format
* for egress device configured in logical endpoint. Packet is then transmitted
@@ -209,8 +453,13 @@ void rmnet_egress_handler(struct sk_buff *skb)
struct rmnet_port *port;
struct rmnet_priv *priv;
u8 mux_id;
+ int err;
+ u32 skb_len;
- sk_pacing_shift_update(skb->sk, 8);
+ trace_rmnet_low(RMNET_MODULE, RMNET_TX_UL_PKT, 0xDEF, 0xDEF, 0xDEF,
+ 0xDEF, (void *)skb, NULL);
+
+ skb_orphan(skb);
orig_dev = skb->dev;
priv = netdev_priv(orig_dev);
@@ -221,10 +470,16 @@ void rmnet_egress_handler(struct sk_buff *skb)
if (!port)
goto drop;
- if (rmnet_map_egress_handler(skb, port, mux_id, orig_dev))
+ skb_len = skb->len;
+ err = rmnet_map_egress_handler(skb, port, mux_id, orig_dev);
+ if (err == -ENOMEM) {
goto drop;
+ } else if (err == -EINPROGRESS) {
+ rmnet_vnd_tx_fixup(orig_dev, skb_len);
+ return;
+ }
- rmnet_vnd_tx_fixup(skb, orig_dev);
+ rmnet_vnd_tx_fixup(orig_dev, skb_len);
dev_queue_xmit(skb);
return;
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h
index c4571dc3239d..c61bbe1543de 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h
@@ -1,16 +1,37 @@
/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (c) 2013, 2016-2017 The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013, 2016-2017, 2019
+ * The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*
* RMNET Data ingress/egress handler
+ *
*/
#ifndef _RMNET_HANDLERS_H_
#define _RMNET_HANDLERS_H_
#include "rmnet_config.h"
+#include "rmnet_descriptor.h"
-void rmnet_egress_handler(struct sk_buff *skb);
+enum rmnet_packet_context {
+ RMNET_NET_RX_CTX,
+ RMNET_WQ_CTX,
+};
+void rmnet_egress_handler(struct sk_buff *skb);
+void rmnet_deliver_skb(struct sk_buff *skb, struct rmnet_port *port);
+void rmnet_deliver_skb_wq(struct sk_buff *skb, struct rmnet_port *port,
+ enum rmnet_packet_context ctx);
+void rmnet_set_skb_proto(struct sk_buff *skb);
+rx_handler_result_t _rmnet_map_ingress_handler(struct sk_buff *skb,
+ struct rmnet_port *port);
rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb);
-
#endif /* _RMNET_HANDLERS_H_ */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
index 576501db2a0b..946a8b245e68 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
@@ -1,10 +1,21 @@
/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*/
#ifndef _RMNET_MAP_H_
#define _RMNET_MAP_H_
-#include <linux/if_rmnet.h>
+
+#include <linux/skbuff.h>
+#include "rmnet_config.h"
struct rmnet_map_control_command {
u8 command_name;
@@ -27,11 +38,162 @@ enum rmnet_map_commands {
RMNET_MAP_COMMAND_NONE,
RMNET_MAP_COMMAND_FLOW_DISABLE,
RMNET_MAP_COMMAND_FLOW_ENABLE,
+ RMNET_MAP_COMMAND_FLOW_START = 7,
+ RMNET_MAP_COMMAND_FLOW_END = 8,
/* These should always be the last 2 elements */
RMNET_MAP_COMMAND_UNKNOWN,
RMNET_MAP_COMMAND_ENUM_LENGTH
};
+enum rmnet_map_v5_header_type {
+ RMNET_MAP_HEADER_TYPE_UNKNOWN,
+ RMNET_MAP_HEADER_TYPE_COALESCING = 0x1,
+ RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD = 0x2,
+ RMNET_MAP_HEADER_TYPE_ENUM_LENGTH
+};
+
+enum rmnet_map_v5_close_type {
+ RMNET_MAP_COAL_CLOSE_NON_COAL,
+ RMNET_MAP_COAL_CLOSE_IP_MISS,
+ RMNET_MAP_COAL_CLOSE_TRANS_MISS,
+ RMNET_MAP_COAL_CLOSE_HW,
+ RMNET_MAP_COAL_CLOSE_COAL,
+};
+
+enum rmnet_map_v5_close_value {
+ RMNET_MAP_COAL_CLOSE_HW_NL,
+ RMNET_MAP_COAL_CLOSE_HW_PKT,
+ RMNET_MAP_COAL_CLOSE_HW_BYTE,
+ RMNET_MAP_COAL_CLOSE_HW_TIME,
+ RMNET_MAP_COAL_CLOSE_HW_EVICT,
+};
+
+/* Main QMAP header */
+struct rmnet_map_header {
+ u8 pad_len:6;
+ u8 next_hdr:1;
+ u8 cd_bit:1;
+ u8 mux_id;
+ __be16 pkt_len;
+} __aligned(1);
+
+/* QMAP v5 headers */
+struct rmnet_map_v5_csum_header {
+ u8 next_hdr:1;
+ u8 header_type:7;
+ u8 hw_reserved:7;
+ u8 csum_valid_required:1;
+ __be16 reserved;
+} __aligned(1);
+
+struct rmnet_map_v5_nl_pair {
+ __be16 pkt_len;
+ u8 csum_error_bitmap;
+ u8 num_packets;
+} __aligned(1);
+
+/* NLO: Number-length object */
+#define RMNET_MAP_V5_MAX_NLOS (6)
+#define RMNET_MAP_V5_MAX_PACKETS (48)
+
+struct rmnet_map_v5_coal_header {
+ u8 next_hdr:1;
+ u8 header_type:7;
+ u8 reserved1:4;
+ u8 num_nlos:3;
+ u8 csum_valid:1;
+ u8 close_type:4;
+ u8 close_value:4;
+ u8 reserved2:4;
+ u8 virtual_channel_id:4;
+
+ struct rmnet_map_v5_nl_pair nl_pairs[RMNET_MAP_V5_MAX_NLOS];
+} __aligned(1);
+
+/* QMAP v4 headers */
+struct rmnet_map_dl_csum_trailer {
+ u8 reserved1;
+ u8 valid:1;
+ u8 reserved2:7;
+ u16 csum_start_offset;
+ u16 csum_length;
+ __be16 csum_value;
+} __aligned(1);
+
+struct rmnet_map_ul_csum_header {
+ __be16 csum_start_offset;
+ u16 csum_insert_offset:14;
+ u16 udp_ind:1;
+ u16 csum_enabled:1;
+} __aligned(1);
+
+struct rmnet_map_control_command_header {
+ u8 command_name;
+ u8 cmd_type:2;
+ u8 reserved:5;
+ u8 e:1;
+ u16 source_id:15;
+ u16 ext:1;
+ u32 transaction_id;
+} __aligned(1);
+
+struct rmnet_map_flow_info_le {
+ __be32 mux_id;
+ __be32 flow_id;
+ __be32 bytes;
+ __be32 pkts;
+} __aligned(1);
+
+struct rmnet_map_flow_info_be {
+ u32 mux_id;
+ u32 flow_id;
+ u32 bytes;
+ u32 pkts;
+} __aligned(1);
+
+struct rmnet_map_dl_ind_hdr {
+ union {
+ struct {
+ u32 seq;
+ u32 bytes;
+ u32 pkts;
+ u32 flows;
+ struct rmnet_map_flow_info_le flow[0];
+ } le __aligned(1);
+ struct {
+ __be32 seq;
+ __be32 bytes;
+ __be32 pkts;
+ __be32 flows;
+ struct rmnet_map_flow_info_be flow[0];
+ } be __aligned(1);
+ } __aligned(1);
+} __aligned(1);
+
+struct rmnet_map_dl_ind_trl {
+ union {
+ __be32 seq_be;
+ u32 seq_le;
+ } __aligned(1);
+} __aligned(1);
+
+typedef void (*dl_hdr_hdl_v2_t)(struct rmnet_map_dl_ind_hdr *hdr,
+ struct rmnet_map_control_command_header *h);
+typedef void (*dl_trl_hdl_v2_t)(struct rmnet_map_dl_ind_trl *hdr,
+ struct rmnet_map_control_command_header *h);
+struct rmnet_map_dl_ind {
+ u8 priority;
+ union {
+ void (*dl_hdr_handler)(struct rmnet_map_dl_ind_hdr *hdr);
+ dl_hdr_hdl_v2_t dl_hdr_handler_v2;
+ } __aligned(1);
+ union {
+ void (*dl_trl_handler)(struct rmnet_map_dl_ind_trl *hdr);
+ dl_trl_hdl_v2_t dl_trl_handler_v2;
+ } __aligned(1);
+ struct list_head list;
+};
+
#define RMNET_MAP_GET_MUX_ID(Y) (((struct rmnet_map_header *) \
(Y)->data)->mux_id)
#define RMNET_MAP_GET_CD_BIT(Y) (((struct rmnet_map_header *) \
@@ -44,6 +206,10 @@ enum rmnet_map_commands {
#define RMNET_MAP_GET_LENGTH(Y) (ntohs(((struct rmnet_map_header *) \
(Y)->data)->pkt_len))
+#define RMNET_MAP_DEAGGR_SPACING 64
+#define RMNET_MAP_DEAGGR_HEADROOM (RMNET_MAP_DEAGGR_SPACING / 2)
+#define RMNET_MAP_DESC_HEADROOM 128
+
#define RMNET_MAP_COMMAND_REQUEST 0
#define RMNET_MAP_COMMAND_ACK 1
#define RMNET_MAP_COMMAND_UNSUPPORTED 2
@@ -52,13 +218,75 @@ enum rmnet_map_commands {
#define RMNET_MAP_NO_PAD_BYTES 0
#define RMNET_MAP_ADD_PAD_BYTES 1
+static inline unsigned char *rmnet_map_data_ptr(struct sk_buff *skb)
+{
+ /* Nonlinear packets we receive are entirely within frag 0 */
+ if (skb_is_nonlinear(skb) && skb->len == skb->data_len)
+ return skb_frag_address(skb_shinfo(skb)->frags);
+
+ return skb->data;
+}
+
+static inline struct rmnet_map_control_command *
+rmnet_map_get_cmd_start(struct sk_buff *skb)
+{
+ unsigned char *data = rmnet_map_data_ptr(skb);
+
+ data += sizeof(struct rmnet_map_header);
+ return (struct rmnet_map_control_command *)data;
+}
+
+static inline u8 rmnet_map_get_next_hdr_type(struct sk_buff *skb)
+{
+ unsigned char *data = rmnet_map_data_ptr(skb);
+
+ data += sizeof(struct rmnet_map_header);
+ return ((struct rmnet_map_v5_coal_header *)data)->header_type;
+}
+
+static inline bool rmnet_map_get_csum_valid(struct sk_buff *skb)
+{
+ unsigned char *data = rmnet_map_data_ptr(skb);
+
+ data += sizeof(struct rmnet_map_header);
+ return ((struct rmnet_map_v5_csum_header *)data)->csum_valid_required;
+}
+
struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb,
struct rmnet_port *port);
struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
- int hdrlen, int pad);
+ int hdrlen, int pad,
+ struct rmnet_port *port);
void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port);
int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len);
void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
- struct net_device *orig_dev);
-
+ struct net_device *orig_dev,
+ int csum_type);
+bool rmnet_map_v5_csum_buggy(struct rmnet_map_v5_coal_header *coal_hdr);
+int rmnet_map_process_next_hdr_packet(struct sk_buff *skb,
+ struct sk_buff_head *list,
+ u16 len);
+int rmnet_map_tx_agg_skip(struct sk_buff *skb, int offset);
+void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port);
+void rmnet_map_tx_aggregate_init(struct rmnet_port *port);
+void rmnet_map_tx_aggregate_exit(struct rmnet_port *port);
+void rmnet_map_dl_hdr_notify(struct rmnet_port *port,
+ struct rmnet_map_dl_ind_hdr *dl_hdr);
+void rmnet_map_dl_hdr_notify_v2(struct rmnet_port *port,
+ struct rmnet_map_dl_ind_hdr *dl_hdr,
+ struct rmnet_map_control_command_header *qcmd);
+void rmnet_map_dl_trl_notify(struct rmnet_port *port,
+ struct rmnet_map_dl_ind_trl *dltrl);
+void rmnet_map_dl_trl_notify_v2(struct rmnet_port *port,
+ struct rmnet_map_dl_ind_trl *dltrl,
+ struct rmnet_map_control_command_header *qcmd);
+int rmnet_map_flow_command(struct sk_buff *skb,
+ struct rmnet_port *port,
+ bool rmnet_perf);
+void rmnet_map_cmd_init(struct rmnet_port *port);
+int rmnet_map_dl_ind_register(struct rmnet_port *port,
+ struct rmnet_map_dl_ind *dl_ind);
+int rmnet_map_dl_ind_deregister(struct rmnet_port *port,
+ struct rmnet_map_dl_ind *dl_ind);
+void rmnet_map_cmd_exit(struct rmnet_port *port);
#endif /* _RMNET_MAP_H_ */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
index beaee4962128..645469a29850 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
@@ -1,5 +1,14 @@
// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*/
#include <linux/netdevice.h>
@@ -8,16 +17,34 @@
#include "rmnet_private.h"
#include "rmnet_vnd.h"
+#define RMNET_DL_IND_HDR_SIZE (sizeof(struct rmnet_map_dl_ind_hdr) + \
+ sizeof(struct rmnet_map_header) + \
+ sizeof(struct rmnet_map_control_command_header))
+
+#define RMNET_MAP_CMD_SIZE (sizeof(struct rmnet_map_header) + \
+ sizeof(struct rmnet_map_control_command_header))
+
+#define RMNET_DL_IND_TRL_SIZE (sizeof(struct rmnet_map_dl_ind_trl) + \
+ sizeof(struct rmnet_map_header) + \
+ sizeof(struct rmnet_map_control_command_header))
+
static u8 rmnet_map_do_flow_control(struct sk_buff *skb,
struct rmnet_port *port,
int enable)
{
+ struct rmnet_map_header *qmap;
+ struct rmnet_map_control_command *cmd;
struct rmnet_endpoint *ep;
struct net_device *vnd;
+ u16 ip_family;
+ u16 fc_seq;
+ u32 qos_id;
u8 mux_id;
int r;
- mux_id = RMNET_MAP_GET_MUX_ID(skb);
+ qmap = (struct rmnet_map_header *)rmnet_map_data_ptr(skb);
+ mux_id = qmap->mux_id;
+ cmd = rmnet_map_get_cmd_start(skb);
if (mux_id >= RMNET_MAX_LOGICAL_EP) {
kfree_skb(skb);
@@ -32,6 +59,10 @@ static u8 rmnet_map_do_flow_control(struct sk_buff *skb,
vnd = ep->egress_dev;
+ ip_family = cmd->flow_control.ip_family;
+ fc_seq = ntohs(cmd->flow_control.flow_control_seq_num);
+ qos_id = ntohl(cmd->flow_control.qos_id);
+
/* Ignore the ip family and pass the sequence number for both v4 and v6
* sequence. User space does not support creating dedicated flows for
* the 2 protocols
@@ -53,12 +84,12 @@ static void rmnet_map_send_ack(struct sk_buff *skb,
struct net_device *dev = skb->dev;
if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4)
- skb_trim(skb,
- skb->len - sizeof(struct rmnet_map_dl_csum_trailer));
+ pskb_trim(skb,
+ skb->len - sizeof(struct rmnet_map_dl_csum_trailer));
skb->protocol = htons(ETH_P_MAP);
- cmd = RMNET_MAP_GET_CMD_START(skb);
+ cmd = rmnet_map_get_cmd_start(skb);
cmd->cmd_type = type & 0x03;
netif_tx_lock(dev);
@@ -66,6 +97,162 @@ static void rmnet_map_send_ack(struct sk_buff *skb,
netif_tx_unlock(dev);
}
+void
+rmnet_map_dl_hdr_notify_v2(struct rmnet_port *port,
+ struct rmnet_map_dl_ind_hdr *dlhdr,
+ struct rmnet_map_control_command_header *qcmd)
+{
+ struct rmnet_map_dl_ind *tmp;
+
+ port->dl_marker_flush = 0;
+
+ list_for_each_entry(tmp, &port->dl_list, list)
+ tmp->dl_hdr_handler_v2(dlhdr, qcmd);
+}
+
+void
+rmnet_map_dl_hdr_notify(struct rmnet_port *port,
+ struct rmnet_map_dl_ind_hdr *dlhdr)
+{
+ struct rmnet_map_dl_ind *tmp;
+
+ port->dl_marker_flush = 0;
+
+ list_for_each_entry(tmp, &port->dl_list, list)
+ tmp->dl_hdr_handler(dlhdr);
+}
+
+void
+rmnet_map_dl_trl_notify_v2(struct rmnet_port *port,
+ struct rmnet_map_dl_ind_trl *dltrl,
+ struct rmnet_map_control_command_header *qcmd)
+{
+ struct rmnet_map_dl_ind *tmp;
+ struct napi_struct *napi;
+
+ list_for_each_entry(tmp, &port->dl_list, list)
+ tmp->dl_trl_handler_v2(dltrl, qcmd);
+
+ if (port->dl_marker_flush) {
+ napi = get_current_napi_context();
+ napi_gro_flush(napi, false);
+ }
+
+ port->dl_marker_flush = -1;
+}
+
+void
+rmnet_map_dl_trl_notify(struct rmnet_port *port,
+ struct rmnet_map_dl_ind_trl *dltrl)
+{
+ struct rmnet_map_dl_ind *tmp;
+ struct napi_struct *napi;
+
+ list_for_each_entry(tmp, &port->dl_list, list)
+ tmp->dl_trl_handler(dltrl);
+
+ if (port->dl_marker_flush) {
+ napi = get_current_napi_context();
+ napi_gro_flush(napi, false);
+ }
+
+ port->dl_marker_flush = -1;
+}
+
+static void rmnet_map_process_flow_start(struct sk_buff *skb,
+ struct rmnet_port *port,
+ bool rmnet_perf)
+{
+ struct rmnet_map_dl_ind_hdr *dlhdr;
+ struct rmnet_map_control_command_header *qcmd;
+ u32 data_format;
+ bool is_dl_mark_v2;
+
+ if (skb->len < RMNET_DL_IND_HDR_SIZE)
+ return;
+
+ data_format = port->data_format;
+ is_dl_mark_v2 = data_format & RMNET_INGRESS_FORMAT_DL_MARKER_V2;
+ if (is_dl_mark_v2) {
+ pskb_pull(skb, sizeof(struct rmnet_map_header));
+ qcmd = (struct rmnet_map_control_command_header *)
+ rmnet_map_data_ptr(skb);
+ port->stats.dl_hdr_last_ep_id = qcmd->source_id;
+ port->stats.dl_hdr_last_qmap_vers = qcmd->reserved;
+ port->stats.dl_hdr_last_trans_id = qcmd->transaction_id;
+ pskb_pull(skb, sizeof(struct rmnet_map_control_command_header));
+ } else {
+ pskb_pull(skb, RMNET_MAP_CMD_SIZE);
+ }
+
+ dlhdr = (struct rmnet_map_dl_ind_hdr *)rmnet_map_data_ptr(skb);
+
+ port->stats.dl_hdr_last_seq = dlhdr->le.seq;
+ port->stats.dl_hdr_last_bytes = dlhdr->le.bytes;
+ port->stats.dl_hdr_last_pkts = dlhdr->le.pkts;
+ port->stats.dl_hdr_last_flows = dlhdr->le.flows;
+ port->stats.dl_hdr_total_bytes += port->stats.dl_hdr_last_bytes;
+ port->stats.dl_hdr_total_pkts += port->stats.dl_hdr_last_pkts;
+ port->stats.dl_hdr_count++;
+
+ if (is_dl_mark_v2)
+ rmnet_map_dl_hdr_notify_v2(port, dlhdr, qcmd);
+ else
+ rmnet_map_dl_hdr_notify(port, dlhdr);
+
+ if (rmnet_perf) {
+ unsigned int pull_size;
+
+ pull_size = sizeof(struct rmnet_map_dl_ind_hdr);
+ if (data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4)
+ pull_size += sizeof(struct rmnet_map_dl_csum_trailer);
+ pskb_pull(skb, pull_size);
+ }
+}
+
+static void rmnet_map_process_flow_end(struct sk_buff *skb,
+ struct rmnet_port *port,
+ bool rmnet_perf)
+{
+ struct rmnet_map_dl_ind_trl *dltrl;
+ struct rmnet_map_control_command_header *qcmd;
+ u32 data_format;
+ bool is_dl_mark_v2;
+
+ if (skb->len < RMNET_DL_IND_TRL_SIZE)
+ return;
+
+ data_format = port->data_format;
+ is_dl_mark_v2 = data_format & RMNET_INGRESS_FORMAT_DL_MARKER_V2;
+ if (is_dl_mark_v2) {
+ pskb_pull(skb, sizeof(struct rmnet_map_header));
+ qcmd = (struct rmnet_map_control_command_header *)
+ rmnet_map_data_ptr(skb);
+ pskb_pull(skb, sizeof(struct rmnet_map_control_command_header));
+ } else {
+ pskb_pull(skb, RMNET_MAP_CMD_SIZE);
+ }
+
+ dltrl = (struct rmnet_map_dl_ind_trl *)rmnet_map_data_ptr(skb);
+
+ port->stats.dl_trl_last_seq = dltrl->seq_le;
+ port->stats.dl_trl_count++;
+
+ if (is_dl_mark_v2)
+ rmnet_map_dl_trl_notify_v2(port, dltrl, qcmd);
+ else
+ rmnet_map_dl_trl_notify(port, dltrl);
+
+ if (rmnet_perf) {
+ unsigned int pull_size;
+
+ pull_size = sizeof(struct rmnet_map_dl_ind_trl);
+ if (data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4)
+ pull_size += sizeof(struct rmnet_map_dl_csum_trailer);
+ pskb_pull(skb, pull_size);
+ }
+}
+
/* Process MAP command frame and send N/ACK message as appropriate. Message cmd
* name is decoded here and appropriate handler is called.
*/
@@ -75,7 +262,7 @@ void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port)
unsigned char command_name;
unsigned char rc = 0;
- cmd = RMNET_MAP_GET_CMD_START(skb);
+ cmd = rmnet_map_get_cmd_start(skb);
command_name = cmd->command_name;
switch (command_name) {
@@ -95,3 +282,108 @@ void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port)
if (rc == RMNET_MAP_COMMAND_ACK)
rmnet_map_send_ack(skb, rc, port);
}
+
+int rmnet_map_flow_command(struct sk_buff *skb, struct rmnet_port *port,
+ bool rmnet_perf)
+{
+ struct rmnet_map_control_command *cmd;
+ unsigned char command_name;
+
+ cmd = rmnet_map_get_cmd_start(skb);
+ command_name = cmd->command_name;
+
+ switch (command_name) {
+ case RMNET_MAP_COMMAND_FLOW_START:
+ rmnet_map_process_flow_start(skb, port, rmnet_perf);
+ break;
+
+ case RMNET_MAP_COMMAND_FLOW_END:
+ rmnet_map_process_flow_end(skb, port, rmnet_perf);
+ break;
+
+ default:
+ return 1;
+ }
+
+ /* rmnet_perf module will handle the consuming */
+ if (!rmnet_perf)
+ consume_skb(skb);
+
+ return 0;
+}
+EXPORT_SYMBOL(rmnet_map_flow_command);
+
+void rmnet_map_cmd_exit(struct rmnet_port *port)
+{
+ struct rmnet_map_dl_ind *tmp, *idx;
+
+ list_for_each_entry_safe(tmp, idx, &port->dl_list, list)
+ list_del_rcu(&tmp->list);
+}
+
+void rmnet_map_cmd_init(struct rmnet_port *port)
+{
+ INIT_LIST_HEAD(&port->dl_list);
+
+ port->dl_marker_flush = -1;
+}
+
+int rmnet_map_dl_ind_register(struct rmnet_port *port,
+ struct rmnet_map_dl_ind *dl_ind)
+{
+ struct rmnet_map_dl_ind *dl_ind_iterator;
+ bool empty_ind_list = true;
+
+ if (!port || !dl_ind || !dl_ind->dl_hdr_handler ||
+ !dl_ind->dl_trl_handler)
+ return -EINVAL;
+
+ list_for_each_entry_rcu(dl_ind_iterator, &port->dl_list, list) {
+ empty_ind_list = false;
+ if (dl_ind_iterator->priority < dl_ind->priority) {
+ if (dl_ind_iterator->list.next) {
+ if (dl_ind->priority
+ < list_entry_rcu(dl_ind_iterator->list.next,
+ typeof(*dl_ind_iterator), list)->priority) {
+ list_add_rcu(&dl_ind->list,
+ &dl_ind_iterator->list);
+ break;
+ }
+ } else {
+ list_add_rcu(&dl_ind->list,
+ &dl_ind_iterator->list);
+ break;
+ }
+ } else {
+ list_add_tail_rcu(&dl_ind->list,
+ &dl_ind_iterator->list);
+ break;
+ }
+ }
+
+ if (empty_ind_list)
+ list_add_rcu(&dl_ind->list, &port->dl_list);
+
+ return 0;
+}
+EXPORT_SYMBOL(rmnet_map_dl_ind_register);
+
+int rmnet_map_dl_ind_deregister(struct rmnet_port *port,
+ struct rmnet_map_dl_ind *dl_ind)
+{
+ struct rmnet_map_dl_ind *tmp;
+
+ if (!port || !dl_ind)
+ return -EINVAL;
+
+ list_for_each_entry(tmp, &port->dl_list, list) {
+ if (tmp == dl_ind) {
+ list_del_rcu(&dl_ind->list);
+ goto done;
+ }
+ }
+
+done:
+ return 0;
+}
+EXPORT_SYMBOL(rmnet_map_dl_ind_deregister);
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
index 21d38167f961..f6d566392e62 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
@@ -1,7 +1,17 @@
// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*
* RMNET Data MAP protocol
+ *
*/
#include <linux/netdevice.h>
@@ -11,10 +21,25 @@
#include "rmnet_config.h"
#include "rmnet_map.h"
#include "rmnet_private.h"
+#include "rmnet_handlers.h"
+#define RMNET_MAP_PKT_COPY_THRESHOLD 64
#define RMNET_MAP_DEAGGR_SPACING 64
#define RMNET_MAP_DEAGGR_HEADROOM (RMNET_MAP_DEAGGR_SPACING / 2)
+struct rmnet_map_coal_metadata {
+ void *ip_header;
+ void *trans_header;
+ u16 ip_len;
+ u16 trans_len;
+ u16 data_offset;
+ u16 data_len;
+ u8 ip_proto;
+ u8 trans_proto;
+ u8 pkt_id;
+ u8 pkt_count;
+};
+
static __sum16 *rmnet_map_get_csum_field(unsigned char protocol,
const void *txporthdr)
{
@@ -48,14 +73,14 @@ rmnet_map_ipv4_dl_csum_trailer(struct sk_buff *skb,
void *txporthdr;
__be16 addend;
- ip4h = (struct iphdr *)(skb->data);
+ ip4h = (struct iphdr *)rmnet_map_data_ptr(skb);
if ((ntohs(ip4h->frag_off) & IP_MF) ||
((ntohs(ip4h->frag_off) & IP_OFFSET) > 0)) {
priv->stats.csum_fragmented_pkt++;
return -EOPNOTSUPP;
}
- txporthdr = skb->data + ip4h->ihl * 4;
+ txporthdr = rmnet_map_data_ptr(skb) + ip4h->ihl * 4;
csum_field = rmnet_map_get_csum_field(ip4h->protocol, txporthdr);
@@ -119,12 +144,12 @@ rmnet_map_ipv6_dl_csum_trailer(struct sk_buff *skb,
u16 csum_value, csum_value_final;
__be16 ip6_hdr_csum, addend;
struct ipv6hdr *ip6h;
- void *txporthdr;
+ void *txporthdr, *data = rmnet_map_data_ptr(skb);
u32 length;
- ip6h = (struct ipv6hdr *)(skb->data);
+ ip6h = data;
- txporthdr = skb->data + sizeof(struct ipv6hdr);
+ txporthdr = data + sizeof(struct ipv6hdr);
csum_field = rmnet_map_get_csum_field(ip6h->nexthdr, txporthdr);
if (!csum_field) {
@@ -135,7 +160,7 @@ rmnet_map_ipv6_dl_csum_trailer(struct sk_buff *skb,
csum_value = ~ntohs(csum_trailer->csum_value);
ip6_hdr_csum = (__force __be16)
~ntohs((__force __be16)ip_compute_csum(ip6h,
- (int)(txporthdr - (void *)(skb->data))));
+ (int)(txporthdr - data)));
ip6_payload_csum = csum16_sub((__force __sum16)csum_value,
ip6_hdr_csum);
@@ -268,7 +293,8 @@ rmnet_map_ipv6_ul_csum_header(void *ip6hdr,
* initialized to 0.
*/
struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
- int hdrlen, int pad)
+ int hdrlen, int pad,
+ struct rmnet_port *port)
{
struct rmnet_map_header *map_header;
u32 padding, map_datalen;
@@ -279,6 +305,10 @@ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
skb_push(skb, sizeof(struct rmnet_map_header));
memset(map_header, 0, sizeof(struct rmnet_map_header));
+ /* Set next_hdr bit for csum offload packets */
+ if (port->data_format & RMNET_FLAGS_EGRESS_MAP_CKSUMV5)
+ map_header->next_hdr = 1;
+
if (pad == RMNET_MAP_NO_PAD_BYTES) {
map_header->pkt_len = htons(map_datalen);
return map_header;
@@ -313,16 +343,25 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb,
{
struct rmnet_map_header *maph;
struct sk_buff *skbn;
+ unsigned char *data = rmnet_map_data_ptr(skb), *next_hdr = NULL;
u32 packet_len;
if (skb->len == 0)
return NULL;
- maph = (struct rmnet_map_header *)skb->data;
+ maph = (struct rmnet_map_header *)data;
packet_len = ntohs(maph->pkt_len) + sizeof(struct rmnet_map_header);
if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV4)
packet_len += sizeof(struct rmnet_map_dl_csum_trailer);
+ else if (port->data_format & RMNET_FLAGS_INGRESS_MAP_CKSUMV5) {
+ if (!maph->cd_bit) {
+ packet_len += sizeof(struct rmnet_map_v5_csum_header);
+
+ /* Coalescing headers require MAPv5 */
+ next_hdr = data + sizeof(*maph);
+ }
+ }
if (((int)skb->len - (int)packet_len) < 0)
return NULL;
@@ -331,14 +370,35 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb,
if (ntohs(maph->pkt_len) == 0)
return NULL;
- skbn = alloc_skb(packet_len + RMNET_MAP_DEAGGR_SPACING, GFP_ATOMIC);
- if (!skbn)
- return NULL;
+ if (next_hdr &&
+ ((struct rmnet_map_v5_coal_header *)next_hdr)->header_type ==
+ RMNET_MAP_HEADER_TYPE_COALESCING)
+ return skb;
+
+ if (skb_is_nonlinear(skb)) {
+ skb_frag_t *frag0 = skb_shinfo(skb)->frags;
+ struct page *page = skb_frag_page(frag0);
- skb_reserve(skbn, RMNET_MAP_DEAGGR_HEADROOM);
- skb_put(skbn, packet_len);
- memcpy(skbn->data, skb->data, packet_len);
- skb_pull(skb, packet_len);
+ skbn = alloc_skb(RMNET_MAP_DEAGGR_HEADROOM, GFP_ATOMIC);
+ if (!skbn)
+ return NULL;
+
+ skb_append_pagefrags(skbn, page, frag0->bv_offset,
+ packet_len);
+ skbn->data_len += packet_len;
+ skbn->len += packet_len;
+ } else {
+ skbn = alloc_skb(packet_len + RMNET_MAP_DEAGGR_SPACING,
+ GFP_ATOMIC);
+ if (!skbn)
+ return NULL;
+
+ skb_reserve(skbn, RMNET_MAP_DEAGGR_HEADROOM);
+ skb_put(skbn, packet_len);
+ memcpy(skbn->data, data, packet_len);
+ }
+
+ pskb_pull(skb, packet_len);
return skbn;
}
@@ -359,7 +419,8 @@ int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len)
return -EOPNOTSUPP;
}
- csum_trailer = (struct rmnet_map_dl_csum_trailer *)(skb->data + len);
+ csum_trailer = (struct rmnet_map_dl_csum_trailer *)
+ (rmnet_map_data_ptr(skb) + len);
if (!csum_trailer->valid) {
priv->stats.csum_valid_unset++;
@@ -382,12 +443,10 @@ int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len)
return 0;
}
+EXPORT_SYMBOL(rmnet_map_checksum_downlink_packet);
-/* Generates UL checksum meta info header for IPv4 and IPv6 over TCP and UDP
- * packets that are supported for UL checksum offload.
- */
-void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
- struct net_device *orig_dev)
+void rmnet_map_v4_checksum_uplink_packet(struct sk_buff *skb,
+ struct net_device *orig_dev)
{
struct rmnet_priv *priv = netdev_priv(orig_dev);
struct rmnet_map_ul_csum_header *ul_header;
@@ -406,10 +465,12 @@ void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
if (skb->protocol == htons(ETH_P_IP)) {
rmnet_map_ipv4_ul_csum_header(iphdr, ul_header, skb);
+ priv->stats.csum_hw++;
return;
} else if (skb->protocol == htons(ETH_P_IPV6)) {
#if IS_ENABLED(CONFIG_IPV6)
rmnet_map_ipv6_ul_csum_header(iphdr, ul_header, skb);
+ priv->stats.csum_hw++;
return;
#else
priv->stats.csum_err_invalid_ip_version++;
@@ -428,3 +489,927 @@ void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
priv->stats.csum_sw++;
}
+
+void rmnet_map_v5_checksum_uplink_packet(struct sk_buff *skb,
+ struct net_device *orig_dev)
+{
+ struct rmnet_priv *priv = netdev_priv(orig_dev);
+ struct rmnet_map_v5_csum_header *ul_header;
+
+ ul_header = (struct rmnet_map_v5_csum_header *)
+ skb_push(skb, sizeof(*ul_header));
+ memset(ul_header, 0, sizeof(*ul_header));
+ ul_header->header_type = RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD;
+
+ if (skb->ip_summed == CHECKSUM_PARTIAL) {
+ void *iph = (char *)ul_header + sizeof(*ul_header);
+ void *trans;
+ __sum16 *check;
+ u8 proto;
+
+ if (skb->protocol == htons(ETH_P_IP)) {
+ u16 ip_len = ((struct iphdr *)iph)->ihl * 4;
+
+ proto = ((struct iphdr *)iph)->protocol;
+ trans = iph + ip_len;
+ } else if (skb->protocol == htons(ETH_P_IPV6)) {
+ u16 ip_len = sizeof(struct ipv6hdr);
+
+ proto = ((struct ipv6hdr *)iph)->nexthdr;
+ trans = iph + ip_len;
+ } else {
+ priv->stats.csum_err_invalid_ip_version++;
+ goto sw_csum;
+ }
+
+ check = rmnet_map_get_csum_field(proto, trans);
+ if (check) {
+ *check = 0;
+ skb->ip_summed = CHECKSUM_NONE;
+ /* Ask for checksum offloading */
+ ul_header->csum_valid_required = 1;
+ priv->stats.csum_hw++;
+ return;
+ }
+ }
+
+sw_csum:
+ priv->stats.csum_sw++;
+}
+
+/* Generates UL checksum meta info header for IPv4 and IPv6 over TCP and UDP
+ * packets that are supported for UL checksum offload.
+ */
+void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
+ struct net_device *orig_dev,
+ int csum_type)
+{
+ switch (csum_type) {
+ case RMNET_FLAGS_EGRESS_MAP_CKSUMV4:
+ rmnet_map_v4_checksum_uplink_packet(skb, orig_dev);
+ break;
+ case RMNET_FLAGS_EGRESS_MAP_CKSUMV5:
+ rmnet_map_v5_checksum_uplink_packet(skb, orig_dev);
+ break;
+ default:
+ break;
+ }
+}
+
+bool rmnet_map_v5_csum_buggy(struct rmnet_map_v5_coal_header *coal_hdr)
+{
+ /* Only applies to frames with a single packet */
+ if (coal_hdr->num_nlos != 1 || coal_hdr->nl_pairs[0].num_packets != 1)
+ return false;
+
+ /* TCP header has FIN or PUSH set */
+ if (coal_hdr->close_type == RMNET_MAP_COAL_CLOSE_COAL)
+ return true;
+
+ /* Hit packet limit, byte limit, or time limit/EOF on DMA */
+ if (coal_hdr->close_type == RMNET_MAP_COAL_CLOSE_HW) {
+ switch (coal_hdr->close_value) {
+ case RMNET_MAP_COAL_CLOSE_HW_PKT:
+ case RMNET_MAP_COAL_CLOSE_HW_BYTE:
+ case RMNET_MAP_COAL_CLOSE_HW_TIME:
+ return true;
+ }
+ }
+
+ return false;
+}
+
+static void rmnet_map_move_headers(struct sk_buff *skb)
+{
+ struct iphdr *iph;
+ u16 ip_len;
+ u16 trans_len = 0;
+ u8 proto;
+
+ /* This only applies to non-linear SKBs */
+ if (!skb_is_nonlinear(skb))
+ return;
+
+ iph = (struct iphdr *)rmnet_map_data_ptr(skb);
+ if (iph->version == 4) {
+ ip_len = iph->ihl * 4;
+ proto = iph->protocol;
+ if (iph->frag_off & htons(IP_OFFSET))
+ /* No transport header information */
+ goto pull;
+ } else if (iph->version == 6) {
+ struct ipv6hdr *ip6h = (struct ipv6hdr *)iph;
+ __be16 frag_off;
+ u8 nexthdr = ip6h->nexthdr;
+
+ ip_len = ipv6_skip_exthdr(skb, sizeof(*ip6h), &nexthdr,
+ &frag_off);
+ if (ip_len < 0)
+ return;
+
+ proto = nexthdr;
+ } else {
+ return;
+ }
+
+ if (proto == IPPROTO_TCP) {
+ struct tcphdr *tp = (struct tcphdr *)((u8 *)iph + ip_len);
+
+ trans_len = tp->doff * 4;
+ } else if (proto == IPPROTO_UDP) {
+ trans_len = sizeof(struct udphdr);
+ } else if (proto == NEXTHDR_FRAGMENT) {
+ /* Non-first fragments don't have the fragment length added by
+ * ipv6_skip_exthdr() and sho up as proto NEXTHDR_FRAGMENT, so
+ * we account for the length here.
+ */
+ ip_len += sizeof(struct frag_hdr);
+ }
+
+pull:
+ __pskb_pull_tail(skb, ip_len + trans_len);
+ skb_reset_network_header(skb);
+ if (trans_len)
+ skb_set_transport_header(skb, ip_len);
+}
+
+static void rmnet_map_nonlinear_copy(struct sk_buff *coal_skb,
+ struct rmnet_map_coal_metadata *coal_meta,
+ struct sk_buff *dest)
+{
+ unsigned char *data_start = rmnet_map_data_ptr(coal_skb) +
+ coal_meta->ip_len + coal_meta->trans_len;
+ u32 copy_len = coal_meta->data_len * coal_meta->pkt_count;
+
+ if (skb_is_nonlinear(coal_skb)) {
+ skb_frag_t *frag0 = skb_shinfo(coal_skb)->frags;
+ struct page *page = skb_frag_page(frag0);
+
+ skb_append_pagefrags(dest, page,
+ frag0->bv_offset + coal_meta->ip_len +
+ coal_meta->trans_len +
+ coal_meta->data_offset,
+ copy_len);
+ dest->data_len += copy_len;
+ dest->len += copy_len;
+ } else {
+ skb_put_data(dest, data_start + coal_meta->data_offset,
+ copy_len);
+ }
+}
+
+/* Fill in GSO metadata to allow the SKB to be segmented by the NW stack
+ * if needed (i.e. forwarding, UDP GRO)
+ */
+static void rmnet_map_gso_stamp(struct sk_buff *skb,
+ struct rmnet_map_coal_metadata *coal_meta)
+{
+ struct skb_shared_info *shinfo = skb_shinfo(skb);
+
+ if (coal_meta->trans_proto == IPPROTO_TCP)
+ shinfo->gso_type = (coal_meta->ip_proto == 4) ?
+ SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
+ else
+ shinfo->gso_type = SKB_GSO_UDP_L4;
+
+ shinfo->gso_size = coal_meta->data_len;
+ shinfo->gso_segs = coal_meta->pkt_count;
+}
+
+/* Handles setting up the partial checksum in the skb. Sets the transport
+ * checksum to the pseudoheader checksum and sets the csum offload metadata
+ */
+static void rmnet_map_partial_csum(struct sk_buff *skb,
+ struct rmnet_map_coal_metadata *coal_meta)
+{
+ unsigned char *data = skb->data;
+ __sum16 pseudo;
+ u16 pkt_len = skb->len - coal_meta->ip_len;
+
+ if (coal_meta->ip_proto == 4) {
+ struct iphdr *iph = (struct iphdr *)data;
+
+ pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
+ pkt_len, coal_meta->trans_proto,
+ 0);
+ } else {
+ struct ipv6hdr *ip6h = (struct ipv6hdr *)data;
+
+ pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
+ pkt_len, coal_meta->trans_proto, 0);
+ }
+
+ if (coal_meta->trans_proto == IPPROTO_TCP) {
+ struct tcphdr *tp = (struct tcphdr *)(data + coal_meta->ip_len);
+
+ tp->check = pseudo;
+ skb->csum_offset = offsetof(struct tcphdr, check);
+ } else {
+ struct udphdr *up = (struct udphdr *)(data + coal_meta->ip_len);
+
+ up->check = pseudo;
+ skb->csum_offset = offsetof(struct udphdr, check);
+ }
+
+ skb->ip_summed = CHECKSUM_PARTIAL;
+ skb->csum_start = skb->data + coal_meta->ip_len - skb->head;
+}
+
+static void
+__rmnet_map_segment_coal_skb(struct sk_buff *coal_skb,
+ struct rmnet_map_coal_metadata *coal_meta,
+ struct sk_buff_head *list, u8 pkt_id,
+ bool csum_valid)
+{
+ struct sk_buff *skbn;
+ struct rmnet_priv *priv = netdev_priv(coal_skb->dev);
+ __sum16 *check = NULL;
+ u32 alloc_len;
+
+ /* We can avoid copying the data if the SKB we got from the lower-level
+ * drivers was nonlinear.
+ */
+ if (skb_is_nonlinear(coal_skb))
+ alloc_len = coal_meta->ip_len + coal_meta->trans_len;
+ else
+ alloc_len = coal_meta->ip_len + coal_meta->trans_len +
+ (coal_meta->data_len * coal_meta->pkt_count);
+
+ skbn = alloc_skb(alloc_len, GFP_ATOMIC);
+ if (!skbn)
+ return;
+
+ skb_reserve(skbn, coal_meta->ip_len + coal_meta->trans_len);
+ rmnet_map_nonlinear_copy(coal_skb, coal_meta, skbn);
+
+ /* Push transport header and update necessary fields */
+ skb_push(skbn, coal_meta->trans_len);
+ memcpy(skbn->data, coal_meta->trans_header, coal_meta->trans_len);
+ skb_reset_transport_header(skbn);
+ if (coal_meta->trans_proto == IPPROTO_TCP) {
+ struct tcphdr *th = tcp_hdr(skbn);
+
+ th->seq = htonl(ntohl(th->seq) + coal_meta->data_offset);
+ check = &th->check;
+ } else if (coal_meta->trans_proto == IPPROTO_UDP) {
+ struct udphdr *uh = udp_hdr(skbn);
+
+ uh->len = htons(skbn->len);
+ check = &uh->check;
+ }
+
+ /* Push IP header and update necessary fields */
+ skb_push(skbn, coal_meta->ip_len);
+ memcpy(skbn->data, coal_meta->ip_header, coal_meta->ip_len);
+ skb_reset_network_header(skbn);
+ if (coal_meta->ip_proto == 4) {
+ struct iphdr *iph = ip_hdr(skbn);
+
+ iph->id = htons(ntohs(iph->id) + coal_meta->pkt_id);
+ iph->tot_len = htons(skbn->len);
+ iph->check = 0;
+ iph->check = ip_fast_csum(iph, iph->ihl);
+ } else {
+ /* Payload length includes any extension headers */
+ ipv6_hdr(skbn)->payload_len = htons(skbn->len -
+ sizeof(struct ipv6hdr));
+ }
+
+ /* Handle checksum status */
+ if (likely(csum_valid)) {
+ /* Set the partial checksum information */
+ rmnet_map_partial_csum(skbn, coal_meta);
+ } else if (check) {
+ /* Unfortunately, we have to fake a bad checksum here, since
+ * the original bad value is lost by the hardware. The only
+ * reliable way to do it is to calculate the actual checksum
+ * and corrupt it.
+ */
+ __wsum csum;
+ unsigned int offset = skb_transport_offset(skbn);
+ __sum16 pseudo;
+
+ /* Calculate pseudo header */
+ if (coal_meta->ip_proto == 4) {
+ struct iphdr *iph = ip_hdr(skbn);
+
+ pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
+ skbn->len -
+ coal_meta->ip_len,
+ coal_meta->trans_proto, 0);
+ } else {
+ struct ipv6hdr *ip6h = ipv6_hdr(skbn);
+
+ pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
+ skbn->len - coal_meta->ip_len,
+ coal_meta->trans_proto, 0);
+ }
+
+ *check = pseudo;
+ csum = skb_checksum(skbn, offset, skbn->len - offset, 0);
+ /* Add 1 to corrupt. This cannot produce a final value of 0
+ * since csum_fold() can't return a value of 0xFFFF.
+ */
+ *check = csum16_add(csum_fold(csum), htons(1));
+ skbn->ip_summed = CHECKSUM_NONE;
+ }
+
+ skbn->dev = coal_skb->dev;
+ priv->stats.coal.coal_reconstruct++;
+
+ /* Stamp GSO information if necessary */
+ if (coal_meta->pkt_count > 1)
+ rmnet_map_gso_stamp(skbn, coal_meta);
+
+ __skb_queue_tail(list, skbn);
+
+ /* Update meta information to move past the data we just segmented */
+ coal_meta->data_offset += coal_meta->data_len * coal_meta->pkt_count;
+ coal_meta->pkt_id = pkt_id + 1;
+ coal_meta->pkt_count = 0;
+}
+
+static bool rmnet_map_validate_csum(struct sk_buff *skb,
+ struct rmnet_map_coal_metadata *meta)
+{
+ u8 *data = rmnet_map_data_ptr(skb);
+ unsigned int datagram_len;
+ __wsum csum;
+ __sum16 pseudo;
+
+ datagram_len = skb->len - meta->ip_len;
+ if (meta->ip_proto == 4) {
+ struct iphdr *iph = (struct iphdr *)data;
+
+ pseudo = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
+ datagram_len,
+ meta->trans_proto, 0);
+ } else {
+ struct ipv6hdr *ip6h = (struct ipv6hdr *)data;
+
+ pseudo = ~csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
+ datagram_len, meta->trans_proto,
+ 0);
+ }
+
+ csum = skb_checksum(skb, meta->ip_len, datagram_len,
+ csum_unfold(pseudo));
+ return !csum_fold(csum);
+}
+
+/* Converts the coalesced SKB into a list of SKBs.
+ * NLOs containing csum erros will not be included.
+ * The original coalesced SKB should be treated as invalid and
+ * must be freed by the caller
+ */
+static void rmnet_map_segment_coal_skb(struct sk_buff *coal_skb,
+ u64 nlo_err_mask,
+ struct sk_buff_head *list)
+{
+ struct iphdr *iph;
+ struct rmnet_priv *priv = netdev_priv(coal_skb->dev);
+ struct rmnet_map_v5_coal_header *coal_hdr;
+ struct rmnet_map_coal_metadata coal_meta;
+ u16 pkt_len;
+ u8 pkt, total_pkt = 0;
+ u8 nlo;
+ bool gro = coal_skb->dev->features & NETIF_F_GRO_HW;
+
+ memset(&coal_meta, 0, sizeof(coal_meta));
+
+ /* Pull off the headers we no longer need */
+ pskb_pull(coal_skb, sizeof(struct rmnet_map_header));
+ coal_hdr = (struct rmnet_map_v5_coal_header *)
+ rmnet_map_data_ptr(coal_skb);
+ pskb_pull(coal_skb, sizeof(*coal_hdr));
+
+ iph = (struct iphdr *)rmnet_map_data_ptr(coal_skb);
+
+ if (iph->version == 4) {
+ coal_meta.ip_proto = 4;
+ coal_meta.ip_len = iph->ihl * 4;
+ coal_meta.trans_proto = iph->protocol;
+ coal_meta.ip_header = iph;
+
+ /* Don't allow coalescing of any packets with IP options */
+ if (iph->ihl != 5)
+ gro = false;
+ } else if (iph->version == 6) {
+ struct ipv6hdr *ip6h = (struct ipv6hdr *)iph;
+ __be16 frag_off;
+ u8 protocol = ip6h->nexthdr;
+
+ coal_meta.ip_proto = 6;
+ coal_meta.ip_len = ipv6_skip_exthdr(coal_skb, sizeof(*ip6h),
+ &protocol, &frag_off);
+ coal_meta.trans_proto = protocol;
+ coal_meta.ip_header = ip6h;
+
+ /* If we run into a problem, or this has a fragment header
+ * (which should technically not be possible, if the HW
+ * works as intended...), bail.
+ */
+ if (coal_meta.ip_len < 0 || frag_off) {
+ priv->stats.coal.coal_ip_invalid++;
+ return;
+ } else if (coal_meta.ip_len > sizeof(*ip6h)) {
+ /* Don't allow coalescing of any packets with IPv6
+ * extension headers.
+ */
+ gro = false;
+ }
+ } else {
+ priv->stats.coal.coal_ip_invalid++;
+ return;
+ }
+
+ if (coal_meta.trans_proto == IPPROTO_TCP) {
+ struct tcphdr *th;
+
+ th = (struct tcphdr *)((u8 *)iph + coal_meta.ip_len);
+ coal_meta.trans_len = th->doff * 4;
+ coal_meta.trans_header = th;
+ } else if (coal_meta.trans_proto == IPPROTO_UDP) {
+ struct udphdr *uh;
+
+ uh = (struct udphdr *)((u8 *)iph + coal_meta.ip_len);
+ coal_meta.trans_len = sizeof(*uh);
+ coal_meta.trans_header = uh;
+ } else {
+ priv->stats.coal.coal_trans_invalid++;
+ return;
+ }
+
+ if (rmnet_map_v5_csum_buggy(coal_hdr)) {
+ rmnet_map_move_headers(coal_skb);
+ /* Mark as valid if it checks out */
+ if (rmnet_map_validate_csum(coal_skb, &coal_meta))
+ coal_skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+ __skb_queue_tail(list, coal_skb);
+ return;
+ }
+
+ /* Fast-forward the case where we have 1 NLO (i.e. 1 packet length),
+ * no checksum errors, and are allowing GRO. We can just reuse this
+ * SKB unchanged.
+ */
+ if (gro && coal_hdr->num_nlos == 1 && coal_hdr->csum_valid) {
+ rmnet_map_move_headers(coal_skb);
+ coal_skb->ip_summed = CHECKSUM_UNNECESSARY;
+ coal_meta.data_len = ntohs(coal_hdr->nl_pairs[0].pkt_len);
+ coal_meta.data_len -= coal_meta.ip_len + coal_meta.trans_len;
+ coal_meta.pkt_count = coal_hdr->nl_pairs[0].num_packets;
+ if (coal_meta.pkt_count > 1) {
+ rmnet_map_partial_csum(coal_skb, &coal_meta);
+ rmnet_map_gso_stamp(coal_skb, &coal_meta);
+ }
+
+ __skb_queue_tail(list, coal_skb);
+ return;
+ }
+
+ /* Segment the coalesced SKB into new packets */
+ for (nlo = 0; nlo < coal_hdr->num_nlos; nlo++) {
+ pkt_len = ntohs(coal_hdr->nl_pairs[nlo].pkt_len);
+ pkt_len -= coal_meta.ip_len + coal_meta.trans_len;
+ coal_meta.data_len = pkt_len;
+ for (pkt = 0; pkt < coal_hdr->nl_pairs[nlo].num_packets;
+ pkt++, total_pkt++, nlo_err_mask >>= 1) {
+ bool csum_err = nlo_err_mask & 1;
+
+ /* Segment the packet if we're not sending the larger
+ * packet up the stack.
+ */
+ if (!gro) {
+ coal_meta.pkt_count = 1;
+ if (csum_err)
+ priv->stats.coal.coal_csum_err++;
+
+ __rmnet_map_segment_coal_skb(coal_skb,
+ &coal_meta, list,
+ total_pkt,
+ !csum_err);
+ continue;
+ }
+
+ if (csum_err) {
+ priv->stats.coal.coal_csum_err++;
+
+ /* Segment out the good data */
+ if (gro && coal_meta.pkt_count)
+ __rmnet_map_segment_coal_skb(coal_skb,
+ &coal_meta,
+ list,
+ total_pkt,
+ true);
+
+ /* Segment out the bad checksum */
+ coal_meta.pkt_count = 1;
+ __rmnet_map_segment_coal_skb(coal_skb,
+ &coal_meta, list,
+ total_pkt, false);
+ } else {
+ coal_meta.pkt_count++;
+ }
+ }
+
+ /* If we're switching NLOs, we need to send out everything from
+ * the previous one, if we haven't done so. NLOs only switch
+ * when the packet length changes.
+ */
+ if (coal_meta.pkt_count)
+ __rmnet_map_segment_coal_skb(coal_skb, &coal_meta, list,
+ total_pkt, true);
+ }
+}
+
+/* Record reason for coalescing pipe closure */
+static void rmnet_map_data_log_close_stats(struct rmnet_priv *priv, u8 type,
+ u8 code)
+{
+ struct rmnet_coal_close_stats *stats = &priv->stats.coal.close;
+
+ switch (type) {
+ case RMNET_MAP_COAL_CLOSE_NON_COAL:
+ stats->non_coal++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_IP_MISS:
+ stats->ip_miss++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_TRANS_MISS:
+ stats->trans_miss++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_HW:
+ switch (code) {
+ case RMNET_MAP_COAL_CLOSE_HW_NL:
+ stats->hw_nl++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_HW_PKT:
+ stats->hw_pkt++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_HW_BYTE:
+ stats->hw_byte++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_HW_TIME:
+ stats->hw_time++;
+ break;
+ case RMNET_MAP_COAL_CLOSE_HW_EVICT:
+ stats->hw_evict++;
+ break;
+ default:
+ break;
+ }
+ break;
+ case RMNET_MAP_COAL_CLOSE_COAL:
+ stats->coal++;
+ break;
+ default:
+ break;
+ }
+}
+
+/* Check if the coalesced header has any incorrect values, in which case, the
+ * entire coalesced skb must be dropped. Then check if there are any
+ * checksum issues
+ */
+static int rmnet_map_data_check_coal_header(struct sk_buff *skb,
+ u64 *nlo_err_mask)
+{
+ struct rmnet_map_v5_coal_header *coal_hdr;
+ unsigned char *data = rmnet_map_data_ptr(skb);
+ struct rmnet_priv *priv = netdev_priv(skb->dev);
+ u64 mask = 0;
+ int i;
+ u8 veid, pkts = 0;
+
+ coal_hdr = ((struct rmnet_map_v5_coal_header *)
+ (data + sizeof(struct rmnet_map_header)));
+ veid = coal_hdr->virtual_channel_id;
+
+ if (coal_hdr->num_nlos == 0 ||
+ coal_hdr->num_nlos > RMNET_MAP_V5_MAX_NLOS) {
+ priv->stats.coal.coal_hdr_nlo_err++;
+ return -EINVAL;
+ }
+
+ for (i = 0; i < RMNET_MAP_V5_MAX_NLOS; i++) {
+ /* If there is a checksum issue, we need to split
+ * up the skb. Rebuild the full csum error field
+ */
+ u8 err = coal_hdr->nl_pairs[i].csum_error_bitmap;
+ u8 pkt = coal_hdr->nl_pairs[i].num_packets;
+
+ mask |= ((u64)err) << (8 * i);
+
+ /* Track total packets in frame */
+ pkts += pkt;
+ if (pkts > RMNET_MAP_V5_MAX_PACKETS) {
+ priv->stats.coal.coal_hdr_pkt_err++;
+ return -EINVAL;
+ }
+ }
+
+ /* Track number of packets we get inside of coalesced frames */
+ priv->stats.coal.coal_pkts += pkts;
+
+ /* Update ethtool stats */
+ rmnet_map_data_log_close_stats(priv,
+ coal_hdr->close_type,
+ coal_hdr->close_value);
+ if (veid < RMNET_MAX_VEID)
+ priv->stats.coal.coal_veid[veid]++;
+
+ *nlo_err_mask = mask;
+
+ return 0;
+}
+
+/* Process a QMAPv5 packet header */
+int rmnet_map_process_next_hdr_packet(struct sk_buff *skb,
+ struct sk_buff_head *list,
+ u16 len)
+{
+ struct rmnet_priv *priv = netdev_priv(skb->dev);
+ u64 nlo_err_mask;
+ int rc = 0;
+
+ switch (rmnet_map_get_next_hdr_type(skb)) {
+ case RMNET_MAP_HEADER_TYPE_COALESCING:
+ priv->stats.coal.coal_rx++;
+ rc = rmnet_map_data_check_coal_header(skb, &nlo_err_mask);
+ if (rc)
+ return rc;
+
+ rmnet_map_segment_coal_skb(skb, nlo_err_mask, list);
+ if (skb_peek(list) != skb)
+ consume_skb(skb);
+ break;
+ case RMNET_MAP_HEADER_TYPE_CSUM_OFFLOAD:
+ if (rmnet_map_get_csum_valid(skb)) {
+ priv->stats.csum_ok++;
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ } else {
+ priv->stats.csum_valid_unset++;
+ }
+
+ /* Pull unnecessary headers and move the rest to the linear
+ * section of the skb.
+ */
+ pskb_pull(skb,
+ (sizeof(struct rmnet_map_header) +
+ sizeof(struct rmnet_map_v5_csum_header)));
+ rmnet_map_move_headers(skb);
+
+ /* Remove padding only for csum offload packets.
+ * Coalesced packets should never have padding.
+ */
+ pskb_trim(skb, len);
+ __skb_queue_tail(list, skb);
+ break;
+ default:
+ rc = -EINVAL;
+ break;
+ }
+
+ return rc;
+}
+
+long rmnet_agg_time_limit __read_mostly = 1000000L;
+long rmnet_agg_bypass_time __read_mostly = 10000000L;
+
+int rmnet_map_tx_agg_skip(struct sk_buff *skb, int offset)
+{
+ u8 *packet_start = skb->data + offset;
+ int is_icmp = 0;
+
+ if (skb->protocol == htons(ETH_P_IP)) {
+ struct iphdr *ip4h = (struct iphdr *)(packet_start);
+
+ if (ip4h->protocol == IPPROTO_ICMP)
+ is_icmp = 1;
+ } else if (skb->protocol == htons(ETH_P_IPV6)) {
+ struct ipv6hdr *ip6h = (struct ipv6hdr *)(packet_start);
+
+ if (ip6h->nexthdr == IPPROTO_ICMPV6) {
+ is_icmp = 1;
+ } else if (ip6h->nexthdr == NEXTHDR_FRAGMENT) {
+ struct frag_hdr *frag;
+
+ frag = (struct frag_hdr *)(packet_start
+ + sizeof(struct ipv6hdr));
+ if (frag->nexthdr == IPPROTO_ICMPV6)
+ is_icmp = 1;
+ }
+ }
+
+ return is_icmp;
+}
+
+static void rmnet_map_flush_tx_packet_work(struct work_struct *work)
+{
+ struct sk_buff *skb = NULL;
+ struct rmnet_port *port;
+ unsigned long flags;
+
+ port = container_of(work, struct rmnet_port, agg_wq);
+
+ spin_lock_irqsave(&port->agg_lock, flags);
+ if (likely(port->agg_state == -EINPROGRESS)) {
+ /* Buffer may have already been shipped out */
+ if (likely(port->agg_skb)) {
+ skb = port->agg_skb;
+ port->agg_skb = NULL;
+ port->agg_count = 0;
+ memset(&port->agg_time, 0, sizeof(struct timespec));
+ }
+ port->agg_state = 0;
+ }
+
+ spin_unlock_irqrestore(&port->agg_lock, flags);
+ if (skb)
+ dev_queue_xmit(skb);
+}
+
+enum hrtimer_restart rmnet_map_flush_tx_packet_queue(struct hrtimer *t)
+{
+ struct rmnet_port *port;
+
+ port = container_of(t, struct rmnet_port, hrtimer);
+
+ schedule_work(&port->agg_wq);
+ return HRTIMER_NORESTART;
+}
+
+static void rmnet_map_linearize_copy(struct sk_buff *dst, struct sk_buff *src)
+{
+ unsigned int linear = src->len - src->data_len, target = src->len;
+ unsigned char *src_buf;
+ struct sk_buff *skb;
+
+ src_buf = src->data;
+ skb_put_data(dst, src_buf, linear);
+ target -= linear;
+
+ skb = src;
+
+ while (target) {
+ unsigned int i = 0, non_linear = 0;
+
+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+ non_linear = skb_frag_size(&skb_shinfo(skb)->frags[i]);
+ src_buf = skb_frag_address(&skb_shinfo(skb)->frags[i]);
+
+ skb_put_data(dst, src_buf, non_linear);
+ target -= non_linear;
+ }
+
+ if (skb_shinfo(skb)->frag_list) {
+ skb = skb_shinfo(skb)->frag_list;
+ continue;
+ }
+
+ if (skb->next)
+ skb = skb->next;
+ }
+}
+
+void rmnet_map_tx_aggregate(struct sk_buff *skb, struct rmnet_port *port)
+{
+ struct timespec64 diff, last;
+ int size, agg_count = 0;
+ struct sk_buff *agg_skb;
+ unsigned long flags;
+
+new_packet:
+ spin_lock_irqsave(&port->agg_lock, flags);
+ memcpy(&last, &port->agg_last, sizeof(struct timespec));
+ ktime_get_real_ts64(&port->agg_last);
+
+ if (!port->agg_skb) {
+ /* Check to see if we should agg first. If the traffic is very
+ * sparse, don't aggregate. We will need to tune this later
+ */
+ diff = timespec64_sub(port->agg_last, last);
+ size = port->egress_agg_params.agg_size - skb->len;
+
+ if (diff.tv_sec > 0 || diff.tv_nsec > rmnet_agg_bypass_time ||
+ size <= 0) {
+ spin_unlock_irqrestore(&port->agg_lock, flags);
+ skb->protocol = htons(ETH_P_MAP);
+ dev_queue_xmit(skb);
+ return;
+ }
+
+ port->agg_skb = alloc_skb(port->egress_agg_params.agg_size,
+ GFP_ATOMIC);
+ if (!port->agg_skb) {
+ port->agg_skb = 0;
+ port->agg_count = 0;
+ memset(&port->agg_time, 0, sizeof(struct timespec));
+ spin_unlock_irqrestore(&port->agg_lock, flags);
+ skb->protocol = htons(ETH_P_MAP);
+ dev_queue_xmit(skb);
+ return;
+ }
+ rmnet_map_linearize_copy(port->agg_skb, skb);
+ port->agg_skb->dev = skb->dev;
+ port->agg_skb->protocol = htons(ETH_P_MAP);
+ port->agg_count = 1;
+ ktime_get_real_ts64(&port->agg_time);
+ dev_kfree_skb_any(skb);
+ goto schedule;
+ }
+ diff = timespec64_sub(port->agg_last, port->agg_time);
+ size = port->egress_agg_params.agg_size - port->agg_skb->len;
+
+ if (skb->len > size ||
+ port->agg_count >= port->egress_agg_params.agg_count ||
+ diff.tv_sec > 0 || diff.tv_nsec > rmnet_agg_time_limit) {
+ agg_skb = port->agg_skb;
+ agg_count = port->agg_count;
+ port->agg_skb = 0;
+ port->agg_count = 0;
+ memset(&port->agg_time, 0, sizeof(struct timespec));
+ port->agg_state = 0;
+ spin_unlock_irqrestore(&port->agg_lock, flags);
+ hrtimer_cancel(&port->hrtimer);
+ dev_queue_xmit(agg_skb);
+ goto new_packet;
+ }
+
+ rmnet_map_linearize_copy(port->agg_skb, skb);
+ port->agg_count++;
+ dev_kfree_skb_any(skb);
+
+schedule:
+ if (port->agg_state != -EINPROGRESS) {
+ port->agg_state = -EINPROGRESS;
+ hrtimer_start(&port->hrtimer,
+ ns_to_ktime(port->egress_agg_params.agg_time),
+ HRTIMER_MODE_REL);
+ }
+ spin_unlock_irqrestore(&port->agg_lock, flags);
+}
+
+void rmnet_map_tx_aggregate_init(struct rmnet_port *port)
+{
+ hrtimer_init(&port->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ port->hrtimer.function = rmnet_map_flush_tx_packet_queue;
+ port->egress_agg_params.agg_size = 8192;
+ port->egress_agg_params.agg_count = 20;
+ port->egress_agg_params.agg_time = 3000000;
+ spin_lock_init(&port->agg_lock);
+
+ INIT_WORK(&port->agg_wq, rmnet_map_flush_tx_packet_work);
+}
+
+void rmnet_map_tx_aggregate_exit(struct rmnet_port *port)
+{
+ unsigned long flags;
+
+ hrtimer_cancel(&port->hrtimer);
+ cancel_work_sync(&port->agg_wq);
+
+ spin_lock_irqsave(&port->agg_lock, flags);
+ if (port->agg_state == -EINPROGRESS) {
+ if (port->agg_skb) {
+ kfree_skb(port->agg_skb);
+ port->agg_skb = NULL;
+ port->agg_count = 0;
+ memset(&port->agg_time, 0, sizeof(struct timespec));
+ }
+
+ port->agg_state = 0;
+ }
+
+ spin_unlock_irqrestore(&port->agg_lock, flags);
+}
+
+void rmnet_map_tx_qmap_cmd(struct sk_buff *qmap_skb)
+{
+ struct rmnet_port *port;
+ struct sk_buff *agg_skb;
+ unsigned long flags;
+
+ port = rmnet_get_port(qmap_skb->dev);
+
+ if (port->data_format & RMNET_EGRESS_FORMAT_AGGREGATION) {
+ spin_lock_irqsave(&port->agg_lock, flags);
+ if (port->agg_skb) {
+ agg_skb = port->agg_skb;
+ port->agg_skb = 0;
+ port->agg_count = 0;
+ memset(&port->agg_time, 0, sizeof(struct timespec));
+ port->agg_state = 0;
+ spin_unlock_irqrestore(&port->agg_lock, flags);
+ hrtimer_cancel(&port->hrtimer);
+ dev_queue_xmit(agg_skb);
+ } else {
+ spin_unlock_irqrestore(&port->agg_lock, flags);
+ }
+ }
+
+ dev_queue_xmit(qmap_skb);
+}
+EXPORT_SYMBOL(rmnet_map_tx_qmap_cmd);
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h
index e1337f164faa..b31d3a889720 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h
@@ -1,5 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (c) 2013-2014, 2016-2018 The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2014, 2016-2019 The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*/
#ifndef _RMNET_PRIVATE_H_
@@ -10,6 +19,14 @@
#define RMNET_NEEDED_HEADROOM 16
#define RMNET_TX_QUEUE_LEN 1000
+/* Constants */
+#define RMNET_EGRESS_FORMAT_AGGREGATION BIT(31)
+#define RMNET_INGRESS_FORMAT_DL_MARKER_V1 BIT(30)
+#define RMNET_INGRESS_FORMAT_DL_MARKER_V2 BIT(29)
+
+#define RMNET_INGRESS_FORMAT_DL_MARKER (RMNET_INGRESS_FORMAT_DL_MARKER_V1 |\
+RMNET_INGRESS_FORMAT_DL_MARKER_V2)
+
/* Replace skb->dev to a virtual rmnet device and pass up the stack */
#define RMNET_EPMODE_VND (1)
/* Pass the frame directly to another device with dev_queue_xmit() */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_trace.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_trace.h
new file mode 100644
index 000000000000..8ef2bbfc0d9c
--- /dev/null
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_trace.h
@@ -0,0 +1,250 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rmnet
+#define TRACE_INCLUDE_FILE rmnet_trace
+
+#if !defined(_RMNET_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
+#define _RMNET_TRACE_H_
+
+#include <linux/netdevice.h>
+#include <linux/skbuff.h>
+#include <linux/tracepoint.h>
+
+/*****************************************************************************/
+/* Trace events for rmnet module */
+/*****************************************************************************/
+DECLARE_EVENT_CLASS
+ (rmnet_mod_template,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2),
+
+ TP_STRUCT__entry(__field(u8, func)
+ __field(u8, evt)
+ __field(u32, uint1)
+ __field(u32, uint2)
+ __field(u64, ulong1)
+ __field(u64, ulong2)
+ __field(void *, ptr1)
+ __field(void *, ptr2)),
+
+ TP_fast_assign(__entry->func = func;
+ __entry->evt = evt;
+ __entry->uint1 = uint1;
+ __entry->uint2 = uint2;
+ __entry->ulong1 = ulong1;
+ __entry->ulong2 = ulong2;
+ __entry->ptr1 = ptr1;
+ __entry->ptr2 = ptr2;),
+
+TP_printk("fun:%u ev:%u u1:%u u2:%u ul1:%llu ul2:%llu p1:0x%pK p2:0x%pK",
+ __entry->func, __entry->evt,
+ __entry->uint1, __entry->uint2,
+ __entry->ulong1, __entry->ulong2,
+ __entry->ptr1, __entry->ptr2)
+
+)
+
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_low,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_high,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_err,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+/*****************************************************************************/
+/* Trace events for rmnet_perf module */
+/*****************************************************************************/
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_perf_low,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_perf_high,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_perf_err,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+/*****************************************************************************/
+/* Trace events for rmnet_shs module */
+/*****************************************************************************/
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_shs_low,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_shs_high,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_shs_err,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_shs_wq_low,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_shs_wq_high,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+DEFINE_EVENT
+ (rmnet_mod_template, rmnet_shs_wq_err,
+
+ TP_PROTO(u8 func, u8 evt, u32 uint1, u32 uint2,
+ u64 ulong1, u64 ulong2, void *ptr1, void *ptr2),
+
+ TP_ARGS(func, evt, uint1, uint2, ulong1, ulong2, ptr1, ptr2)
+
+);
+
+DECLARE_EVENT_CLASS
+ (rmnet_freq_template,
+
+ TP_PROTO(u8 core, u32 newfreq),
+
+ TP_ARGS(core, newfreq),
+
+ TP_STRUCT__entry(__field(u8, core)
+ __field(u32, newfreq)
+ ),
+
+ TP_fast_assign(__entry->core = core;
+ __entry->newfreq = newfreq;
+
+ ),
+
+TP_printk("freq policy core:%u freq floor :%u",
+ __entry->core, __entry->newfreq)
+
+);
+
+DEFINE_EVENT
+ (rmnet_freq_template, rmnet_freq_boost,
+
+ TP_PROTO(u8 core, u32 newfreq),
+
+ TP_ARGS(core, newfreq)
+);
+
+DEFINE_EVENT
+ (rmnet_freq_template, rmnet_freq_reset,
+
+ TP_PROTO(u8 core, u32 newfreq),
+
+ TP_ARGS(core, newfreq)
+);
+
+TRACE_EVENT
+ (rmnet_freq_update,
+
+ TP_PROTO(u8 core, u32 lowfreq, u32 highfreq),
+
+ TP_ARGS(core, lowfreq, highfreq),
+
+ TP_STRUCT__entry(__field(u8, core)
+ __field(u32, lowfreq)
+ __field(u32, highfreq)
+ ),
+
+ TP_fast_assign(__entry->core = core;
+ __entry->lowfreq = lowfreq;
+ __entry->highfreq = highfreq;
+ ),
+
+TP_printk("freq policy update core:%u policy freq floor :%u freq ceil :%u",
+ __entry->core, __entry->lowfreq, __entry->highfreq)
+);
+#endif /* _RMNET_TRACE_H_ */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH ../drivers/net/ethernet/qualcomm/rmnet
+#include <trace/define_trace.h>
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
index 509dfc895a33..5deb1c1657ba 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
@@ -1,11 +1,23 @@
// SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
*
* RMNET Data virtual network driver
+ *
*/
#include <linux/etherdevice.h>
#include <linux/if_arp.h>
+#include <linux/ip.h>
#include <net/pkt_sched.h>
#include "rmnet_config.h"
#include "rmnet_handlers.h"
@@ -15,7 +27,7 @@
/* RX/TX Fixup */
-void rmnet_vnd_rx_fixup(struct sk_buff *skb, struct net_device *dev)
+void rmnet_vnd_rx_fixup(struct net_device *dev, u32 skb_len)
{
struct rmnet_priv *priv = netdev_priv(dev);
struct rmnet_pcpu_stats *pcpu_ptr;
@@ -24,11 +36,11 @@ void rmnet_vnd_rx_fixup(struct sk_buff *skb, struct net_device *dev)
u64_stats_update_begin(&pcpu_ptr->syncp);
pcpu_ptr->stats.rx_pkts++;
- pcpu_ptr->stats.rx_bytes += skb->len;
+ pcpu_ptr->stats.rx_bytes += skb_len;
u64_stats_update_end(&pcpu_ptr->syncp);
}
-void rmnet_vnd_tx_fixup(struct sk_buff *skb, struct net_device *dev)
+void rmnet_vnd_tx_fixup(struct net_device *dev, u32 skb_len)
{
struct rmnet_priv *priv = netdev_priv(dev);
struct rmnet_pcpu_stats *pcpu_ptr;
@@ -37,7 +49,7 @@ void rmnet_vnd_tx_fixup(struct sk_buff *skb, struct net_device *dev)
u64_stats_update_begin(&pcpu_ptr->syncp);
pcpu_ptr->stats.tx_pkts++;
- pcpu_ptr->stats.tx_bytes += skb->len;
+ pcpu_ptr->stats.tx_bytes += skb_len;
u64_stats_update_end(&pcpu_ptr->syncp);
}
@@ -152,6 +164,43 @@ static const char rmnet_gstrings_stats[][ETH_GSTRING_LEN] = {
"Checksum skipped on ip fragment",
"Checksum skipped",
"Checksum computed in software",
+ "Checksum computed in hardware",
+ "Coalescing packets received",
+ "Coalesced packets",
+ "Coalescing header NLO errors",
+ "Coalescing header pcount errors",
+ "Coalescing checksum errors",
+ "Coalescing packet reconstructs",
+ "Coalescing IP version invalid",
+ "Coalescing L4 header invalid",
+ "Coalescing close Non-coalescable",
+ "Coalescing close L3 mismatch",
+ "Coalescing close L4 mismatch",
+ "Coalescing close HW NLO limit",
+ "Coalescing close HW packet limit",
+ "Coalescing close HW byte limit",
+ "Coalescing close HW time limit",
+ "Coalescing close HW eviction",
+ "Coalescing close Coalescable",
+ "Coalescing packets over VEID0",
+ "Coalescing packets over VEID1",
+ "Coalescing packets over VEID2",
+ "Coalescing packets over VEID3",
+};
+
+static const char rmnet_port_gstrings_stats[][ETH_GSTRING_LEN] = {
+ "MAP Cmd last version",
+ "MAP Cmd last ep id",
+ "MAP Cmd last transaction id",
+ "DL header last seen sequence",
+ "DL header last seen bytes",
+ "DL header last seen packets",
+ "DL header last seen flows",
+ "DL header pkts received",
+ "DL header total bytes received",
+ "DL header total pkts received",
+ "DL trailer last seen sequence",
+ "DL trailer pkts received",
};
static void rmnet_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
@@ -160,6 +209,9 @@ static void rmnet_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
case ETH_SS_STATS:
memcpy(buf, &rmnet_gstrings_stats,
sizeof(rmnet_gstrings_stats));
+ memcpy(buf + sizeof(rmnet_gstrings_stats),
+ &rmnet_port_gstrings_stats,
+ sizeof(rmnet_port_gstrings_stats));
break;
}
}
@@ -168,7 +220,8 @@ static int rmnet_get_sset_count(struct net_device *dev, int sset)
{
switch (sset) {
case ETH_SS_STATS:
- return ARRAY_SIZE(rmnet_gstrings_stats);
+ return ARRAY_SIZE(rmnet_gstrings_stats) +
+ ARRAY_SIZE(rmnet_port_gstrings_stats);
default:
return -EOPNOTSUPP;
}
@@ -179,17 +232,42 @@ static void rmnet_get_ethtool_stats(struct net_device *dev,
{
struct rmnet_priv *priv = netdev_priv(dev);
struct rmnet_priv_stats *st = &priv->stats;
+ struct rmnet_port_priv_stats *stp;
+ struct rmnet_port *port;
+
+ port = rmnet_get_port(priv->real_dev);
- if (!data)
+ if (!data || !port)
return;
+ stp = &port->stats;
+
memcpy(data, st, ARRAY_SIZE(rmnet_gstrings_stats) * sizeof(u64));
+ memcpy(data + ARRAY_SIZE(rmnet_gstrings_stats), stp,
+ ARRAY_SIZE(rmnet_port_gstrings_stats) * sizeof(u64));
+}
+
+static int rmnet_stats_reset(struct net_device *dev)
+{
+ struct rmnet_priv *priv = netdev_priv(dev);
+ struct rmnet_port_priv_stats *stp;
+ struct rmnet_port *port;
+
+ port = rmnet_get_port(priv->real_dev);
+ if (!port)
+ return -EINVAL;
+
+ stp = &port->stats;
+
+ memset(stp, 0, sizeof(*stp));
+ return 0;
}
static const struct ethtool_ops rmnet_ethtool_ops = {
.get_ethtool_stats = rmnet_get_ethtool_stats,
.get_strings = rmnet_get_strings,
.get_sset_count = rmnet_get_sset_count,
+ .nway_reset = rmnet_stats_reset,
};
/* Called by kernel whenever a new rmnet<n> device is created. Sets MTU,
@@ -200,7 +278,7 @@ void rmnet_vnd_setup(struct net_device *rmnet_dev)
rmnet_dev->netdev_ops = &rmnet_vnd_ops;
rmnet_dev->mtu = RMNET_DFLT_PACKET_SIZE;
rmnet_dev->needed_headroom = RMNET_NEEDED_HEADROOM;
- eth_random_addr(rmnet_dev->dev_addr);
+ random_ether_addr(rmnet_dev->dev_addr);
rmnet_dev->tx_queue_len = RMNET_TX_QUEUE_LEN;
/* Raw IP mode */
@@ -209,12 +287,7 @@ void rmnet_vnd_setup(struct net_device *rmnet_dev)
rmnet_dev->hard_header_len = 0;
rmnet_dev->flags &= ~(IFF_BROADCAST | IFF_MULTICAST);
- rmnet_dev->needs_free_netdev = true;
rmnet_dev->ethtool_ops = &rmnet_ethtool_ops;
-
- /* This perm addr will be used as interface identifier by IPv6 */
- rmnet_dev->addr_assign_type = NET_ADDR_RANDOM;
- eth_random_addr(rmnet_dev->perm_addr);
}
/* Exposed API */
@@ -236,6 +309,7 @@ int rmnet_vnd_newlink(u8 id, struct net_device *rmnet_dev,
rmnet_dev->hw_features = NETIF_F_RXCSUM;
rmnet_dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
rmnet_dev->hw_features |= NETIF_F_SG;
+ rmnet_dev->hw_features |= NETIF_F_GRO_HW;
priv->real_dev = real_dev;
@@ -263,6 +337,7 @@ int rmnet_vnd_dellink(u8 id, struct rmnet_port *port,
ep->egress_dev = NULL;
port->nr_rmnet_devs--;
+
return 0;
}
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.h
index 54cbaf3c3bc4..5d78d3512ead 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.h
@@ -1,7 +1,17 @@
/* SPDX-License-Identifier: GPL-2.0-only */
-/* Copyright (c) 2013-2017, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2019, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
*
* RMNET Data Virtual Network Device APIs
+ *
*/
#ifndef _RMNET_VND_H_
@@ -14,8 +24,8 @@ int rmnet_vnd_newlink(u8 id, struct net_device *rmnet_dev,
struct rmnet_endpoint *ep);
int rmnet_vnd_dellink(u8 id, struct rmnet_port *port,
struct rmnet_endpoint *ep);
-void rmnet_vnd_rx_fixup(struct sk_buff *skb, struct net_device *dev);
-void rmnet_vnd_tx_fixup(struct sk_buff *skb, struct net_device *dev);
+void rmnet_vnd_rx_fixup(struct net_device *dev, u32 skb_len);
+void rmnet_vnd_tx_fixup(struct net_device *dev, u32 skb_len);
u8 rmnet_vnd_get_mux(struct net_device *rmnet_dev);
void rmnet_vnd_setup(struct net_device *dev);
#endif /* _RMNET_VND_H_ */
diff --git a/include/linux/ipc_logging.h b/include/linux/ipc_logging.h
new file mode 100644
index 000000000000..d2c41efa1347
--- /dev/null
+++ b/include/linux/ipc_logging.h
@@ -0,0 +1,291 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2012-2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+#ifndef _IPC_LOGGING_H
+#define _IPC_LOGGING_H
+
+#include <linux/types.h>
+
+#define MAX_MSG_SIZE 255
+
+enum {
+ TSV_TYPE_MSG_START = 1,
+ TSV_TYPE_SKB = TSV_TYPE_MSG_START,
+ TSV_TYPE_STRING,
+ TSV_TYPE_MSG_END = TSV_TYPE_STRING,
+};
+
+struct tsv_header {
+ unsigned char type;
+ unsigned char size; /* size of data field */
+};
+
+struct encode_context {
+ struct tsv_header hdr;
+ char buff[MAX_MSG_SIZE];
+ int offset;
+};
+
+struct decode_context {
+ int output_format; /* 0 = debugfs */
+ char *buff; /* output buffer */
+ int size; /* size of output buffer */
+};
+
+#if defined(CONFIG_IPC_LOGGING)
+/*
+ * ipc_log_context_create: Create a debug log context
+ * Should not be called from atomic context
+ *
+ * @max_num_pages: Number of pages of logging space required (max. 10)
+ * @mod_name : Name of the directory entry under DEBUGFS
+ * @user_version : Version number of user-defined message formats
+ *
+ * returns context id on success, NULL on failure
+ */
+void *ipc_log_context_create(int max_num_pages, const char *modname,
+ uint16_t user_version);
+
+/*
+ * msg_encode_start: Start encoding a log message
+ *
+ * @ectxt: Temporary storage to hold the encoded message
+ * @type: Root event type defined by the module which is logging
+ */
+void msg_encode_start(struct encode_context *ectxt, uint32_t type);
+
+/*
+ * tsv_timestamp_write: Writes the current timestamp count
+ *
+ * @ectxt: Context initialized by calling msg_encode_start()
+ */
+int tsv_timestamp_write(struct encode_context *ectxt);
+
+/*
+ * tsv_qtimer_write: Writes the current QTimer timestamp count
+ *
+ * @ectxt: Context initialized by calling msg_encode_start()
+ */
+int tsv_qtimer_write(struct encode_context *ectxt);
+
+/*
+ * tsv_pointer_write: Writes a data pointer
+ *
+ * @ectxt: Context initialized by calling msg_encode_start()
+ * @pointer: Pointer value to write
+ */
+int tsv_pointer_write(struct encode_context *ectxt, void *pointer);
+
+/*
+ * tsv_int32_write: Writes a 32-bit integer value
+ *
+ * @ectxt: Context initialized by calling msg_encode_start()
+ * @n: Integer to write
+ */
+int tsv_int32_write(struct encode_context *ectxt, int32_t n);
+
+/*
+ * tsv_int32_write: Writes a 32-bit integer value
+ *
+ * @ectxt: Context initialized by calling msg_encode_start()
+ * @n: Integer to write
+ */
+int tsv_byte_array_write(struct encode_context *ectxt,
+ void *data, int data_size);
+
+/*
+ * msg_encode_end: Complete the message encode process
+ *
+ * @ectxt: Temporary storage which holds the encoded message
+ */
+void msg_encode_end(struct encode_context *ectxt);
+
+/*
+ * msg_encode_end: Complete the message encode process
+ *
+ * @ectxt: Temporary storage which holds the encoded message
+ */
+void ipc_log_write(void *ctxt, struct encode_context *ectxt);
+
+/*
+ * ipc_log_string: Helper function to log a string
+ *
+ * @ilctxt: Debug Log Context created using ipc_log_context_create()
+ * @fmt: Data specified using format specifiers
+ */
+int ipc_log_string(void *ilctxt, const char *fmt, ...) __printf(2, 3);
+
+/**
+ * ipc_log_extract - Reads and deserializes log
+ *
+ * @ilctxt: logging context
+ * @buff: buffer to receive the data
+ * @size: size of the buffer
+ * @returns: 0 if no data read; >0 number of bytes read; < 0 error
+ *
+ * If no data is available to be read, then the ilctxt::read_avail
+ * completion is reinitialized. This allows clients to block
+ * until new log data is save.
+ */
+int ipc_log_extract(void *ilctxt, char *buff, int size);
+
+/*
+ * Print a string to decode context.
+ * @dctxt Decode context
+ * @args printf args
+ */
+#define IPC_SPRINTF_DECODE(dctxt, args...) \
+do { \
+ int i; \
+ i = scnprintf(dctxt->buff, dctxt->size, args); \
+ dctxt->buff += i; \
+ dctxt->size -= i; \
+} while (0)
+
+/*
+ * tsv_timestamp_read: Reads a timestamp
+ *
+ * @ectxt: Context retrieved by reading from log space
+ * @dctxt: Temporary storage to hold the decoded message
+ * @format: Output format while dumping through DEBUGFS
+ */
+void tsv_timestamp_read(struct encode_context *ectxt,
+ struct decode_context *dctxt, const char *format);
+
+/*
+ * tsv_qtimer_read: Reads a QTimer timestamp
+ *
+ * @ectxt: Context retrieved by reading from log space
+ * @dctxt: Temporary storage to hold the decoded message
+ * @format: Output format while dumping through DEBUGFS
+ */
+void tsv_qtimer_read(struct encode_context *ectxt,
+ struct decode_context *dctxt, const char *format);
+
+/*
+ * tsv_pointer_read: Reads a data pointer
+ *
+ * @ectxt: Context retrieved by reading from log space
+ * @dctxt: Temporary storage to hold the decoded message
+ * @format: Output format while dumping through DEBUGFS
+ */
+void tsv_pointer_read(struct encode_context *ectxt,
+ struct decode_context *dctxt, const char *format);
+
+/*
+ * tsv_int32_read: Reads a 32-bit integer value
+ *
+ * @ectxt: Context retrieved by reading from log space
+ * @dctxt: Temporary storage to hold the decoded message
+ * @format: Output format while dumping through DEBUGFS
+ */
+int32_t tsv_int32_read(struct encode_context *ectxt,
+ struct decode_context *dctxt, const char *format);
+
+/*
+ * tsv_int32_read: Reads a 32-bit integer value
+ *
+ * @ectxt: Context retrieved by reading from log space
+ * @dctxt: Temporary storage to hold the decoded message
+ * @format: Output format while dumping through DEBUGFS
+ */
+void tsv_byte_array_read(struct encode_context *ectxt,
+ struct decode_context *dctxt, const char *format);
+
+/*
+ * add_deserialization_func: Register a deserialization function to
+ * unpack the subevents of a main event
+ *
+ * @ctxt: Debug log context to which the deserialization function has
+ * to be registered
+ * @type: Main/Root event, defined by the module which is logging, to
+ * which this deserialization function has to be registered.
+ * @dfune: Deserialization function to be registered
+ *
+ * return 0 on success, -ve value on FAILURE
+ */
+int add_deserialization_func(void *ctxt, int type,
+ void (*dfunc)(struct encode_context *,
+ struct decode_context *));
+
+/*
+ * ipc_log_context_destroy: Destroy debug log context
+ *
+ * @ctxt: debug log context created by calling ipc_log_context_create API.
+ */
+int ipc_log_context_destroy(void *ctxt);
+
+#else
+
+static inline void *ipc_log_context_create(int max_num_pages,
+ const char *modname, uint16_t user_version)
+{ return NULL; }
+
+static inline void msg_encode_start(struct encode_context *ectxt,
+ uint32_t type) { }
+
+static inline int tsv_timestamp_write(struct encode_context *ectxt)
+{ return -EINVAL; }
+
+static inline int tsv_qtimer_write(struct encode_context *ectxt)
+{ return -EINVAL; }
+
+static inline int tsv_pointer_write(struct encode_context *ectxt, void *pointer)
+{ return -EINVAL; }
+
+static inline int tsv_int32_write(struct encode_context *ectxt, int32_t n)
+{ return -EINVAL; }
+
+static inline int tsv_byte_array_write(struct encode_context *ectxt,
+ void *data, int data_size)
+{ return -EINVAL; }
+
+static inline void msg_encode_end(struct encode_context *ectxt) { }
+
+static inline void ipc_log_write(void *ctxt, struct encode_context *ectxt) { }
+
+static inline int ipc_log_string(void *ilctxt, const char *fmt, ...)
+{ return -EINVAL; }
+
+static inline int ipc_log_extract(void *ilctxt, char *buff, int size)
+{ return -EINVAL; }
+
+#define IPC_SPRINTF_DECODE(dctxt, args...) do { } while (0)
+
+static inline void tsv_timestamp_read(struct encode_context *ectxt,
+ struct decode_context *dctxt, const char *format) { }
+
+static inline void tsv_qtimer_read(struct encode_context *ectxt,
+ struct decode_context *dctxt, const char *format) { }
+
+static inline void tsv_pointer_read(struct encode_context *ectxt,
+ struct decode_context *dctxt, const char *format) { }
+
+static inline int32_t tsv_int32_read(struct encode_context *ectxt,
+ struct decode_context *dctxt, const char *format)
+{ return 0; }
+
+static inline void tsv_byte_array_read(struct encode_context *ectxt,
+ struct decode_context *dctxt, const char *format) { }
+
+static inline int add_deserialization_func(void *ctxt, int type,
+ void (*dfunc)(struct encode_context *,
+ struct decode_context *))
+{ return 0; }
+
+static inline int ipc_log_context_destroy(void *ctxt)
+{ return 0; }
+
+#endif
+
+#endif
diff --git a/include/linux/mhi.h b/include/linux/mhi.h
new file mode 100644
index 000000000000..36373b2d56f4
--- /dev/null
+++ b/include/linux/mhi.h
@@ -0,0 +1,743 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. */
+
+
+#ifndef _MHI_H_
+#define _MHI_H_
+
+#include <linux/dma-direction.h>
+
+struct mhi_chan;
+struct mhi_event;
+struct mhi_ctxt;
+struct mhi_cmd;
+struct image_info;
+struct bhi_vec_entry;
+struct mhi_timesync;
+struct mhi_buf_info;
+
+/**
+ * enum MHI_CB - MHI callback
+ * @MHI_CB_IDLE: MHI entered idle state
+ * @MHI_CB_PENDING_DATA: New data available for client to process
+ * @MHI_CB_LPM_ENTER: MHI host entered low power mode
+ * @MHI_CB_LPM_EXIT: MHI host about to exit low power mode
+ * @MHI_CB_EE_RDDM: MHI device entered RDDM execution enviornment
+ * @MHI_CB_SYS_ERROR: MHI device enter error state (may recover)
+ * @MHI_CB_FATAL_ERROR: MHI device entered fatal error
+ */
+enum MHI_CB {
+ MHI_CB_IDLE,
+ MHI_CB_PENDING_DATA,
+ MHI_CB_LPM_ENTER,
+ MHI_CB_LPM_EXIT,
+ MHI_CB_EE_RDDM,
+ MHI_CB_SYS_ERROR,
+ MHI_CB_FATAL_ERROR,
+};
+
+/**
+ * enum MHI_DEBUG_LEVL - various debugging level
+ */
+enum MHI_DEBUG_LEVEL {
+ MHI_MSG_LVL_VERBOSE,
+ MHI_MSG_LVL_INFO,
+ MHI_MSG_LVL_ERROR,
+ MHI_MSG_LVL_CRITICAL,
+ MHI_MSG_LVL_MASK_ALL,
+};
+
+/**
+ * enum MHI_FLAGS - Transfer flags
+ * @MHI_EOB: End of buffer for bulk transfer
+ * @MHI_EOT: End of transfer
+ * @MHI_CHAIN: Linked transfer
+ */
+enum MHI_FLAGS {
+ MHI_EOB,
+ MHI_EOT,
+ MHI_CHAIN,
+};
+
+/**
+ * enum mhi_device_type - Device types
+ * @MHI_XFER_TYPE: Handles data transfer
+ * @MHI_TIMESYNC_TYPE: Use for timesync feature
+ * @MHI_CONTROLLER_TYPE: Control device
+ */
+enum mhi_device_type {
+ MHI_XFER_TYPE,
+ MHI_TIMESYNC_TYPE,
+ MHI_CONTROLLER_TYPE,
+};
+
+/**
+ * enum mhi_ee - device current execution enviornment
+ * @MHI_EE_PBL - device in PBL
+ * @MHI_EE_SBL - device in SBL
+ * @MHI_EE_AMSS - device in mission mode (firmware fully loaded)
+ * @MHI_EE_RDDM - device in ram dump collection mode
+ * @MHI_EE_WFW - device in WLAN firmware mode
+ * @MHI_EE_PTHRU - device in PBL but configured in pass thru mode
+ * @MHI_EE_EDL - device in emergency download mode
+ */
+enum mhi_ee {
+ MHI_EE_PBL = 0x0,
+ MHI_EE_SBL = 0x1,
+ MHI_EE_AMSS = 0x2,
+ MHI_EE_RDDM = 0x3,
+ MHI_EE_WFW = 0x4,
+ MHI_EE_PTHRU = 0x5,
+ MHI_EE_EDL = 0x6,
+ MHI_EE_MAX_SUPPORTED = MHI_EE_EDL,
+ MHI_EE_DISABLE_TRANSITION, /* local EE, not related to mhi spec */
+ MHI_EE_MAX,
+};
+
+/**
+ * enum mhi_dev_state - device current MHI state
+ */
+enum mhi_dev_state {
+ MHI_STATE_RESET = 0x0,
+ MHI_STATE_READY = 0x1,
+ MHI_STATE_M0 = 0x2,
+ MHI_STATE_M1 = 0x3,
+ MHI_STATE_M2 = 0x4,
+ MHI_STATE_M3 = 0x5,
+ MHI_STATE_BHI = 0x7,
+ MHI_STATE_SYS_ERR = 0xFF,
+ MHI_STATE_MAX,
+};
+
+extern const char * const mhi_ee_str[MHI_EE_MAX];
+#define TO_MHI_EXEC_STR(ee) (((ee) >= MHI_EE_MAX) ? \
+ "INVALID_EE" : mhi_ee_str[ee])
+
+/**
+ * struct image_info - firmware and rddm table
+ * @mhi_buf - Contain device firmware and rddm table
+ * @entries - # of entries in table
+ */
+struct image_info {
+ struct mhi_buf *mhi_buf;
+ struct bhi_vec_entry *bhi_vec;
+ u32 entries;
+};
+
+/**
+ * struct mhi_controller - Master controller structure for external modem
+ * @dev: Device associated with this controller
+ * @of_node: DT that has MHI configuration information
+ * @regs: Points to base of MHI MMIO register space
+ * @bhi: Points to base of MHI BHI register space
+ * @bhie: Points to base of MHI BHIe register space
+ * @wake_db: MHI WAKE doorbell register address
+ * @dev_id: PCIe device id of the external device
+ * @domain: PCIe domain the device connected to
+ * @bus: PCIe bus the device assigned to
+ * @slot: PCIe slot for the modem
+ * @iova_start: IOMMU starting address for data
+ * @iova_stop: IOMMU stop address for data
+ * @fw_image: Firmware image name for normal booting
+ * @edl_image: Firmware image name for emergency download mode
+ * @fbc_download: MHI host needs to do complete image transfer
+ * @rddm_size: RAM dump size that host should allocate for debugging purpose
+ * @sbl_size: SBL image size
+ * @seg_len: BHIe vector size
+ * @fbc_image: Points to firmware image buffer
+ * @rddm_image: Points to RAM dump buffer
+ * @max_chan: Maximum number of channels controller support
+ * @mhi_chan: Points to channel configuration table
+ * @lpm_chans: List of channels that require LPM notifications
+ * @total_ev_rings: Total # of event rings allocated
+ * @hw_ev_rings: Number of hardware event rings
+ * @sw_ev_rings: Number of software event rings
+ * @msi_required: Number of msi required to operate
+ * @msi_allocated: Number of msi allocated by bus master
+ * @irq: base irq # to request
+ * @mhi_event: MHI event ring configurations table
+ * @mhi_cmd: MHI command ring configurations table
+ * @mhi_ctxt: MHI device context, shared memory between host and device
+ * @timeout_ms: Timeout in ms for state transitions
+ * @pm_state: Power management state
+ * @ee: MHI device execution environment
+ * @dev_state: MHI STATE
+ * @status_cb: CB function to notify various power states to but master
+ * @link_status: Query link status in case of abnormal value read from device
+ * @runtime_get: Async runtime resume function
+ * @runtimet_put: Release votes
+ * @time_get: Return host time in us
+ * @lpm_disable: Request controller to disable link level low power modes
+ * @lpm_enable: Controller may enable link level low power modes again
+ * @priv_data: Points to bus master's private data
+ */
+struct mhi_controller {
+ struct list_head node;
+ struct mhi_device *mhi_dev;
+
+ /* device node for iommu ops */
+ struct device *dev;
+ struct device_node *of_node;
+
+ /* mmio base */
+ phys_addr_t base_addr;
+ void __iomem *regs;
+ void __iomem *bhi;
+ void __iomem *bhie;
+ void __iomem *wake_db;
+
+ /* device topology */
+ u32 dev_id;
+ u32 domain;
+ u32 bus;
+ u32 slot;
+
+ /* addressing window */
+ dma_addr_t iova_start;
+ dma_addr_t iova_stop;
+
+ /* fw images */
+ const char *fw_image;
+ const char *edl_image;
+
+ /* mhi host manages downloading entire fbc images */
+ bool fbc_download;
+ size_t rddm_size;
+ size_t sbl_size;
+ size_t seg_len;
+ u32 session_id;
+ u32 sequence_id;
+ struct image_info *fbc_image;
+ struct image_info *rddm_image;
+
+ /* physical channel config data */
+ u32 max_chan;
+ struct mhi_chan *mhi_chan;
+ struct list_head lpm_chans; /* these chan require lpm notification */
+
+ /* physical event config data */
+ u32 total_ev_rings;
+ u32 hw_ev_rings;
+ u32 sw_ev_rings;
+ u32 msi_required;
+ u32 msi_allocated;
+ int *irq; /* interrupt table */
+ struct mhi_event *mhi_event;
+
+ /* cmd rings */
+ struct mhi_cmd *mhi_cmd;
+
+ /* mhi context (shared with device) */
+ struct mhi_ctxt *mhi_ctxt;
+
+ u32 timeout_ms;
+
+ /* caller should grab pm_mutex for suspend/resume operations */
+ struct mutex pm_mutex;
+ bool pre_init;
+ rwlock_t pm_lock;
+ u32 pm_state;
+ enum mhi_ee ee;
+ enum mhi_dev_state dev_state;
+ bool wake_set;
+ atomic_t dev_wake;
+ atomic_t alloc_size;
+ struct list_head transition_list;
+ spinlock_t transition_lock;
+ spinlock_t wlock;
+
+ /* debug counters */
+ u32 M0, M2, M3;
+
+ /* worker for different state transitions */
+ struct work_struct st_worker;
+ struct work_struct fw_worker;
+ struct work_struct syserr_worker;
+ wait_queue_head_t state_event;
+
+ /* shadow functions */
+ void (*status_cb)(struct mhi_controller *mhi_cntrl, void *priv,
+ enum MHI_CB reason);
+ int (*link_status)(struct mhi_controller *mhi_cntrl, void *priv);
+ void (*wake_get)(struct mhi_controller *mhi_cntrl, bool override);
+ void (*wake_put)(struct mhi_controller *mhi_cntrl, bool override);
+ int (*runtime_get)(struct mhi_controller *mhi_cntrl, void *priv);
+ void (*runtime_put)(struct mhi_controller *mhi_cntrl, void *priv);
+ u64 (*time_get)(struct mhi_controller *mhi_cntrl, void *priv);
+ int (*lpm_disable)(struct mhi_controller *mhi_cntrl, void *priv);
+ int (*lpm_enable)(struct mhi_controller *mhi_cntrl, void *priv);
+ int (*map_single)(struct mhi_controller *mhi_cntrl,
+ struct mhi_buf_info *buf);
+ void (*unmap_single)(struct mhi_controller *mhi_cntrl,
+ struct mhi_buf_info *buf);
+
+ /* channel to control DTR messaging */
+ struct mhi_device *dtr_dev;
+
+ /* bounce buffer settings */
+ bool bounce_buf;
+ size_t buffer_len;
+
+ /* supports time sync feature */
+ struct mhi_timesync *mhi_tsync;
+ struct mhi_device *tsync_dev;
+
+ /* kernel log level */
+ enum MHI_DEBUG_LEVEL klog_lvl;
+
+ /* private log level controller driver to set */
+ enum MHI_DEBUG_LEVEL log_lvl;
+
+ /* controller specific data */
+ void *priv_data;
+ void *log_buf;
+ struct dentry *dentry;
+ struct dentry *parent;
+ struct notifier_block mhi_panic_notifier;
+};
+
+/**
+ * struct mhi_device - mhi device structure associated bind to channel
+ * @dev: Device associated with the channels
+ * @mtu: Maximum # of bytes controller support
+ * @ul_chan_id: MHI channel id for UL transfer
+ * @dl_chan_id: MHI channel id for DL transfer
+ * @tiocm: Device current terminal settings
+ * @priv: Driver private data
+ */
+struct mhi_device {
+ struct device dev;
+ u32 dev_id;
+ u32 domain;
+ u32 bus;
+ u32 slot;
+ size_t mtu;
+ int ul_chan_id;
+ int dl_chan_id;
+ int ul_event_id;
+ int dl_event_id;
+ u32 tiocm;
+ const struct mhi_device_id *id;
+ const char *chan_name;
+ struct mhi_controller *mhi_cntrl;
+ struct mhi_chan *ul_chan;
+ struct mhi_chan *dl_chan;
+ atomic_t dev_wake;
+ enum mhi_device_type dev_type;
+ void *priv_data;
+ int (*ul_xfer)(struct mhi_device *mhi_dev, struct mhi_chan *mhi_chan,
+ void *buf, size_t len, enum MHI_FLAGS flags);
+ int (*dl_xfer)(struct mhi_device *mhi_dev, struct mhi_chan *mhi_chan,
+ void *buf, size_t size, enum MHI_FLAGS flags);
+ void (*status_cb)(struct mhi_device *mhi_dev, enum MHI_CB reason);
+};
+
+/**
+ * struct mhi_result - Completed buffer information
+ * @buf_addr: Address of data buffer
+ * @dir: Channel direction
+ * @bytes_xfer: # of bytes transferred
+ * @transaction_status: Status of last trasnferred
+ */
+struct mhi_result {
+ void *buf_addr;
+ enum dma_data_direction dir;
+ size_t bytes_xferd;
+ int transaction_status;
+};
+
+/**
+ * struct mhi_buf - Describes the buffer
+ * @page: buffer as a page
+ * @buf: cpu address for the buffer
+ * @phys_addr: physical address of the buffer
+ * @dma_addr: iommu address for the buffer
+ * @skb: skb of ip packet
+ * @len: # of bytes
+ * @name: Buffer label, for offload channel configurations name must be:
+ * ECA - Event context array data
+ * CCA - Channel context array data
+ */
+struct mhi_buf {
+ struct list_head node;
+ struct page *page;
+ void *buf;
+ phys_addr_t phys_addr;
+ dma_addr_t dma_addr;
+ struct sk_buff *skb;
+ size_t len;
+ const char *name; /* ECA, CCA */
+};
+
+/**
+ * struct mhi_driver - mhi driver information
+ * @id_table: NULL terminated channel ID names
+ * @ul_xfer_cb: UL data transfer callback
+ * @dl_xfer_cb: DL data transfer callback
+ * @status_cb: Asynchronous status callback
+ */
+struct mhi_driver {
+ const struct mhi_device_id *id_table;
+ int (*probe)(struct mhi_device *mhi_dev,
+ const struct mhi_device_id *id);
+ void (*remove)(struct mhi_device *mhi_dev);
+ void (*ul_xfer_cb)(struct mhi_device *mhi_dev, struct mhi_result *res);
+ void (*dl_xfer_cb)(struct mhi_device *mhi_dev, struct mhi_result *res);
+ void (*status_cb)(struct mhi_device *mhi_dev, enum MHI_CB mhi_cb);
+ struct device_driver driver;
+};
+
+#define to_mhi_driver(drv) container_of(drv, struct mhi_driver, driver)
+#define to_mhi_device(dev) container_of(dev, struct mhi_device, dev)
+
+static inline void mhi_device_set_devdata(struct mhi_device *mhi_dev,
+ void *priv)
+{
+ mhi_dev->priv_data = priv;
+}
+
+static inline void *mhi_device_get_devdata(struct mhi_device *mhi_dev)
+{
+ return mhi_dev->priv_data;
+}
+
+/**
+ * mhi_queue_transfer - Queue a buffer to hardware
+ * All transfers are asyncronous transfers
+ * @mhi_dev: Device associated with the channels
+ * @dir: Data direction
+ * @buf: Data buffer (skb for hardware channels)
+ * @len: Size in bytes
+ * @mflags: Interrupt flags for the device
+ */
+static inline int mhi_queue_transfer(struct mhi_device *mhi_dev,
+ enum dma_data_direction dir,
+ void *buf,
+ size_t len,
+ enum MHI_FLAGS mflags)
+{
+ if (dir == DMA_TO_DEVICE)
+ return mhi_dev->ul_xfer(mhi_dev, mhi_dev->ul_chan, buf, len,
+ mflags);
+ else
+ return mhi_dev->dl_xfer(mhi_dev, mhi_dev->dl_chan, buf, len,
+ mflags);
+}
+
+static inline void *mhi_controller_get_devdata(struct mhi_controller *mhi_cntrl)
+{
+ return mhi_cntrl->priv_data;
+}
+
+static inline void mhi_free_controller(struct mhi_controller *mhi_cntrl)
+{
+ kfree(mhi_cntrl);
+}
+
+/**
+ * mhi_driver_register - Register driver with MHI framework
+ * @mhi_drv: mhi_driver structure
+ */
+int mhi_driver_register(struct mhi_driver *mhi_drv);
+
+/**
+ * mhi_driver_unregister - Unregister a driver for mhi_devices
+ * @mhi_drv: mhi_driver structure
+ */
+void mhi_driver_unregister(struct mhi_driver *mhi_drv);
+
+/**
+ * mhi_device_configure - configure ECA or CCA context
+ * For offload channels that client manage, call this
+ * function to configure channel context or event context
+ * array associated with the channel
+ * @mhi_div: Device associated with the channels
+ * @dir: Direction of the channel
+ * @mhi_buf: Configuration data
+ * @elements: # of configuration elements
+ */
+int mhi_device_configure(struct mhi_device *mhi_div,
+ enum dma_data_direction dir,
+ struct mhi_buf *mhi_buf,
+ int elements);
+
+/**
+ * mhi_device_get - disable all low power modes
+ * Only disables lpm, does not immediately exit low power mode
+ * if controller already in a low power mode
+ * @mhi_dev: Device associated with the channels
+ */
+void mhi_device_get(struct mhi_device *mhi_dev);
+
+/**
+ * mhi_device_get_sync - disable all low power modes
+ * Synchronously disable all low power, exit low power mode if
+ * controller already in a low power state
+ * @mhi_dev: Device associated with the channels
+ */
+int mhi_device_get_sync(struct mhi_device *mhi_dev);
+
+/**
+ * mhi_device_put - re-enable low power modes
+ * @mhi_dev: Device associated with the channels
+ */
+void mhi_device_put(struct mhi_device *mhi_dev);
+
+/**
+ * mhi_prepare_for_transfer - setup channel for data transfer
+ * Moves both UL and DL channel from RESET to START state
+ * @mhi_dev: Device associated with the channels
+ */
+int mhi_prepare_for_transfer(struct mhi_device *mhi_dev);
+
+/**
+ * mhi_unprepare_from_transfer -unprepare the channels
+ * Moves both UL and DL channels to RESET state
+ * @mhi_dev: Device associated with the channels
+ */
+void mhi_unprepare_from_transfer(struct mhi_device *mhi_dev);
+
+/**
+ * mhi_get_no_free_descriptors - Get transfer ring length
+ * Get # of TD available to queue buffers
+ * @mhi_dev: Device associated with the channels
+ * @dir: Direction of the channel
+ */
+int mhi_get_no_free_descriptors(struct mhi_device *mhi_dev,
+ enum dma_data_direction dir);
+
+/**
+ * mhi_poll - poll for any available data to consume
+ * This is only applicable for DL direction
+ * @mhi_dev: Device associated with the channels
+ * @budget: In descriptors to service before returning
+ */
+int mhi_poll(struct mhi_device *mhi_dev, u32 budget);
+
+/**
+ * mhi_ioctl - user space IOCTL support for MHI channels
+ * Native support for setting TIOCM
+ * @mhi_dev: Device associated with the channels
+ * @cmd: IOCTL cmd
+ * @arg: Optional parameter, iotcl cmd specific
+ */
+long mhi_ioctl(struct mhi_device *mhi_dev, unsigned int cmd, unsigned long arg);
+
+/**
+ * mhi_alloc_controller - Allocate mhi_controller structure
+ * Allocate controller structure and additional data for controller
+ * private data. You may get the private data pointer by calling
+ * mhi_controller_get_devdata
+ * @size: # of additional bytes to allocate
+ */
+struct mhi_controller *mhi_alloc_controller(size_t size);
+
+/**
+ * of_register_mhi_controller - Register MHI controller
+ * Registers MHI controller with MHI bus framework. DT must be supported
+ * @mhi_cntrl: MHI controller to register
+ */
+int of_register_mhi_controller(struct mhi_controller *mhi_cntrl);
+
+void mhi_unregister_mhi_controller(struct mhi_controller *mhi_cntrl);
+
+/**
+ * mhi_bdf_to_controller - Look up a registered controller
+ * Search for controller based on device identification
+ * @domain: RC domain of the device
+ * @bus: Bus device connected to
+ * @slot: Slot device assigned to
+ * @dev_id: Device Identification
+ */
+struct mhi_controller *mhi_bdf_to_controller(u32 domain, u32 bus, u32 slot,
+ u32 dev_id);
+
+/**
+ * mhi_prepare_for_power_up - Do pre-initialization before power up
+ * This is optional, call this before power up if controller do not
+ * want bus framework to automatically free any allocated memory during shutdown
+ * process.
+ * @mhi_cntrl: MHI controller
+ */
+int mhi_prepare_for_power_up(struct mhi_controller *mhi_cntrl);
+
+/**
+ * mhi_async_power_up - Starts MHI power up sequence
+ * @mhi_cntrl: MHI controller
+ */
+int mhi_async_power_up(struct mhi_controller *mhi_cntrl);
+int mhi_sync_power_up(struct mhi_controller *mhi_cntrl);
+
+/**
+ * mhi_power_down - Start MHI power down sequence
+ * @mhi_cntrl: MHI controller
+ * @graceful: link is still accessible, do a graceful shutdown process otherwise
+ * we will shutdown host w/o putting device into RESET state
+ */
+void mhi_power_down(struct mhi_controller *mhi_cntrl, bool graceful);
+
+/**
+ * mhi_unprepare_after_powre_down - free any allocated memory for power up
+ * @mhi_cntrl: MHI controller
+ */
+void mhi_unprepare_after_power_down(struct mhi_controller *mhi_cntrl);
+
+/**
+ * mhi_pm_suspend - Move MHI into a suspended state
+ * Transition to MHI state M3 state from M0||M1||M2 state
+ * @mhi_cntrl: MHI controller
+ */
+int mhi_pm_suspend(struct mhi_controller *mhi_cntrl);
+
+/**
+ * mhi_pm_resume - Resume MHI from suspended state
+ * Transition to MHI state M0 state from M3 state
+ * @mhi_cntrl: MHI controller
+ */
+int mhi_pm_resume(struct mhi_controller *mhi_cntrl);
+
+/**
+ * mhi_download_rddm_img - Download ramdump image from device for
+ * debugging purpose.
+ * @mhi_cntrl: MHI controller
+ * @in_panic: If we trying to capture image while in kernel panic
+ */
+int mhi_download_rddm_img(struct mhi_controller *mhi_cntrl, bool in_panic);
+
+/**
+ * mhi_force_rddm_mode - Force external device into rddm mode
+ * to collect device ramdump. This is useful if host driver assert
+ * and we need to see device state as well.
+ * @mhi_cntrl: MHI controller
+ */
+int mhi_force_rddm_mode(struct mhi_controller *mhi_cntrl);
+
+/**
+ * mhi_get_remote_time_sync - Get external soc time relative to local soc time
+ * using MMIO method.
+ * @mhi_dev: Device associated with the channels
+ * @t_host: Pointer to output local soc time
+ * @t_dev: Pointer to output remote soc time
+ */
+int mhi_get_remote_time_sync(struct mhi_device *mhi_dev,
+ u64 *t_host,
+ u64 *t_dev);
+
+/**
+ * mhi_get_mhi_state - Return MHI state of device
+ * @mhi_cntrl: MHI controller
+ */
+enum mhi_dev_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl);
+
+/**
+ * mhi_set_mhi_state - Set device state
+ * @mhi_cntrl: MHI controller
+ * @state: state to set
+ */
+void mhi_set_mhi_state(struct mhi_controller *mhi_cntrl,
+ enum mhi_dev_state state);
+
+
+/**
+ * mhi_is_active - helper function to determine if MHI in active state
+ * @mhi_dev: client device
+ */
+static inline bool mhi_is_active(struct mhi_device *mhi_dev)
+{
+ struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl;
+
+ return (mhi_cntrl->dev_state >= MHI_STATE_M0 &&
+ mhi_cntrl->dev_state <= MHI_STATE_M3);
+}
+
+/**
+ * mhi_debug_reg_dump - dump MHI registers for debug purpose
+ * @mhi_cntrl: MHI controller
+ */
+void mhi_debug_reg_dump(struct mhi_controller *mhi_cntrl);
+
+void mhi_wdt_panic_handler(void);
+
+int get_mhi_pci_status(void);
+
+#ifndef CONFIG_ARCH_QCOM
+
+#ifdef CONFIG_MHI_DEBUG
+
+#define MHI_VERB(fmt, ...) do { \
+ if (mhi_cntrl->klog_lvl <= MHI_MSG_LVL_VERBOSE) \
+ pr_debug("[D][%s] " fmt, __func__, ##__VA_ARGS__);\
+} while (0)
+
+#else
+
+#define MHI_VERB(fmt, ...)
+
+#endif
+
+#define MHI_LOG(fmt, ...) do { \
+ if (mhi_cntrl->klog_lvl <= MHI_MSG_LVL_INFO) \
+ pr_info("[I][%s] " fmt, __func__, ##__VA_ARGS__);\
+} while (0)
+
+#define MHI_ERR(fmt, ...) do { \
+ if (mhi_cntrl->klog_lvl <= MHI_MSG_LVL_ERROR) \
+ pr_err("[E][%s] " fmt, __func__, ##__VA_ARGS__); \
+} while (0)
+
+#define MHI_CRITICAL(fmt, ...) do { \
+ if (mhi_cntrl->klog_lvl <= MHI_MSG_LVL_CRITICAL) \
+ pr_alert("[C][%s] " fmt, __func__, ##__VA_ARGS__); \
+} while (0)
+
+#else /* ARCH QCOM */
+
+#include <linux/ipc_logging.h>
+
+#ifdef CONFIG_MHI_DEBUG
+
+#define MHI_VERB(fmt, ...) do { \
+ if (mhi_cntrl->klog_lvl <= MHI_MSG_LVL_VERBOSE) \
+ pr_err("[D][%s] " fmt, __func__, ##__VA_ARGS__);\
+ if (mhi_cntrl->log_buf && \
+ (mhi_cntrl->log_lvl <= MHI_MSG_LVL_VERBOSE)) \
+ ipc_log_string(mhi_cntrl->log_buf, "[D][%s] " fmt, \
+ __func__, ##__VA_ARGS__); \
+} while (0)
+
+#else
+
+#define MHI_VERB(fmt, ...)
+
+#endif
+
+#define MHI_LOG(fmt, ...) do { \
+ if (mhi_cntrl->klog_lvl <= MHI_MSG_LVL_INFO) \
+ pr_err("[I][%s] " fmt, __func__, ##__VA_ARGS__);\
+ if (mhi_cntrl->log_buf && \
+ (mhi_cntrl->log_lvl <= MHI_MSG_LVL_INFO)) \
+ ipc_log_string(mhi_cntrl->log_buf, "[I][%s] " fmt, \
+ __func__, ##__VA_ARGS__); \
+} while (0)
+
+#define MHI_ERR(fmt, ...) do { \
+ if (mhi_cntrl->klog_lvl <= MHI_MSG_LVL_ERROR) \
+ pr_err("[E][%s] " fmt, __func__, ##__VA_ARGS__); \
+ if (mhi_cntrl->log_buf && \
+ (mhi_cntrl->log_lvl <= MHI_MSG_LVL_ERROR)) \
+ ipc_log_string(mhi_cntrl->log_buf, "[E][%s] " fmt, \
+ __func__, ##__VA_ARGS__); \
+} while (0)
+
+#define MHI_CRITICAL(fmt, ...) do { \
+ if (mhi_cntrl->klog_lvl <= MHI_MSG_LVL_CRITICAL) \
+ pr_err("[C][%s] " fmt, __func__, ##__VA_ARGS__); \
+ if (mhi_cntrl->log_buf && \
+ (mhi_cntrl->log_lvl <= MHI_MSG_LVL_CRITICAL)) \
+ ipc_log_string(mhi_cntrl->log_buf, "[C][%s] " fmt, \
+ __func__, ##__VA_ARGS__); \
+} while (0)
+
+#endif
+
+#endif /* _MHI_H_ */
diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h
index 5714fd35a83c..eda8e11fabf6 100644
--- a/include/linux/mod_devicetable.h
+++ b/include/linux/mod_devicetable.h
@@ -587,9 +587,9 @@ struct platform_device_id {
#define MDIO_NAME_SIZE 32
#define MDIO_MODULE_PREFIX "mdio:"
-#define MDIO_ID_FMT "%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d%d"
+#define MDIO_ID_FMT "%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u"
#define MDIO_ID_ARGS(_id) \
- (_id)>>31, ((_id)>>30) & 1, ((_id)>>29) & 1, ((_id)>>28) & 1, \
+ ((_id)>>31) & 1, ((_id)>>30) & 1, ((_id)>>29) & 1, ((_id)>>28) & 1, \
((_id)>>27) & 1, ((_id)>>26) & 1, ((_id)>>25) & 1, ((_id)>>24) & 1, \
((_id)>>23) & 1, ((_id)>>22) & 1, ((_id)>>21) & 1, ((_id)>>20) & 1, \
((_id)>>19) & 1, ((_id)>>18) & 1, ((_id)>>17) & 1, ((_id)>>16) & 1, \
@@ -657,6 +657,10 @@ struct mips_cdmm_device_id {
/*
* MODULE_DEVICE_TABLE expects this struct to be called x86cpu_device_id.
* Although gcc seems to ignore this error, clang fails without this define.
+ *
+ * Note: The ordering of the struct is different from upstream because the
+ * static initializers in kernels < 5.7 still use C89 style while upstream
+ * has been converted to proper C99 initializers.
*/
#define x86cpu_device_id x86_cpu_id
struct x86_cpu_id {
@@ -665,6 +669,7 @@ struct x86_cpu_id {
__u16 model;
__u16 feature; /* bit index */
kernel_ulong_t driver_data;
+ __u16 steppings;
};
#define X86_FEATURE_MATCH(x) \
@@ -673,6 +678,7 @@ struct x86_cpu_id {
#define X86_VENDOR_ANY 0xffff
#define X86_FAMILY_ANY 0
#define X86_MODEL_ANY 0
+#define X86_STEPPING_ANY 0
#define X86_FEATURE_ANY 0 /* Same as FPU, you can't test for that */
/*
@@ -821,4 +827,16 @@ struct wmi_device_id {
const void *context;
};
+#define MHI_NAME_SIZE 32
+
+/**
+ * struct mhi_device_id - MHI device identification
+ * @chan: MHI channel name
+ * @driver_data: driver data;
+ */
+
+struct mhi_device_id {
+ const char chan[MHI_NAME_SIZE];
+ kernel_ulong_t driver_data;
+};
#endif /* LINUX_MOD_DEVICETABLE_H */
diff --git a/include/linux/msm-bus.h b/include/linux/msm-bus.h
new file mode 100644
index 000000000000..2d346f698a68
--- /dev/null
+++ b/include/linux/msm-bus.h
@@ -0,0 +1,214 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2010-2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+#ifndef _ARCH_ARM_MACH_MSM_BUS_H
+#define _ARCH_ARM_MACH_MSM_BUS_H
+
+#include <linux/types.h>
+#include <linux/input.h>
+#include <linux/platform_device.h>
+
+/*
+ * Macros for clients to convert their data to ib and ab
+ * Ws : Time window over which to transfer the data in SECONDS
+ * Bs : Size of the data block in bytes
+ * Per : Recurrence period
+ * Tb : Throughput bandwidth to prevent stalling
+ * R : Ratio of actual bandwidth used to Tb
+ * Ib : Instantaneous bandwidth
+ * Ab : Arbitrated bandwidth
+ *
+ * IB_RECURRBLOCK and AB_RECURRBLOCK:
+ * These are used if the requirement is to transfer a
+ * recurring block of data over a known time window.
+ *
+ * IB_THROUGHPUTBW and AB_THROUGHPUTBW:
+ * These are used for CPU style masters. Here the requirement
+ * is to have minimum throughput bandwidth available to avoid
+ * stalling.
+ */
+#define IB_RECURRBLOCK(Ws, Bs) ((Ws) == 0 ? 0 : ((Bs)/(Ws)))
+#define AB_RECURRBLOCK(Ws, Per) ((Ws) == 0 ? 0 : ((Bs)/(Per)))
+#define IB_THROUGHPUTBW(Tb) (Tb)
+#define AB_THROUGHPUTBW(Tb, R) ((Tb) * (R))
+
+struct msm_bus_vectors {
+ int src; /* Master */
+ int dst; /* Slave */
+ uint64_t ab; /* Arbitrated bandwidth */
+ uint64_t ib; /* Instantaneous bandwidth */
+};
+
+struct msm_bus_paths {
+ int num_paths;
+ struct msm_bus_vectors *vectors;
+};
+
+struct msm_bus_scale_pdata {
+ struct msm_bus_paths *usecase;
+ int num_usecases;
+ const char *name;
+ /*
+ * If the active_only flag is set to 1, the BW request is applied
+ * only when at least one CPU is active (powered on). If the flag
+ * is set to 0, then the BW request is always applied irrespective
+ * of the CPU state.
+ */
+ unsigned int active_only;
+};
+
+struct msm_bus_client_handle {
+ char *name;
+ int mas;
+ int slv;
+ int first_hop;
+ struct device *mas_dev;
+ u64 cur_act_ib;
+ u64 cur_act_ab;
+ u64 cur_slp_ib;
+ u64 cur_slp_ab;
+ bool active_only;
+};
+
+/* Scaling APIs */
+
+/*
+ * This function returns a handle to the client. This should be used to
+ * call msm_bus_scale_client_update_request.
+ * The function returns 0 if bus driver is unable to register a client
+ */
+
+#if (defined(CONFIG_MSM_BUS_SCALING) || defined(CONFIG_BUS_TOPOLOGY_ADHOC))
+int __init msm_bus_fabric_init_driver(void);
+uint32_t msm_bus_scale_register_client(struct msm_bus_scale_pdata *pdata);
+int msm_bus_scale_client_update_request(uint32_t cl, unsigned int index);
+void msm_bus_scale_unregister_client(uint32_t cl);
+int msm_bus_scale_client_update_context(uint32_t cl, bool active_only,
+ unsigned int ctx_idx);
+
+struct msm_bus_client_handle*
+msm_bus_scale_register(uint32_t mas, uint32_t slv, char *name,
+ bool active_only);
+void msm_bus_scale_unregister(struct msm_bus_client_handle *cl);
+int msm_bus_scale_update_bw(struct msm_bus_client_handle *cl, u64 ab, u64 ib);
+int msm_bus_scale_update_bw_context(struct msm_bus_client_handle *cl,
+ u64 act_ab, u64 act_ib, u64 slp_ib, u64 slp_ab);
+/* AXI Port configuration APIs */
+int msm_bus_axi_porthalt(int master_port);
+int msm_bus_axi_portunhalt(int master_port);
+
+#else
+static inline int __init msm_bus_fabric_init_driver(void) { return 0; }
+static struct msm_bus_client_handle dummy_cl;
+
+static inline uint32_t
+msm_bus_scale_register_client(struct msm_bus_scale_pdata *pdata)
+{
+ return 1;
+}
+
+static inline int
+msm_bus_scale_client_update_request(uint32_t cl, unsigned int index)
+{
+ return 0;
+}
+
+static inline int
+msm_bus_scale_client_update_context(uint32_t cl, bool active_only,
+ unsigned int ctx_idx)
+{
+ return 0;
+}
+
+static inline void
+msm_bus_scale_unregister_client(uint32_t cl)
+{
+}
+
+static inline int msm_bus_axi_porthalt(int master_port)
+{
+ return 0;
+}
+
+static inline int msm_bus_axi_portunhalt(int master_port)
+{
+ return 0;
+}
+
+static inline struct msm_bus_client_handle*
+msm_bus_scale_register(uint32_t mas, uint32_t slv, char *name,
+ bool active_only)
+{
+ return &dummy_cl;
+}
+
+static inline void msm_bus_scale_unregister(struct msm_bus_client_handle *cl)
+{
+}
+
+static inline int
+msm_bus_scale_update_bw(struct msm_bus_client_handle *cl, u64 ab, u64 ib)
+{
+ return 0;
+}
+
+static inline int
+msm_bus_scale_update_bw_context(struct msm_bus_client_handle *cl, u64 act_ab,
+ u64 act_ib, u64 slp_ib, u64 slp_ab)
+
+{
+ return 0;
+}
+
+#endif
+
+#if defined(CONFIG_OF) && defined(CONFIG_MSM_BUS_SCALING)
+struct msm_bus_scale_pdata *msm_bus_pdata_from_node(
+ struct platform_device *pdev, struct device_node *of_node);
+struct msm_bus_scale_pdata *msm_bus_cl_get_pdata(struct platform_device *pdev);
+void msm_bus_cl_clear_pdata(struct msm_bus_scale_pdata *pdata);
+#else
+static inline struct msm_bus_scale_pdata
+*msm_bus_cl_get_pdata(struct platform_device *pdev)
+{
+ return NULL;
+}
+
+static inline struct msm_bus_scale_pdata *msm_bus_pdata_from_node(
+ struct platform_device *pdev, struct device_node *of_node)
+{
+ return NULL;
+}
+
+static inline void msm_bus_cl_clear_pdata(struct msm_bus_scale_pdata *pdata)
+{
+}
+#endif
+
+#ifdef CONFIG_DEBUG_BUS_VOTER
+int msm_bus_floor_vote_context(const char *name, u64 floor_hz,
+ bool active_only);
+int msm_bus_floor_vote(const char *name, u64 floor_hz);
+#else
+static inline int msm_bus_floor_vote(const char *name, u64 floor_hz)
+{
+ return -EINVAL;
+}
+
+static inline int msm_bus_floor_vote_context(const char *name, u64 floor_hz,
+ bool active_only)
+{
+ return -EINVAL;
+}
+#endif /*defined(CONFIG_DEBUG_BUS_VOTER) && defined(CONFIG_BUS_TOPOLOGY_ADHOC)*/
+#endif /*_ARCH_ARM_MACH_MSM_BUS_H*/
diff --git a/include/linux/msm_pcie.h b/include/linux/msm_pcie.h
new file mode 100644
index 000000000000..1f89cea940ea
--- /dev/null
+++ b/include/linux/msm_pcie.h
@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __MSM_PCIE_H
+#define __MSM_PCIE_H
+
+#include <linux/types.h>
+#include <linux/pci.h>
+
+enum msm_pcie_config {
+ MSM_PCIE_CONFIG_INVALID = 0,
+ MSM_PCIE_CONFIG_NO_CFG_RESTORE = 0x1,
+ MSM_PCIE_CONFIG_LINKDOWN = 0x2,
+ MSM_PCIE_CONFIG_NO_RECOVERY = 0x4,
+};
+
+enum msm_pcie_pm_opt {
+ MSM_PCIE_SUSPEND,
+ MSM_PCIE_RESUME,
+ MSM_PCIE_DISABLE_PC,
+ MSM_PCIE_ENABLE_PC,
+};
+
+enum msm_pcie_event {
+ MSM_PCIE_EVENT_INVALID = 0,
+ MSM_PCIE_EVENT_LINKDOWN = 0x1,
+ MSM_PCIE_EVENT_LINKUP = 0x2,
+ MSM_PCIE_EVENT_WAKEUP = 0x4,
+};
+
+enum msm_pcie_trigger {
+ MSM_PCIE_TRIGGER_CALLBACK,
+ MSM_PCIE_TRIGGER_COMPLETION,
+};
+
+struct msm_pcie_notify {
+ enum msm_pcie_event event;
+ void *user;
+ void *data;
+ u32 options;
+};
+
+struct msm_pcie_register_event {
+ u32 events;
+ void *user;
+ enum msm_pcie_trigger mode;
+ void (*callback)(struct msm_pcie_notify *notify);
+ struct msm_pcie_notify notify;
+ struct completion *completion;
+ u32 options;
+};
+
+/**
+ * msm_pcie_pm_control - control the power state of a PCIe link.
+ * @pm_opt: power management operation
+ * @busnr: bus number of PCIe endpoint
+ * @user: handle of the caller
+ * @data: private data from the caller
+ * @options: options for pm control
+ *
+ * This function gives PCIe endpoint device drivers the control to change
+ * the power state of a PCIe link for their device.
+ *
+ * Return: 0 on success, negative value on error
+ */
+int msm_pcie_pm_control(enum msm_pcie_pm_opt pm_opt, u32 busnr, void *user,
+ void *data, u32 options);
+
+/**
+ * msm_pcie_register_event - register an event with PCIe bus driver.
+ * @reg: event structure
+ *
+ * This function gives PCIe endpoint device drivers an option to register
+ * events with PCIe bus driver.
+ *
+ * Return: 0 on success, negative value on error
+ */
+int msm_pcie_register_event(struct msm_pcie_register_event *reg);
+
+/**
+ * msm_pcie_deregister_event - deregister an event with PCIe bus driver.
+ * @reg: event structure
+ *
+ * This function gives PCIe endpoint device drivers an option to deregister
+ * events with PCIe bus driver.
+ *
+ * Return: 0 on success, negative value on error
+ */
+int msm_pcie_deregister_event(struct msm_pcie_register_event *reg);
+
+/**
+ * msm_pcie_recover_config - recover config space.
+ * @dev: pci device structure
+ *
+ * This function recovers the config space of both RC and Endpoint.
+ *
+ * Return: 0 on success, negative value on error
+ */
+int msm_pcie_recover_config(struct pci_dev *dev);
+
+/**
+ * msm_pcie_enumerate - enumerate Endpoints.
+ * @rc_idx: RC that Endpoints connect to.
+ *
+ * This function enumerates Endpoints connected to RC.
+ *
+ * Return: 0 on success, negative value on error
+ */
+int msm_pcie_enumerate(u32 rc_idx);
+
+/**
+ * msm_pcie_recover_config - recover config space.
+ * @dev: pci device structure
+ *
+ * This function recovers the config space of both RC and Endpoint.
+ *
+ * Return: 0 on success, negative value on error
+ */
+int msm_pcie_recover_config(struct pci_dev *dev);
+
+/**
+ * msm_pcie_shadow_control - control the shadowing of PCIe config space.
+ * @dev: pci device structure
+ * @enable: shadowing should be enabled or disabled
+ *
+ * This function gives PCIe endpoint device drivers the control to enable
+ * or disable the shadowing of PCIe config space.
+ *
+ * Return: 0 on success, negative value on error
+ */
+int msm_pcie_shadow_control(struct pci_dev *dev, bool enable);
+
+/*
+ * msm_pcie_debug_info - run a PCIe specific debug testcase.
+ * @dev: pci device structure
+ * @option: specifies which PCIe debug testcase to execute
+ * @base: PCIe specific range
+ * @offset: offset of destination register
+ * @mask: mask the bit(s) of destination register
+ * @value: value to be written to destination register
+ *
+ * This function gives PCIe endpoint device drivers the control to
+ * run a debug testcase.
+ *
+ * Return: 0 on success, negative value on error
+ */
+int msm_pcie_debug_info(struct pci_dev *dev, u32 option, u32 base,
+ u32 offset, u32 mask, u32 value);
+
+/*
+ * msm_pcie_configure_sid - calculates the SID for a PCIe endpoint.
+ * @dev: device structure
+ * @sid: the calculated SID
+ * @domain: the domain number of the Root Complex
+ *
+ * This function calculates the SID for a PCIe endpoint device.
+ *
+ * Return: 0 on success, negative value on error
+ */
+int msm_pcie_configure_sid(struct device *dev, u32 *sid,
+ int *domain);
+#endif
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index c20f190b4c18..3602ee3c8039 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -72,6 +72,8 @@ void netdev_set_default_ethtool_ops(struct net_device *dev,
#define NET_RX_SUCCESS 0 /* keep 'em coming, baby */
#define NET_RX_DROP 1 /* packet dropped */
+#define MAX_NEST_DEV 8
+
/*
* Transmit return codes: transmit return codes originate from three different
* namespaces:
@@ -1761,7 +1763,7 @@ enum netdev_priv_flags {
* for hardware timestamping
* @sfp_bus: attached &struct sfp_bus structure.
* @qdisc_tx_busylock_key: lockdep class annotating Qdisc->busylock
- spinlock
+ * spinlock
* @qdisc_running_key: lockdep class annotating Qdisc->running seqcount
* @qdisc_xmit_lock_key: lockdep class annotating
* netdev_queue->_xmit_lock spinlock
@@ -1867,6 +1869,11 @@ struct net_device {
unsigned char if_port;
unsigned char dma;
+ /* Note : dev->mtu is often read without holding a lock.
+ * Writers usually hold RTNL.
+ * It is recommended to use READ_ONCE() to annotate the reads,
+ * and to use WRITE_ONCE() to annotate the writes.
+ */
unsigned int mtu;
unsigned int min_mtu;
unsigned int max_mtu;
@@ -2973,6 +2980,7 @@ extern int netdev_flow_limit_table_len;
*/
struct softnet_data {
struct list_head poll_list;
+ struct napi_struct *current_napi;
struct sk_buff_head process_queue;
/* stats */
@@ -3631,6 +3639,7 @@ struct sk_buff *napi_get_frags(struct napi_struct *napi);
gro_result_t napi_gro_frags(struct napi_struct *napi);
struct packet_offload *gro_find_receive_by_type(__be16 type);
struct packet_offload *gro_find_complete_by_type(__be16 type);
+extern struct napi_struct *get_current_napi_context(void);
static inline void napi_free_frags(struct napi_struct *napi)
{
@@ -3661,6 +3670,8 @@ int dev_set_alias(struct net_device *, const char *, size_t);
int dev_get_alias(const struct net_device *, char *, size_t);
int dev_change_net_namespace(struct net_device *, struct net *, const char *);
int __dev_set_mtu(struct net_device *, int);
+int dev_validate_mtu(struct net_device *dev, int mtu,
+ struct netlink_ext_ack *extack);
int dev_set_mtu_ext(struct net_device *dev, int mtu,
struct netlink_ext_ack *extack);
int dev_set_mtu(struct net_device *, int);
@@ -4287,11 +4298,8 @@ void *netdev_lower_get_next(struct net_device *dev,
ldev; \
ldev = netdev_lower_get_next(dev, &(iter)))
-struct net_device *netdev_all_lower_get_next(struct net_device *dev,
+struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
struct list_head **iter);
-struct net_device *netdev_all_lower_get_next_rcu(struct net_device *dev,
- struct list_head **iter);
-
int netdev_walk_all_lower_dev(struct net_device *dev,
int (*fn)(struct net_device *lower_dev,
void *data),
diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index 4a8c02cafa9a..6fa81e2db70f 100644
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -535,6 +535,7 @@ enum {
IFLA_VXLAN_GPE,
IFLA_VXLAN_TTL_INHERIT,
IFLA_VXLAN_DF,
+ IFLA_VXLAN_FAN_MAP = 33,
__IFLA_VXLAN_MAX
};
#define IFLA_VXLAN_MAX (__IFLA_VXLAN_MAX - 1)
@@ -1013,6 +1014,9 @@ enum {
#define RMNET_FLAGS_INGRESS_MAP_COMMANDS (1U << 1)
#define RMNET_FLAGS_INGRESS_MAP_CKSUMV4 (1U << 2)
#define RMNET_FLAGS_EGRESS_MAP_CKSUMV4 (1U << 3)
+#define RMNET_FLAGS_INGRESS_COALESCE (1U << 4)
+#define RMNET_FLAGS_INGRESS_MAP_CKSUMV5 (1U << 5)
+#define RMNET_FLAGS_EGRESS_MAP_CKSUMV5 (1U << 6)
enum {
IFLA_RMNET_UNSPEC,
diff --git a/include/uapi/linux/msm_rmnet.h b/include/uapi/linux/msm_rmnet.h
new file mode 100644
index 000000000000..5bad7bf02235
--- /dev/null
+++ b/include/uapi/linux/msm_rmnet.h
@@ -0,0 +1,170 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2012-2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+#ifndef _UAPI_MSM_RMNET_H_
+#define _UAPI_MSM_RMNET_H_
+/* Bitmap macros for RmNET driver operation mode. */
+#define RMNET_MODE_NONE (0x00)
+#define RMNET_MODE_LLP_ETH (0x01)
+#define RMNET_MODE_LLP_IP (0x02)
+#define RMNET_MODE_QOS (0x04)
+#define RMNET_MODE_MASK (RMNET_MODE_LLP_ETH | \
+ RMNET_MODE_LLP_IP | \
+ RMNET_MODE_QOS)
+
+#define RMNET_IS_MODE_QOS(mode) \
+ ((mode & RMNET_MODE_QOS) == RMNET_MODE_QOS)
+#define RMNET_IS_MODE_IP(mode) \
+ ((mode & RMNET_MODE_LLP_IP) == RMNET_MODE_LLP_IP)
+
+/* IOCTL command enum
+ * Values chosen to not conflict with other drivers in the ecosystem
+ */
+enum rmnet_ioctl_cmds_e {
+ RMNET_IOCTL_SET_LLP_ETHERNET = 0x000089F1, /* Set Ethernet protocol */
+ RMNET_IOCTL_SET_LLP_IP = 0x000089F2, /* Set RAWIP protocol */
+ RMNET_IOCTL_GET_LLP = 0x000089F3, /* Get link protocol */
+ RMNET_IOCTL_SET_QOS_ENABLE = 0x000089F4, /* Set QoS header enabled */
+ RMNET_IOCTL_SET_QOS_DISABLE = 0x000089F5, /* Set QoS header disabled*/
+ RMNET_IOCTL_GET_QOS = 0x000089F6, /* Get QoS header state */
+ RMNET_IOCTL_GET_OPMODE = 0x000089F7, /* Get operation mode */
+ RMNET_IOCTL_OPEN = 0x000089F8, /* Open transport port */
+ RMNET_IOCTL_CLOSE = 0x000089F9, /* Close transport port */
+ RMNET_IOCTL_FLOW_ENABLE = 0x000089FA, /* Flow enable */
+ RMNET_IOCTL_FLOW_DISABLE = 0x000089FB, /* Flow disable */
+ RMNET_IOCTL_FLOW_SET_HNDL = 0x000089FC, /* Set flow handle */
+ RMNET_IOCTL_EXTENDED = 0x000089FD, /* Extended IOCTLs */
+ RMNET_IOCTL_MAX
+};
+
+enum rmnet_ioctl_extended_cmds_e {
+/* RmNet Data Required IOCTLs */
+ RMNET_IOCTL_GET_SUPPORTED_FEATURES = 0x0000, /* Get features */
+ RMNET_IOCTL_SET_MRU = 0x0001, /* Set MRU */
+ RMNET_IOCTL_GET_MRU = 0x0002, /* Get MRU */
+ RMNET_IOCTL_GET_EPID = 0x0003, /* Get endpoint ID */
+ RMNET_IOCTL_GET_DRIVER_NAME = 0x0004, /* Get driver name */
+ RMNET_IOCTL_ADD_MUX_CHANNEL = 0x0005, /* Add MUX ID */
+ RMNET_IOCTL_SET_EGRESS_DATA_FORMAT = 0x0006, /* Set EDF */
+ RMNET_IOCTL_SET_INGRESS_DATA_FORMAT = 0x0007, /* Set IDF */
+ RMNET_IOCTL_SET_AGGREGATION_COUNT = 0x0008, /* Set agg count */
+ RMNET_IOCTL_GET_AGGREGATION_COUNT = 0x0009, /* Get agg count */
+ RMNET_IOCTL_SET_AGGREGATION_SIZE = 0x000A, /* Set agg size */
+ RMNET_IOCTL_GET_AGGREGATION_SIZE = 0x000B, /* Get agg size */
+ RMNET_IOCTL_FLOW_CONTROL = 0x000C, /* Do flow control */
+ RMNET_IOCTL_GET_DFLT_CONTROL_CHANNEL = 0x000D, /* For legacy use */
+ RMNET_IOCTL_GET_HWSW_MAP = 0x000E, /* Get HW/SW map */
+ RMNET_IOCTL_SET_RX_HEADROOM = 0x000F, /* RX Headroom */
+ RMNET_IOCTL_GET_EP_PAIR = 0x0010, /* Endpoint pair */
+ RMNET_IOCTL_SET_QOS_VERSION = 0x0011, /* 8/6 byte QoS hdr*/
+ RMNET_IOCTL_GET_QOS_VERSION = 0x0012, /* 8/6 byte QoS hdr*/
+ RMNET_IOCTL_GET_SUPPORTED_QOS_MODES = 0x0013, /* Get QoS modes */
+ RMNET_IOCTL_SET_SLEEP_STATE = 0x0014, /* Set sleep state */
+ RMNET_IOCTL_SET_XLAT_DEV_INFO = 0x0015, /* xlat dev name */
+ RMNET_IOCTL_DEREGISTER_DEV = 0x0016, /* Dereg a net dev */
+ RMNET_IOCTL_GET_SG_SUPPORT = 0x0017, /* Query sg support*/
+ RMNET_IOCTL_EXTENDED_MAX = 0x0018
+};
+
+/* Return values for the RMNET_IOCTL_GET_SUPPORTED_FEATURES IOCTL */
+#define RMNET_IOCTL_FEAT_NOTIFY_MUX_CHANNEL (1<<0)
+#define RMNET_IOCTL_FEAT_SET_EGRESS_DATA_FORMAT (1<<1)
+#define RMNET_IOCTL_FEAT_SET_INGRESS_DATA_FORMAT (1<<2)
+#define RMNET_IOCTL_FEAT_SET_AGGREGATION_COUNT (1<<3)
+#define RMNET_IOCTL_FEAT_GET_AGGREGATION_COUNT (1<<4)
+#define RMNET_IOCTL_FEAT_SET_AGGREGATION_SIZE (1<<5)
+#define RMNET_IOCTL_FEAT_GET_AGGREGATION_SIZE (1<<6)
+#define RMNET_IOCTL_FEAT_FLOW_CONTROL (1<<7)
+#define RMNET_IOCTL_FEAT_GET_DFLT_CONTROL_CHANNEL (1<<8)
+#define RMNET_IOCTL_FEAT_GET_HWSW_MAP (1<<9)
+
+/* Input values for the RMNET_IOCTL_SET_EGRESS_DATA_FORMAT IOCTL */
+#define RMNET_IOCTL_EGRESS_FORMAT_MAP (1<<1)
+#define RMNET_IOCTL_EGRESS_FORMAT_AGGREGATION (1<<2)
+#define RMNET_IOCTL_EGRESS_FORMAT_MUXING (1<<3)
+#define RMNET_IOCTL_EGRESS_FORMAT_CHECKSUM (1<<4)
+
+/* Input values for the RMNET_IOCTL_SET_INGRESS_DATA_FORMAT IOCTL */
+#define RMNET_IOCTL_INGRESS_FORMAT_MAP (1<<1)
+#define RMNET_IOCTL_INGRESS_FORMAT_DEAGGREGATION (1<<2)
+#define RMNET_IOCTL_INGRESS_FORMAT_DEMUXING (1<<3)
+#define RMNET_IOCTL_INGRESS_FORMAT_CHECKSUM (1<<4)
+#define RMNET_IOCTL_INGRESS_FORMAT_AGG_DATA (1<<5)
+
+/* User space may not have this defined. */
+#ifndef IFNAMSIZ
+#define IFNAMSIZ 16
+#endif
+
+struct rmnet_ioctl_extended_s {
+ uint32_t extended_ioctl;
+ union {
+ uint32_t data; /* Generic data field for most extended IOCTLs */
+
+ /* Return values for
+ * RMNET_IOCTL_GET_DRIVER_NAME
+ * RMNET_IOCTL_GET_DFLT_CONTROL_CHANNEL
+ */
+ int8_t if_name[IFNAMSIZ];
+
+ /* Input values for the RMNET_IOCTL_ADD_MUX_CHANNEL IOCTL */
+ struct {
+ uint32_t mux_id;
+ int8_t vchannel_name[IFNAMSIZ];
+ } rmnet_mux_val;
+
+ /* Input values for the RMNET_IOCTL_FLOW_CONTROL IOCTL */
+ struct {
+ uint8_t flow_mode;
+ uint8_t mux_id;
+ } flow_control_prop;
+
+ /* Return values for RMNET_IOCTL_GET_EP_PAIR */
+ struct {
+ uint32_t consumer_pipe_num;
+ uint32_t producer_pipe_num;
+ } ipa_ep_pair;
+
+ struct {
+ uint32_t __data; /* Placeholder for legacy data*/
+ uint32_t agg_size;
+ uint32_t agg_count;
+ } ingress_format;
+ } u;
+};
+
+struct rmnet_ioctl_data_s {
+ union {
+ uint32_t operation_mode;
+ uint32_t tcm_handle;
+ } u;
+};
+
+#define RMNET_IOCTL_QOS_MODE_6 (1<<0)
+#define RMNET_IOCTL_QOS_MODE_8 (1<<1)
+
+/* QMI QoS header definition */
+#define QMI_QOS_HDR_S qmi_qos_hdr_s
+struct QMI_QOS_HDR_S {
+ unsigned char version;
+ unsigned char flags;
+ uint32_t flow_id;
+} __attribute((__packed__));
+
+/* QMI QoS 8-byte header. */
+struct qmi_qos_hdr8_s {
+ struct QMI_QOS_HDR_S hdr;
+ uint8_t reserved[2];
+} __attribute((__packed__));
+
+#endif /* _UAPI_MSM_RMNET_H_ */
diff --git a/mm/memblock.c b/mm/memblock.c
index c4b16cae2bc9..ad0312d7fe8e 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -1616,6 +1616,7 @@ phys_addr_t __init_memblock memblock_start_of_DRAM(void)
{
return memblock.memory.regions[0].base;
}
+EXPORT_SYMBOL(memblock_start_of_DRAM);
phys_addr_t __init_memblock memblock_end_of_DRAM(void)
{
@@ -1623,6 +1624,7 @@ phys_addr_t __init_memblock memblock_end_of_DRAM(void)
return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
}
+EXPORT_SYMBOL(memblock_end_of_DRAM);
static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
{
diff --git a/net/core/dev.c b/net/core/dev.c
index 99ac84ff398f..c00251780581 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -146,7 +146,6 @@
#include "net-sysfs.h"
#define MAX_GRO_SKBS 8
-#define MAX_NEST_DEV 8
/* This should be increased if a protocol with a bigger head is added. */
#define GRO_MAX_HEAD (MAX_HEADER + 128)
@@ -2796,6 +2795,8 @@ static u16 skb_tx_hash(const struct net_device *dev,
if (skb_rx_queue_recorded(skb)) {
hash = skb_get_rx_queue(skb);
+ if (hash >= qoffset)
+ hash -= qoffset;
while (unlikely(hash >= qcount))
hash -= qcount;
return hash + qoffset;
@@ -3386,26 +3387,8 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
qdisc_calculate_pkt_len(skb, q);
if (q->flags & TCQ_F_NOLOCK) {
- if ((q->flags & TCQ_F_CAN_BYPASS) && q->empty &&
- qdisc_run_begin(q)) {
- if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
- &q->state))) {
- __qdisc_drop(skb, &to_free);
- rc = NET_XMIT_DROP;
- goto end_run;
- }
- qdisc_bstats_cpu_update(q, skb);
-
- rc = NET_XMIT_SUCCESS;
- if (sch_direct_xmit(skb, q, dev, txq, NULL, true))
- __qdisc_run(q);
-
-end_run:
- qdisc_run_end(q);
- } else {
- rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
- qdisc_run(q);
- }
+ rc = q->enqueue(skb, q, &to_free) & NET_XMIT_MASK;
+ qdisc_run(q);
if (unlikely(to_free))
kfree_skb_list(to_free);
@@ -3880,7 +3863,8 @@ EXPORT_SYMBOL(netdev_max_backlog);
int netdev_tstamp_prequeue __read_mostly = 1;
int netdev_budget __read_mostly = 300;
-unsigned int __read_mostly netdev_budget_usecs = 2000;
+/* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
+unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
int weight_p __read_mostly = 64; /* old backlog weight */
int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
@@ -4256,14 +4240,14 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb,
/* Reinjected packets coming from act_mirred or similar should
* not get XDP generic processing.
*/
- if (skb_cloned(skb) || skb_is_tc_redirected(skb))
+ if (skb_is_redirected(skb))
return XDP_PASS;
/* XDP packets must be linear and must have sufficient headroom
* of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
* native XDP provides, thus we need to do it here as well.
*/
- if (skb_is_nonlinear(skb) ||
+ if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
skb_headroom(skb) < XDP_PACKET_HEADROOM) {
int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
int troom = skb->tail + skb->data_len - skb->end;
@@ -4729,11 +4713,12 @@ static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
return 0;
}
-static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc,
+static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
struct packet_type **ppt_prev)
{
struct packet_type *ptype, *pt_prev;
rx_handler_func_t *rx_handler;
+ struct sk_buff *skb = *pskb;
struct net_device *orig_dev;
bool deliver_exact = false;
int ret = NET_RX_DROP;
@@ -4764,8 +4749,10 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc,
ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
preempt_enable();
- if (ret2 != XDP_PASS)
- return NET_RX_DROP;
+ if (ret2 != XDP_PASS) {
+ ret = NET_RX_DROP;
+ goto out;
+ }
skb_reset_mac_len(skb);
}
@@ -4805,7 +4792,7 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc,
goto out;
}
#endif
- skb_reset_tc(skb);
+ skb_reset_redirect(skb);
skip_classify:
if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
goto drop;
@@ -4915,6 +4902,13 @@ static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc,
}
out:
+ /* The invariant here is that if *ppt_prev is not NULL
+ * then skb should also be non-NULL.
+ *
+ * Apparently *ppt_prev assignment above holds this invariant due to
+ * skb dereferencing near it.
+ */
+ *pskb = skb;
return ret;
}
@@ -4924,7 +4918,7 @@ static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
struct packet_type *pt_prev = NULL;
int ret;
- ret = __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);
+ ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
if (pt_prev)
ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
skb->dev, pt_prev, orig_dev);
@@ -5002,7 +4996,7 @@ static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemallo
struct packet_type *pt_prev = NULL;
skb_list_del_init(skb);
- __netif_receive_skb_core(skb, pfmemalloc, &pt_prev);
+ __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
if (!pt_prev)
continue;
if (pt_curr != pt_prev || od_curr != orig_dev) {
@@ -5270,9 +5264,29 @@ static void flush_all_backlogs(void)
put_online_cpus();
}
+/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
+static void gro_normal_list(struct napi_struct *napi)
+{
+ if (!napi->rx_count)
+ return;
+ netif_receive_skb_list_internal(&napi->rx_list);
+ INIT_LIST_HEAD(&napi->rx_list);
+ napi->rx_count = 0;
+}
+
+/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
+ * pass the whole batch up to the stack.
+ */
+static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb)
+{
+ list_add_tail(&skb->list, &napi->rx_list);
+ if (++napi->rx_count >= gro_normal_batch)
+ gro_normal_list(napi);
+}
+
INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));
INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));
-static int napi_gro_complete(struct sk_buff *skb)
+static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
{
struct packet_offload *ptype;
__be16 type = skb->protocol;
@@ -5305,7 +5319,8 @@ static int napi_gro_complete(struct sk_buff *skb)
}
out:
- return netif_receive_skb_internal(skb);
+ gro_normal_one(napi, skb);
+ return NET_RX_SUCCESS;
}
static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
@@ -5318,7 +5333,7 @@ static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
return;
skb_list_del_init(skb);
- napi_gro_complete(skb);
+ napi_gro_complete(napi, skb);
napi->gro_hash[index].count--;
}
@@ -5421,7 +5436,7 @@ static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
}
}
-static void gro_flush_oldest(struct list_head *head)
+static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head)
{
struct sk_buff *oldest;
@@ -5437,7 +5452,7 @@ static void gro_flush_oldest(struct list_head *head)
* SKB to the chain.
*/
skb_list_del_init(oldest);
- napi_gro_complete(oldest);
+ napi_gro_complete(napi, oldest);
}
INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,
@@ -5513,7 +5528,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
if (pp) {
skb_list_del_init(pp);
- napi_gro_complete(pp);
+ napi_gro_complete(napi, pp);
napi->gro_hash[hash].count--;
}
@@ -5524,7 +5539,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
goto normal;
if (unlikely(napi->gro_hash[hash].count >= MAX_GRO_SKBS)) {
- gro_flush_oldest(gro_head);
+ gro_flush_oldest(napi, gro_head);
} else {
napi->gro_hash[hash].count++;
}
@@ -5672,26 +5687,6 @@ struct sk_buff *napi_get_frags(struct napi_struct *napi)
}
EXPORT_SYMBOL(napi_get_frags);
-/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
-static void gro_normal_list(struct napi_struct *napi)
-{
- if (!napi->rx_count)
- return;
- netif_receive_skb_list_internal(&napi->rx_list);
- INIT_LIST_HEAD(&napi->rx_list);
- napi->rx_count = 0;
-}
-
-/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
- * pass the whole batch up to the stack.
- */
-static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb)
-{
- list_add_tail(&skb->list, &napi->rx_list);
- if (++napi->rx_count >= gro_normal_batch)
- gro_normal_list(napi);
-}
-
static gro_result_t napi_frags_finish(struct napi_struct *napi,
struct sk_buff *skb,
gro_result_t ret)
@@ -5979,8 +5974,6 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
NAPIF_STATE_IN_BUSY_POLL)))
return false;
- gro_normal_list(n);
-
if (n->gro_bitmask) {
unsigned long timeout = 0;
@@ -5996,6 +5989,9 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
hrtimer_start(&n->timer, ns_to_ktime(timeout),
HRTIMER_MODE_REL_PINNED);
}
+
+ gro_normal_list(n);
+
if (unlikely(!list_empty(&n->poll_list))) {
/* If n->poll_list is not empty, we need to mask irqs */
local_irq_save(flags);
@@ -6289,6 +6285,14 @@ void netif_napi_del(struct napi_struct *napi)
}
EXPORT_SYMBOL(netif_napi_del);
+struct napi_struct *get_current_napi_context(void)
+{
+ struct softnet_data *sd = this_cpu_ptr(&softnet_data);
+
+ return sd->current_napi;
+}
+EXPORT_SYMBOL(get_current_napi_context);
+
static int napi_poll(struct napi_struct *n, struct list_head *repoll)
{
void *have;
@@ -6327,8 +6331,6 @@ static int napi_poll(struct napi_struct *n, struct list_head *repoll)
goto out_unlock;
}
- gro_normal_list(n);
-
if (n->gro_bitmask) {
/* flush too old packets
* If HZ < 1000, flush all packets.
@@ -6336,6 +6338,8 @@ static int napi_poll(struct napi_struct *n, struct list_head *repoll)
napi_gro_flush(n, HZ >= 1000);
}
+ gro_normal_list(n);
+
/* Some drivers may have called napi_schedule
* prior to exhausting their budget.
*/
@@ -6930,8 +6934,8 @@ static int __netdev_walk_all_lower_dev(struct net_device *dev,
return 0;
}
-static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
- struct list_head **iter)
+struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
+ struct list_head **iter)
{
struct netdev_adjacent *lower;
@@ -6943,6 +6947,7 @@ static struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
return lower->dev;
}
+EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
static u8 __netdev_upper_depth(struct net_device *dev)
{
@@ -7967,11 +7972,28 @@ int __dev_set_mtu(struct net_device *dev, int new_mtu)
if (ops->ndo_change_mtu)
return ops->ndo_change_mtu(dev, new_mtu);
- dev->mtu = new_mtu;
+ /* Pairs with all the lockless reads of dev->mtu in the stack */
+ WRITE_ONCE(dev->mtu, new_mtu);
return 0;
}
EXPORT_SYMBOL(__dev_set_mtu);
+int dev_validate_mtu(struct net_device *dev, int new_mtu,
+ struct netlink_ext_ack *extack)
+{
+ /* MTU must be positive, and in range */
+ if (new_mtu < 0 || new_mtu < dev->min_mtu) {
+ NL_SET_ERR_MSG(extack, "mtu less than device minimum");
+ return -EINVAL;
+ }
+
+ if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
+ NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
+ return -EINVAL;
+ }
+ return 0;
+}
+
/**
* dev_set_mtu_ext - Change maximum transfer unit
* @dev: device
@@ -7988,16 +8010,9 @@ int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
if (new_mtu == dev->mtu)
return 0;
- /* MTU must be positive, and in range */
- if (new_mtu < 0 || new_mtu < dev->min_mtu) {
- NL_SET_ERR_MSG(extack, "mtu less than device minimum");
- return -EINVAL;
- }
-
- if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
- NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
- return -EINVAL;
- }
+ err = dev_validate_mtu(dev, new_mtu, extack);
+ if (err)
+ return err;
if (!netif_device_present(dev))
return -ENODEV;
@@ -8598,11 +8613,13 @@ static void netdev_sync_lower_features(struct net_device *upper,
netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
&feature, lower->name);
lower->wanted_features &= ~feature;
- netdev_update_features(lower);
+ __netdev_update_features(lower);
if (unlikely(lower->features & feature))
netdev_WARN(upper, "failed to disable %pNF on %s!\n",
&feature, lower->name);
+ else
+ netdev_features_change(lower);
}
}
}
@@ -8952,22 +8969,10 @@ static void netdev_unregister_lockdep_key(struct net_device *dev)
void netdev_update_lockdep_key(struct net_device *dev)
{
- struct netdev_queue *queue;
- int i;
-
- lockdep_unregister_key(&dev->qdisc_xmit_lock_key);
lockdep_unregister_key(&dev->addr_list_lock_key);
-
- lockdep_register_key(&dev->qdisc_xmit_lock_key);
lockdep_register_key(&dev->addr_list_lock_key);
lockdep_set_class(&dev->addr_list_lock, &dev->addr_list_lock_key);
- for (i = 0; i < dev->num_tx_queues; i++) {
- queue = netdev_get_tx_queue(dev, i);
-
- lockdep_set_class(&queue->_xmit_lock,
- &dev->qdisc_xmit_lock_key);
- }
}
EXPORT_SYMBOL(netdev_update_lockdep_key);
@@ -9084,8 +9089,10 @@ int register_netdevice(struct net_device *dev)
goto err_uninit;
ret = netdev_register_kobject(dev);
- if (ret)
+ if (ret) {
+ dev->reg_state = NETREG_UNREGISTERED;
goto err_uninit;
+ }
dev->reg_state = NETREG_REGISTERED;
__netdev_update_features(dev);
@@ -9715,6 +9722,7 @@ EXPORT_SYMBOL(unregister_netdev);
int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
{
+ struct net *net_old = dev_net(dev);
int err, new_nsid, new_ifindex;
ASSERT_RTNL();
@@ -9730,7 +9738,7 @@ int dev_change_net_namespace(struct net_device *dev, struct net *net, const char
/* Get out if there is nothing todo */
err = 0;
- if (net_eq(dev_net(dev), net))
+ if (net_eq(net_old, net))
goto out;
/* Pick the destination device name, and ensure
@@ -9803,6 +9811,12 @@ int dev_change_net_namespace(struct net_device *dev, struct net *net, const char
err = device_rename(&dev->dev, dev->name);
WARN_ON(err);
+ /* Adapt owner in case owning user namespace of target network
+ * namespace is different from the original one.
+ */
+ err = netdev_change_owner(dev, net_old, net);
+ WARN_ON(err);
+
/* Add the device back in the hashes */
list_netdevice(dev);
--
2.25.1
Powered by blists - more mailing lists