| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20251223035113.31122-10-illusion.wang@nebula-matrix.com>
Date: Tue, 23 Dec 2025 11:50:32 +0800
From: "illusion.wang" <illusion.wang@...ula-matrix.com>
To: dimon.zhao@...ula-matrix.com,
illusion.wang@...ula-matrix.com,
alvin.wang@...ula-matrix.com,
sam.chen@...ula-matrix.com,
netdev@...r.kernel.org
Cc: linux-kernel@...r.kernel.org (open list)
Subject: [PATCH v1 net-next 09/15] net/nebula-matrix: add txrx resource definitions and implementation
txrx resource management functions include:
TX/RX Ring Management: Allocate/release DMA memory and descriptors.
Data Transmission: Support TSO (TCP Segmentation Offload), checksum offloading, and
VLAN tag insertion.
Data Reception: Support NAPI interrupt aggregation, checksum offloading, and VLAN tag stripping.
Resource Management: Cache receive buffers and pre-allocate resources.
Statistics and Debugging: Collect transmission/reception statistics.
Signed-off-by: illusion.wang <illusion.wang@...ula-matrix.com>
Change-Id: I269e5d6e28a0adda0be6aaca0b81f00730ae0c40
---
.../net/ethernet/nebula-matrix/nbl/Makefile | 1 +
.../net/ethernet/nebula-matrix/nbl/nbl_core.h | 17 +
.../nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c | 19 +
.../nbl_hw_leonis/nbl_resource_leonis.c | 13 +-
.../nebula-matrix/nbl/nbl_hw/nbl_resource.h | 4 +
.../nebula-matrix/nbl/nbl_hw/nbl_txrx.c | 2026 +++++++++++++++++
.../nebula-matrix/nbl/nbl_hw/nbl_txrx.h | 188 ++
.../nbl/nbl_include/nbl_def_hw.h | 3 +
8 files changed, 2270 insertions(+), 1 deletion(-)
create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_txrx.c
create mode 100644 drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_txrx.h
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/Makefile b/drivers/net/ethernet/nebula-matrix/nbl/Makefile
index 54372a723455..96c265b8bf79 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/Makefile
+++ b/drivers/net/ethernet/nebula-matrix/nbl/Makefile
@@ -13,6 +13,7 @@ nbl_core-objs += nbl_common/nbl_common.o \
nbl_hw/nbl_hw_leonis/nbl_hw_leonis_regs.o \
nbl_hw/nbl_resource.o \
nbl_hw/nbl_interrupt.o \
+ nbl_hw/nbl_txrx.o \
nbl_hw/nbl_queue.o \
nbl_hw/nbl_vsi.o \
nbl_hw/nbl_adminq.o \
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h
index 6f4bfceaf995..aea8fed8ff11 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h
@@ -26,6 +26,11 @@
#define NBL_ADAPTER_TO_CHAN_OPS_TBL(adapter) ((adapter)->intf.channel_ops_tbl)
#define NBL_ADAPTER_TO_RES_PT_OPS(adapter) (&(NBL_ADAPTER_TO_SERV_OPS_TBL(adapter)->pt_ops))
+
+#define NBL_NETDEV_PRIV_TO_ADAPTER(priv) ((priv)->adapter)
+
+#define NBL_NETDEV_TO_ADAPTER(netdev) \
+ (NBL_NETDEV_PRIV_TO_ADAPTER((struct nbl_netdev_priv *)netdev_priv(netdev)))
#define NBL_CAP_SET_BIT(loc) (1 << (loc))
#define NBL_CAP_TEST_BIT(val, loc) (((val) >> (loc)) & 0x1)
@@ -85,6 +90,18 @@ struct nbl_adapter {
DECLARE_BITMAP(state, NBL_STATE_NBITS);
};
+struct nbl_netdev_priv {
+ struct nbl_adapter *adapter;
+ struct net_device *netdev;
+ u16 tx_queue_num;
+ u16 rx_queue_num;
+ u16 queue_size;
+ /* default traffic destination in kernel/dpdk/coexist scene */
+ u16 data_vsi;
+ u16 user_vsi;
+ s64 last_st_time;
+};
+
struct nbl_adapter *nbl_core_init(struct pci_dev *pdev, struct nbl_init_param *param);
void nbl_core_remove(struct nbl_adapter *adapter);
#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c
index 363abf326031..cc09abd15408 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c
@@ -2307,6 +2307,23 @@ static void nbl_hw_cfg_mailbox_qinfo(void *priv, u16 func_id, u16 bus, u16 devid
(u8 *)&mb_qinfo_map, sizeof(mb_qinfo_map));
}
+static void nbl_hw_update_tail_ptr(void *priv, struct nbl_notify_param *param)
+{
+ struct nbl_hw_mgt *hw_mgt = (struct nbl_hw_mgt *)priv;
+ u8 __iomem *notify_addr = hw_mgt->hw_addr;
+ u32 local_qid = param->notify_qid;
+ u32 tail_ptr = param->tail_ptr;
+
+ writel((((u32)tail_ptr << 16) | (u32)local_qid), notify_addr);
+}
+
+static u8 *nbl_hw_get_tail_ptr(void *priv)
+{
+ struct nbl_hw_mgt *hw_mgt = (struct nbl_hw_mgt *)priv;
+
+ return hw_mgt->hw_addr;
+}
+
static void nbl_hw_set_promisc_mode(void *priv, u16 vsi_id, u16 eth_id, u16 mode)
{
struct nbl_ipro_upsport_tbl upsport;
@@ -2860,6 +2877,8 @@ static struct nbl_hw_ops hw_ops = {
.update_adminq_queue_tail_ptr = nbl_hw_update_adminq_queue_tail_ptr,
.check_adminq_dma_err = nbl_hw_check_adminq_dma_err,
+ .update_tail_ptr = nbl_hw_update_tail_ptr,
+ .get_tail_ptr = nbl_hw_get_tail_ptr,
.set_spoof_check_addr = nbl_hw_set_spoof_check_addr,
.set_spoof_check_enable = nbl_hw_set_spoof_check_enable,
.get_hw_addr = nbl_hw_get_hw_addr,
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c
index 8020bdaeceea..87c77e55415f 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c
@@ -574,6 +574,11 @@ static int nbl_res_setup_ops(struct device *dev, struct nbl_resource_ops_tbl **r
ret = nbl_queue_setup_ops_leonis(&res_ops);
if (ret)
goto setup_fail;
+
+ ret = nbl_txrx_setup_ops(&res_ops);
+ if (ret)
+ goto setup_fail;
+
ret = nbl_intr_setup_ops(&res_ops);
if (ret)
goto setup_fail;
@@ -887,6 +892,7 @@ static void nbl_res_stop(struct nbl_resource_mgt_leonis *res_mgt_leonis)
struct nbl_resource_mgt *res_mgt = &res_mgt_leonis->res_mgt;
nbl_queue_mgt_stop(res_mgt);
+ nbl_txrx_mgt_stop(res_mgt);
nbl_intr_mgt_stop(res_mgt);
nbl_adminq_mgt_stop(res_mgt);
nbl_vsi_mgt_stop(res_mgt);
@@ -972,7 +978,12 @@ static int nbl_res_start(struct nbl_resource_mgt_leonis *res_mgt_leonis,
nbl_res_set_fix_capability(res_mgt, NBL_HIGH_THROUGHPUT_CAP);
nbl_res_set_fix_capability(res_mgt, NBL_DVN_DESC_REQ_SYSFS_CAP);
- nbl_res_set_fix_capability(res_mgt, NBL_NEED_DESTROY_CHIP);
+ }
+
+ if (caps.has_net) {
+ ret = nbl_txrx_mgt_start(res_mgt);
+ if (ret)
+ goto start_fail;
}
nbl_res_set_fix_capability(res_mgt, NBL_HWMON_TEMP_CAP);
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h
index 21f9444822d8..cd722964b75c 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h
@@ -832,6 +832,10 @@ int nbl_intr_setup_ops(struct nbl_resource_ops *resource_ops);
int nbl_queue_mgt_start(struct nbl_resource_mgt *res_mgt);
void nbl_queue_mgt_stop(struct nbl_resource_mgt *res_mgt);
+int nbl_txrx_mgt_start(struct nbl_resource_mgt *res_mgt);
+void nbl_txrx_mgt_stop(struct nbl_resource_mgt *res_mgt);
+int nbl_txrx_setup_ops(struct nbl_resource_ops *resource_ops);
+
int nbl_vsi_mgt_start(struct nbl_resource_mgt *res_mgt);
void nbl_vsi_mgt_stop(struct nbl_resource_mgt *res_mgt);
int nbl_vsi_setup_ops(struct nbl_resource_ops *resource_ops);
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_txrx.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_txrx.c
new file mode 100644
index 000000000000..e79e07e6e918
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_txrx.c
@@ -0,0 +1,2026 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2025 Nebula Matrix Limited.
+ * Author:
+ */
+
+#include "nbl_txrx.h"
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <net/ipv6.h>
+#include <linux/sctp.h>
+#include <linux/if_vlan.h>
+#include <net/page_pool/helpers.h>
+
+static bool nbl_txrx_within_vsi(struct nbl_txrx_vsi_info *vsi_info, u16 ring_index)
+{
+ return ring_index >= vsi_info->ring_offset &&
+ ring_index < vsi_info->ring_offset + vsi_info->ring_num;
+}
+
+static struct netdev_queue *txring_txq(const struct nbl_res_tx_ring *ring)
+{
+ return netdev_get_tx_queue(ring->netdev, ring->queue_index);
+}
+
+static struct nbl_res_tx_ring *
+nbl_alloc_tx_ring(struct nbl_resource_mgt *res_mgt, struct net_device *netdev, u16 ring_index,
+ u16 desc_num)
+{
+ struct nbl_hw_ops *hw_ops = NBL_RES_MGT_TO_HW_OPS(res_mgt);
+ struct nbl_common_info *common = NBL_RES_MGT_TO_COMMON(res_mgt);
+ struct nbl_txrx_mgt *txrx_mgt = res_mgt->txrx_mgt;
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ struct nbl_res_tx_ring *ring;
+
+ ring = devm_kzalloc(dev, sizeof(struct nbl_res_tx_ring), GFP_KERNEL);
+ if (!ring)
+ return NULL;
+
+ ring->vsi_info = txrx_mgt->vsi_info;
+ ring->dma_dev = common->dma_dev;
+ ring->product_type = common->product_type;
+ ring->eth_id = common->eth_id;
+ ring->queue_index = ring_index;
+ ring->notify_addr = hw_ops->get_tail_ptr(NBL_RES_MGT_TO_HW_PRIV(res_mgt));
+ ring->notify_qid = NBL_RES_NOFITY_QID(res_mgt, ring_index * 2 + 1);
+ ring->netdev = netdev;
+ ring->desc_num = desc_num;
+ ring->used_wrap_counter = 1;
+ ring->avail_used_flags |= BIT(NBL_PACKED_DESC_F_AVAIL);
+
+ return ring;
+}
+
+static int nbl_alloc_tx_rings(struct nbl_resource_mgt *res_mgt, struct net_device *netdev,
+ u16 tx_num, u16 desc_num)
+{
+ struct nbl_txrx_mgt *txrx_mgt = res_mgt->txrx_mgt;
+ struct nbl_common_info *common = NBL_RES_MGT_TO_COMMON(res_mgt);
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ struct nbl_res_tx_ring *ring;
+ u32 ring_index;
+
+ if (txrx_mgt->tx_rings) {
+ nbl_err(common, NBL_DEBUG_RESOURCE,
+ "Try to allocate tx_rings which already exists\n");
+ return -EINVAL;
+ }
+
+ txrx_mgt->tx_ring_num = tx_num;
+
+ txrx_mgt->tx_rings = devm_kcalloc(dev, tx_num,
+ sizeof(struct nbl_res_tx_ring *), GFP_KERNEL);
+ if (!txrx_mgt->tx_rings)
+ return -ENOMEM;
+
+ for (ring_index = 0; ring_index < tx_num; ring_index++) {
+ ring = txrx_mgt->tx_rings[ring_index];
+ WARN_ON(ring);
+ ring = nbl_alloc_tx_ring(res_mgt, netdev, ring_index, desc_num);
+ if (!ring)
+ goto alloc_tx_ring_failed;
+
+ WRITE_ONCE(txrx_mgt->tx_rings[ring_index], ring);
+ }
+
+ return 0;
+
+alloc_tx_ring_failed:
+ while (ring_index--)
+ devm_kfree(dev, txrx_mgt->tx_rings[ring_index]);
+ devm_kfree(dev, txrx_mgt->tx_rings);
+ txrx_mgt->tx_rings = NULL;
+ return -ENOMEM;
+}
+
+static void nbl_free_tx_rings(struct nbl_resource_mgt *res_mgt)
+{
+ struct nbl_txrx_mgt *txrx_mgt = res_mgt->txrx_mgt;
+ struct nbl_res_tx_ring *ring;
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ u16 ring_count;
+ u16 ring_index;
+
+ ring_count = txrx_mgt->tx_ring_num;
+ for (ring_index = 0; ring_index < ring_count; ring_index++) {
+ ring = txrx_mgt->tx_rings[ring_index];
+ devm_kfree(dev, ring);
+ }
+ devm_kfree(dev, txrx_mgt->tx_rings);
+ txrx_mgt->tx_rings = NULL;
+}
+
+static int nbl_alloc_rx_rings(struct nbl_resource_mgt *res_mgt, struct net_device *netdev,
+ u16 rx_num, u16 desc_num)
+{
+ struct nbl_common_info *common = NBL_RES_MGT_TO_COMMON(res_mgt);
+ struct nbl_txrx_mgt *txrx_mgt = res_mgt->txrx_mgt;
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ struct nbl_res_rx_ring *ring;
+ u32 ring_index;
+
+ if (txrx_mgt->rx_rings) {
+ nbl_err(common, NBL_DEBUG_RESOURCE,
+ "Try to allocate rx_rings which already exists\n");
+ return -EINVAL;
+ }
+
+ txrx_mgt->rx_ring_num = rx_num;
+
+ txrx_mgt->rx_rings = devm_kcalloc(dev, rx_num,
+ sizeof(struct nbl_res_rx_ring *), GFP_KERNEL);
+ if (!txrx_mgt->rx_rings)
+ return -ENOMEM;
+
+ for (ring_index = 0; ring_index < rx_num; ring_index++) {
+ ring = txrx_mgt->rx_rings[ring_index];
+ WARN_ON(ring);
+ ring = devm_kzalloc(dev, sizeof(struct nbl_res_rx_ring), GFP_KERNEL);
+ if (!ring)
+ goto alloc_rx_ring_failed;
+
+ ring->common = common;
+ ring->txrx_mgt = txrx_mgt;
+ ring->dma_dev = common->dma_dev;
+ ring->queue_index = ring_index;
+ ring->notify_qid = NBL_RES_NOFITY_QID(res_mgt, ring_index * 2);
+ ring->netdev = netdev;
+ ring->desc_num = desc_num;
+ /* RX buffer length is determined by mtu,
+ * when netdev up we will set buf_len according to its mtu
+ */
+ ring->buf_len = PAGE_SIZE / 2 - NBL_RX_PAD;
+
+ ring->used_wrap_counter = 1;
+ ring->avail_used_flags |= BIT(NBL_PACKED_DESC_F_AVAIL);
+ WRITE_ONCE(txrx_mgt->rx_rings[ring_index], ring);
+ }
+
+ return 0;
+
+alloc_rx_ring_failed:
+ while (ring_index--)
+ devm_kfree(dev, txrx_mgt->rx_rings[ring_index]);
+ devm_kfree(dev, txrx_mgt->rx_rings);
+ txrx_mgt->rx_rings = NULL;
+ return -ENOMEM;
+}
+
+static void nbl_free_rx_rings(struct nbl_resource_mgt *res_mgt)
+{
+ struct nbl_txrx_mgt *txrx_mgt = res_mgt->txrx_mgt;
+ struct nbl_res_rx_ring *ring;
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ u16 ring_count;
+ u16 ring_index;
+
+ ring_count = txrx_mgt->rx_ring_num;
+ for (ring_index = 0; ring_index < ring_count; ring_index++) {
+ ring = txrx_mgt->rx_rings[ring_index];
+ devm_kfree(dev, ring);
+ }
+ devm_kfree(dev, txrx_mgt->rx_rings);
+ txrx_mgt->rx_rings = NULL;
+}
+
+static int nbl_alloc_vectors(struct nbl_resource_mgt *res_mgt, u16 num)
+{
+ struct nbl_common_info *common = NBL_RES_MGT_TO_COMMON(res_mgt);
+ struct nbl_txrx_mgt *txrx_mgt = res_mgt->txrx_mgt;
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ struct nbl_res_vector *vector;
+ u32 index;
+
+ if (txrx_mgt->vectors) {
+ nbl_err(common, NBL_DEBUG_RESOURCE,
+ "Try to allocate vectors which already exists\n");
+ return -EINVAL;
+ }
+
+ txrx_mgt->vectors = devm_kcalloc(dev, num, sizeof(struct nbl_res_vector *),
+ GFP_KERNEL);
+ if (!txrx_mgt->vectors)
+ return -ENOMEM;
+
+ for (index = 0; index < num; index++) {
+ vector = txrx_mgt->vectors[index];
+ WARN_ON(vector);
+ vector = devm_kzalloc(dev, sizeof(struct nbl_res_vector), GFP_KERNEL);
+ if (!vector)
+ goto alloc_vector_failed;
+
+ vector->rx_ring = txrx_mgt->rx_rings[index];
+ vector->tx_ring = txrx_mgt->tx_rings[index];
+ WRITE_ONCE(txrx_mgt->vectors[index], vector);
+ }
+ return 0;
+
+alloc_vector_failed:
+ while (index--)
+ devm_kfree(dev, txrx_mgt->vectors[index]);
+ devm_kfree(dev, txrx_mgt->vectors);
+ txrx_mgt->vectors = NULL;
+ return -ENOMEM;
+}
+
+static void nbl_free_vectors(struct nbl_resource_mgt *res_mgt)
+{
+ struct nbl_txrx_mgt *txrx_mgt = res_mgt->txrx_mgt;
+ struct nbl_res_vector *vector;
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ u16 count, index;
+
+ count = txrx_mgt->rx_ring_num;
+ for (index = 0; index < count; index++) {
+ vector = txrx_mgt->vectors[index];
+ devm_kfree(dev, vector);
+ }
+ devm_kfree(dev, txrx_mgt->vectors);
+ txrx_mgt->vectors = NULL;
+}
+
+static int nbl_res_txrx_alloc_rings(void *priv, struct net_device *netdev,
+ struct nbl_ring_param *param)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ int err = 0;
+
+ err = nbl_alloc_tx_rings(res_mgt, netdev, param->tx_ring_num, param->queue_size);
+ if (err)
+ return err;
+
+ err = nbl_alloc_rx_rings(res_mgt, netdev, param->rx_ring_num, param->queue_size);
+ if (err)
+ goto alloc_rx_rings_err;
+
+ err = nbl_alloc_vectors(res_mgt, param->rx_ring_num);
+ if (err)
+ goto alloc_vectors_err;
+
+ nbl_info(res_mgt->common, NBL_DEBUG_RESOURCE,
+ "Alloc rings for %d tx, %d rx, %d desc\n",
+ param->tx_ring_num, param->rx_ring_num, param->queue_size);
+ return 0;
+
+alloc_vectors_err:
+ nbl_free_rx_rings(res_mgt);
+alloc_rx_rings_err:
+ nbl_free_tx_rings(res_mgt);
+ return err;
+}
+
+static void nbl_res_txrx_remove_rings(void *priv)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+
+ nbl_free_vectors(res_mgt);
+ nbl_free_tx_rings(res_mgt);
+ nbl_free_rx_rings(res_mgt);
+ nbl_info(res_mgt->common, NBL_DEBUG_RESOURCE, "Remove rings");
+}
+
+static dma_addr_t nbl_res_txrx_start_tx_ring(void *priv, u8 ring_index)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ struct device *dma_dev = NBL_RES_MGT_TO_DMA_DEV(res_mgt);
+ struct nbl_res_tx_ring *tx_ring = NBL_RES_MGT_TO_TX_RING(res_mgt, ring_index);
+
+ if (tx_ring->tx_bufs) {
+ nbl_err(res_mgt->common, NBL_DEBUG_RESOURCE,
+ "Try to setup a TX ring with buffer management array already allocated\n");
+ return (dma_addr_t)NULL;
+ }
+
+ tx_ring->tx_bufs = devm_kcalloc(dev, tx_ring->desc_num, sizeof(*tx_ring->tx_bufs),
+ GFP_KERNEL);
+ if (!tx_ring->tx_bufs)
+ return (dma_addr_t)NULL;
+
+ /* Alloc twice memory, and second half is used to back up the desc for desc checking */
+ tx_ring->size = ALIGN(tx_ring->desc_num * sizeof(struct nbl_ring_desc), PAGE_SIZE);
+ tx_ring->desc = dmam_alloc_coherent(dma_dev, tx_ring->size, &tx_ring->dma,
+ GFP_KERNEL | __GFP_ZERO);
+ if (!tx_ring->desc)
+ goto alloc_dma_err;
+
+ tx_ring->next_to_use = 0;
+ tx_ring->next_to_clean = 0;
+ tx_ring->tail_ptr = 0;
+
+ tx_ring->valid = true;
+ nbl_debug(res_mgt->common, NBL_DEBUG_RESOURCE, "Start tx ring %d", ring_index);
+ return tx_ring->dma;
+
+alloc_dma_err:
+ devm_kfree(dev, tx_ring->tx_bufs);
+ tx_ring->tx_bufs = NULL;
+ tx_ring->size = 0;
+ return (dma_addr_t)NULL;
+}
+
+static inline bool nbl_rx_cache_get(struct nbl_res_rx_ring *rx_ring, struct nbl_dma_info *dma_info)
+{
+ struct nbl_page_cache *cache = &rx_ring->page_cache;
+ struct nbl_rx_queue_stats *stats = &rx_ring->rx_stats;
+
+ if (unlikely(cache->head == cache->tail)) {
+ stats->rx_cache_empty++;
+ return false;
+ }
+
+ if (page_ref_count(cache->page_cache[cache->head].page) != 1) {
+ stats->rx_cache_busy++;
+ return false;
+ }
+
+ *dma_info = cache->page_cache[cache->head];
+ cache->head = (cache->head + 1) & (NBL_MAX_CACHE_SIZE - 1);
+ stats->rx_cache_reuse++;
+
+ dma_sync_single_for_device(rx_ring->dma_dev, dma_info->addr,
+ dma_info->size, DMA_FROM_DEVICE);
+ return true;
+}
+
+static inline int nbl_page_alloc_pool(struct nbl_res_rx_ring *rx_ring,
+ struct nbl_dma_info *dma_info)
+{
+ if (nbl_rx_cache_get(rx_ring, dma_info))
+ return 0;
+
+ dma_info->page = page_pool_dev_alloc_pages(rx_ring->page_pool);
+ if (unlikely(!dma_info->page))
+ return -ENOMEM;
+
+ dma_info->addr = dma_map_page_attrs(rx_ring->dma_dev, dma_info->page, 0, dma_info->size,
+ DMA_FROM_DEVICE, NBL_RX_DMA_ATTR);
+
+ if (unlikely(dma_mapping_error(rx_ring->dma_dev, dma_info->addr))) {
+ page_pool_recycle_direct(rx_ring->page_pool, dma_info->page);
+ dma_info->page = NULL;
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static inline int nbl_get_rx_frag(struct nbl_res_rx_ring *rx_ring, struct nbl_rx_buffer *buffer)
+{
+ int err = 0;
+
+ /* first buffer alloc page */
+ if (buffer->first_in_page)
+ err = nbl_page_alloc_pool(rx_ring, buffer->di);
+
+ return err;
+}
+
+static inline bool nbl_alloc_rx_bufs(struct nbl_res_rx_ring *rx_ring, u16 count)
+{
+ u32 buf_len;
+ u16 next_to_use, head;
+ __le16 head_flags = 0;
+ struct nbl_ring_desc *rx_desc, *head_desc;
+ struct nbl_rx_buffer *rx_buf;
+ int i;
+
+ if (unlikely(!rx_ring || !count)) {
+ nbl_warn(NBL_RING_TO_COMMON(rx_ring), NBL_DEBUG_RESOURCE,
+ "invalid input parameters, rx_ring is %p, count is %d.\n", rx_ring, count);
+ return -EINVAL;
+ }
+
+ buf_len = rx_ring->buf_len;
+ next_to_use = rx_ring->next_to_use;
+
+ head = next_to_use;
+ head_desc = NBL_RX_DESC(rx_ring, next_to_use);
+ rx_desc = NBL_RX_DESC(rx_ring, next_to_use);
+ rx_buf = NBL_RX_BUF(rx_ring, next_to_use);
+
+ if (unlikely(!rx_desc || !rx_buf)) {
+ nbl_warn(NBL_RING_TO_COMMON(rx_ring), NBL_DEBUG_RESOURCE,
+ "invalid input parameters, next_to_use:%d, rx_desc is %p, rx_buf is %p.\n",
+ next_to_use, rx_desc, rx_buf);
+ return -EINVAL;
+ }
+
+ do {
+ if (nbl_get_rx_frag(rx_ring, rx_buf))
+ break;
+
+ for (i = 0; i < rx_ring->frags_num_per_page; i++, rx_desc++, rx_buf++) {
+ rx_desc->addr = cpu_to_le64(rx_buf->di->addr + rx_buf->offset);
+ rx_desc->len = cpu_to_le32(buf_len);
+ rx_desc->id = cpu_to_le16(next_to_use);
+
+ if (likely(head != next_to_use || i))
+ rx_desc->flags = cpu_to_le16(rx_ring->avail_used_flags |
+ NBL_PACKED_DESC_F_WRITE);
+ else
+ head_flags = cpu_to_le16(rx_ring->avail_used_flags |
+ NBL_PACKED_DESC_F_WRITE);
+ }
+
+ next_to_use += rx_ring->frags_num_per_page;
+ rx_ring->tail_ptr += rx_ring->frags_num_per_page;
+ count -= rx_ring->frags_num_per_page;
+ if (next_to_use == rx_ring->desc_num) {
+ next_to_use = 0;
+ rx_desc = NBL_RX_DESC(rx_ring, next_to_use);
+ rx_buf = NBL_RX_BUF(rx_ring, next_to_use);
+ rx_ring->avail_used_flags ^=
+ BIT(NBL_PACKED_DESC_F_AVAIL) |
+ BIT(NBL_PACKED_DESC_F_USED);
+ }
+ } while (count);
+
+ if (next_to_use != head) {
+ /* wmb */
+ wmb();
+ head_desc->flags = head_flags;
+ rx_ring->next_to_use = next_to_use;
+ }
+
+ return !!count;
+}
+
+static void nbl_unmap_and_free_tx_resource(struct nbl_res_tx_ring *ring,
+ struct nbl_tx_buffer *tx_buffer,
+ bool free, bool in_napi)
+{
+ struct device *dma_dev = NBL_RING_TO_DMA_DEV(ring);
+
+ if (tx_buffer->skb) {
+ if (likely(free)) {
+ if (in_napi)
+ napi_consume_skb(tx_buffer->skb, NBL_TX_POLL_WEIGHT);
+ else
+ dev_kfree_skb_any(tx_buffer->skb);
+ }
+
+ if (dma_unmap_len(tx_buffer, len))
+ dma_unmap_single(dma_dev, dma_unmap_addr(tx_buffer, dma),
+ dma_unmap_len(tx_buffer, len),
+ DMA_TO_DEVICE);
+ } else if (tx_buffer->page && dma_unmap_len(tx_buffer, len)) {
+ dma_unmap_page(dma_dev, dma_unmap_addr(tx_buffer, dma),
+ dma_unmap_len(tx_buffer, len),
+ DMA_TO_DEVICE);
+ } else if (dma_unmap_len(tx_buffer, len)) {
+ dma_unmap_single(dma_dev, dma_unmap_addr(tx_buffer, dma),
+ dma_unmap_len(tx_buffer, len),
+ DMA_TO_DEVICE);
+ }
+
+ tx_buffer->next_to_watch = NULL;
+ tx_buffer->skb = NULL;
+ tx_buffer->page = 0;
+ tx_buffer->bytecount = 0;
+ tx_buffer->gso_segs = 0;
+ dma_unmap_len_set(tx_buffer, len, 0);
+}
+
+static void nbl_free_tx_ring_bufs(struct nbl_res_tx_ring *tx_ring)
+{
+ struct nbl_tx_buffer *tx_buffer;
+ u16 i;
+
+ i = tx_ring->next_to_clean;
+ tx_buffer = NBL_TX_BUF(tx_ring, i);
+ while (i != tx_ring->next_to_use) {
+ nbl_unmap_and_free_tx_resource(tx_ring, tx_buffer, true, false);
+ i++;
+ tx_buffer++;
+ if (i == tx_ring->desc_num) {
+ i = 0;
+ tx_buffer = NBL_TX_BUF(tx_ring, i);
+ }
+ }
+
+ tx_ring->next_to_clean = 0;
+ tx_ring->next_to_use = 0;
+ tx_ring->tail_ptr = 0;
+
+ tx_ring->used_wrap_counter = 1;
+ tx_ring->avail_used_flags = BIT(NBL_PACKED_DESC_F_AVAIL);
+ memset(tx_ring->desc, 0, tx_ring->size);
+}
+
+static void nbl_res_txrx_stop_tx_ring(void *priv, u8 ring_index)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ struct device *dma_dev = NBL_RES_MGT_TO_DMA_DEV(res_mgt);
+ struct nbl_res_tx_ring *tx_ring = NBL_RES_MGT_TO_TX_RING(res_mgt, ring_index);
+ struct nbl_res_vector *vector = NBL_RES_MGT_TO_VECTOR(res_mgt, ring_index);
+
+ vector->started = false;
+ /* Flush napi task, to ensue the sched napi finish. So napi will no to access the
+ * ring memory(wild point), bacause the vector->started has set false.
+ */
+ napi_synchronize(&vector->nbl_napi.napi);
+ tx_ring->valid = false;
+
+ nbl_free_tx_ring_bufs(tx_ring);
+ WRITE_ONCE(NBL_RES_MGT_TO_TX_RING(res_mgt, ring_index), tx_ring);
+
+ devm_kfree(dev, tx_ring->tx_bufs);
+ tx_ring->tx_bufs = NULL;
+
+ dmam_free_coherent(dma_dev, tx_ring->size, tx_ring->desc, tx_ring->dma);
+ tx_ring->desc = NULL;
+ tx_ring->dma = (dma_addr_t)NULL;
+ tx_ring->size = 0;
+
+ if (nbl_txrx_within_vsi(&tx_ring->vsi_info[NBL_VSI_DATA], tx_ring->queue_index))
+ netdev_tx_reset_queue(txring_txq(tx_ring));
+
+ nbl_debug(res_mgt->common, NBL_DEBUG_RESOURCE, "Stop tx ring %d", ring_index);
+}
+
+static inline bool nbl_dev_page_is_reusable(struct page *page, u8 nid)
+{
+ return likely(page_to_nid(page) == nid && !page_is_pfmemalloc(page));
+}
+
+static inline int nbl_rx_cache_put(struct nbl_res_rx_ring *rx_ring, struct nbl_dma_info *dma_info)
+{
+ struct nbl_page_cache *cache = &rx_ring->page_cache;
+ u32 tail_next = (cache->tail + 1) & (NBL_MAX_CACHE_SIZE - 1);
+ struct nbl_rx_queue_stats *stats = &rx_ring->rx_stats;
+
+ if (tail_next == cache->head) {
+ stats->rx_cache_full++;
+ return 0;
+ }
+
+ if (!nbl_dev_page_is_reusable(dma_info->page, rx_ring->nid)) {
+ stats->rx_cache_waive++;
+ return 1;
+ }
+
+ cache->page_cache[cache->tail] = *dma_info;
+ cache->tail = tail_next;
+
+ return 2;
+}
+
+static inline void nbl_page_release_dynamic(struct nbl_res_rx_ring *rx_ring,
+ struct nbl_dma_info *dma_info, bool recycle)
+{
+ u32 ret;
+
+ if (likely(recycle)) {
+ ret = nbl_rx_cache_put(rx_ring, dma_info);
+ if (ret == 2)
+ return;
+ if (ret == 1)
+ goto free_page;
+ dma_unmap_page_attrs(rx_ring->dma_dev, dma_info->addr, dma_info->size,
+ DMA_FROM_DEVICE, NBL_RX_DMA_ATTR);
+ page_pool_recycle_direct(rx_ring->page_pool, dma_info->page);
+
+ return;
+ }
+free_page:
+ dma_unmap_page_attrs(rx_ring->dma_dev, dma_info->addr, dma_info->size,
+ DMA_FROM_DEVICE, NBL_RX_DMA_ATTR);
+ page_pool_put_page(rx_ring->page_pool, dma_info->page, dma_info->size, true);
+}
+
+static inline void nbl_put_rx_frag(struct nbl_res_rx_ring *rx_ring,
+ struct nbl_rx_buffer *buffer, bool recycle)
+{
+ if (buffer->last_in_page)
+ nbl_page_release_dynamic(rx_ring, buffer->di, recycle);
+}
+
+static void nbl_free_rx_ring_bufs(struct nbl_res_rx_ring *rx_ring)
+{
+ struct nbl_rx_buffer *rx_buf;
+ u16 i;
+
+ i = rx_ring->next_to_clean;
+ rx_buf = NBL_RX_BUF(rx_ring, i);
+ while (i != rx_ring->next_to_use) {
+ nbl_put_rx_frag(rx_ring, rx_buf, false);
+ i++;
+ rx_buf++;
+ if (i == rx_ring->desc_num) {
+ i = 0;
+ rx_buf = NBL_RX_BUF(rx_ring, i);
+ }
+ }
+
+ for (i = rx_ring->page_cache.head; i != rx_ring->page_cache.tail;
+ i = (i + 1) & (NBL_MAX_CACHE_SIZE - 1)) {
+ struct nbl_dma_info *dma_info = &rx_ring->page_cache.page_cache[i];
+
+ nbl_page_release_dynamic(rx_ring, dma_info, false);
+ }
+
+ rx_ring->next_to_clean = 0;
+ rx_ring->next_to_use = 0;
+ rx_ring->tail_ptr = 0;
+ rx_ring->page_cache.head = 0;
+ rx_ring->page_cache.tail = 0;
+
+ rx_ring->used_wrap_counter = 1;
+ rx_ring->avail_used_flags = BIT(NBL_PACKED_DESC_F_AVAIL);
+ memset(rx_ring->desc, 0, rx_ring->size);
+}
+
+static dma_addr_t nbl_res_txrx_start_rx_ring(void *priv, u8 ring_index, bool use_napi)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct nbl_common_info *common = NBL_RES_MGT_TO_COMMON(res_mgt);
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ struct device *dma_dev = NBL_RES_MGT_TO_DMA_DEV(res_mgt);
+ struct nbl_res_rx_ring *rx_ring = NBL_RES_MGT_TO_RX_RING(res_mgt, ring_index);
+ struct nbl_res_vector *vector = NBL_RES_MGT_TO_VECTOR(res_mgt, ring_index);
+ struct page_pool_params pp_params = {0};
+ int pkt_len, hw_mtu, max_linear_len;
+ int buf_size;
+ int order = 0;
+ int i, j;
+ u16 rx_pad, tailroom;
+
+ if (rx_ring->rx_bufs) {
+ nbl_err(common, NBL_DEBUG_RESOURCE,
+ "Try to setup a RX ring with buffer management array already allocated\n");
+ return (dma_addr_t)NULL;
+ }
+ hw_mtu = rx_ring->netdev->mtu + NBL_PKT_HDR_PAD + NBL_BUFFER_HDR_LEN;
+ tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+ rx_pad = NBL_RX_PAD;
+ max_linear_len = NBL_RX_BUFSZ;
+ pkt_len = SKB_DATA_ALIGN(hw_mtu + rx_pad) + tailroom;
+ rx_ring->linear_skb = true;
+ if (pkt_len > max_linear_len) {
+ rx_ring->linear_skb = false;
+ rx_pad = 0;
+ tailroom = 0;
+ pkt_len = SKB_DATA_ALIGN(hw_mtu);
+ }
+ buf_size = NBL_RX_BUFSZ;
+ WARN_ON(buf_size > PAGE_SIZE);
+ rx_ring->frags_num_per_page = (PAGE_SIZE * (1 << order)) / buf_size;
+ WARN_ON(rx_ring->frags_num_per_page > NBL_MAX_BATCH_DESC);
+ rx_ring->buf_len = buf_size - rx_pad - tailroom;
+
+ pp_params.order = order;
+ pp_params.flags = 0;
+ pp_params.pool_size = rx_ring->desc_num;
+ pp_params.nid = dev_to_node(dev);
+ pp_params.dev = dev;
+ pp_params.dma_dir = DMA_FROM_DEVICE;
+
+ if (dev_to_node(dev) == NUMA_NO_NODE)
+ rx_ring->nid = 0;
+ else
+ rx_ring->nid = dev_to_node(dev);
+
+ rx_ring->page_pool = page_pool_create(&pp_params);
+ if (IS_ERR(rx_ring->page_pool)) {
+ nbl_err(common, NBL_DEBUG_RESOURCE, "Page_pool Allocate %u Failed failed\n",
+ rx_ring->queue_index);
+ return (dma_addr_t)NULL;
+ }
+
+ rx_ring->di = kvzalloc_node(array_size(rx_ring->desc_num / rx_ring->frags_num_per_page,
+ sizeof(struct nbl_dma_info)),
+ GFP_KERNEL, dev_to_node(dev));
+ if (!rx_ring->di) {
+ nbl_err(common, NBL_DEBUG_RESOURCE, "Dma info Allocate %u Failed failed\n",
+ rx_ring->queue_index);
+ goto alloc_di_err;
+ }
+
+ rx_ring->rx_bufs = devm_kcalloc(dev, rx_ring->desc_num, sizeof(*rx_ring->rx_bufs),
+ GFP_KERNEL);
+ if (!rx_ring->rx_bufs)
+ goto alloc_buffers_err;
+
+ /* Alloc twice memory, and second half is used to back up the desc for desc checking */
+ rx_ring->size = ALIGN(rx_ring->desc_num * sizeof(struct nbl_ring_desc), PAGE_SIZE);
+ rx_ring->desc = dmam_alloc_coherent(dma_dev, rx_ring->size, &rx_ring->dma,
+ GFP_KERNEL | __GFP_ZERO);
+ if (!rx_ring->desc) {
+ nbl_err(common, NBL_DEBUG_RESOURCE,
+ "Allocate %u bytes descriptor DMA memory for RX queue %u failed\n",
+ rx_ring->size, rx_ring->queue_index);
+ goto alloc_dma_err;
+ }
+
+ rx_ring->next_to_use = 0;
+ rx_ring->next_to_clean = 0;
+ rx_ring->tail_ptr = 0;
+
+ j = 0;
+ for (i = 0; i < rx_ring->desc_num / rx_ring->frags_num_per_page; i++) {
+ struct nbl_dma_info *di = &rx_ring->di[i];
+ struct nbl_rx_buffer *buffer = &rx_ring->rx_bufs[j];
+ int f;
+
+ di->size = (PAGE_SIZE * (1 << order));
+ for (f = 0; f < rx_ring->frags_num_per_page; f++, j++) {
+ buffer = &rx_ring->rx_bufs[j];
+ buffer->di = di;
+ buffer->size = buf_size;
+ buffer->offset = rx_pad + f * buf_size;
+ buffer->rx_pad = rx_pad;
+ buffer->first_in_page = (f == 0);
+ buffer->last_in_page = (f == rx_ring->frags_num_per_page - 1);
+ }
+ }
+
+ if (nbl_alloc_rx_bufs(rx_ring, rx_ring->desc_num - NBL_MAX_BATCH_DESC))
+ goto alloc_rx_bufs_err;
+
+ rx_ring->valid = true;
+ if (use_napi && vector)
+ vector->started = true;
+
+ nbl_debug(common, NBL_DEBUG_RESOURCE, "Start rx ring %d", ring_index);
+ return rx_ring->dma;
+
+alloc_rx_bufs_err:
+ nbl_free_rx_ring_bufs(rx_ring);
+ dmam_free_coherent(dma_dev, rx_ring->size, rx_ring->desc, rx_ring->dma);
+ rx_ring->desc = NULL;
+ rx_ring->dma = (dma_addr_t)NULL;
+alloc_dma_err:
+ devm_kfree(dev, rx_ring->rx_bufs);
+ rx_ring->rx_bufs = NULL;
+alloc_buffers_err:
+ kvfree(rx_ring->di);
+alloc_di_err:
+ page_pool_destroy(rx_ring->page_pool);
+ rx_ring->size = 0;
+ return (dma_addr_t)NULL;
+}
+
+static void nbl_res_txrx_stop_rx_ring(void *priv, u8 ring_index)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct device *dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ struct device *dma_dev = NBL_RES_MGT_TO_DMA_DEV(res_mgt);
+ struct nbl_res_rx_ring *rx_ring = NBL_RES_MGT_TO_RX_RING(res_mgt, ring_index);
+
+ rx_ring->valid = false;
+
+ nbl_free_rx_ring_bufs(rx_ring);
+ WRITE_ONCE(NBL_RES_MGT_TO_RX_RING(res_mgt, ring_index), rx_ring);
+
+ devm_kfree(dev, rx_ring->rx_bufs);
+ kvfree(rx_ring->di);
+ rx_ring->rx_bufs = NULL;
+
+ dmam_free_coherent(dma_dev, rx_ring->size, rx_ring->desc, rx_ring->dma);
+ rx_ring->desc = NULL;
+ rx_ring->dma = (dma_addr_t)NULL;
+ rx_ring->size = 0;
+
+ page_pool_destroy(rx_ring->page_pool);
+
+ nbl_debug(res_mgt->common, NBL_DEBUG_RESOURCE, "Stop rx ring %d", ring_index);
+}
+
+static inline bool nbl_ring_desc_used(struct nbl_ring_desc *ring_desc, bool used_wrap_counter)
+{
+ bool avail;
+ bool used;
+ u16 flags;
+
+ flags = le16_to_cpu(ring_desc->flags);
+ avail = !!(flags & BIT(NBL_PACKED_DESC_F_AVAIL));
+ used = !!(flags & BIT(NBL_PACKED_DESC_F_USED));
+
+ return avail == used && used == used_wrap_counter;
+}
+
+static int nbl_res_txrx_clean_tx_irq(struct nbl_res_tx_ring *tx_ring)
+{
+ struct nbl_tx_buffer *tx_buffer;
+ struct nbl_ring_desc *tx_desc;
+ unsigned int i = tx_ring->next_to_clean;
+ unsigned int total_tx_pkts = 0;
+ unsigned int total_tx_bytes = 0;
+ unsigned int total_tx_descs = 0;
+ int count = 64;
+
+ tx_buffer = NBL_TX_BUF(tx_ring, i);
+ tx_desc = NBL_TX_DESC(tx_ring, i);
+ i -= tx_ring->desc_num;
+
+ do {
+ struct nbl_ring_desc *end_desc = tx_buffer->next_to_watch;
+
+ if (!end_desc)
+ break;
+
+ /* smp_rmb */
+ smp_rmb();
+
+ if (!nbl_ring_desc_used(tx_desc, tx_ring->used_wrap_counter))
+ break;
+
+ total_tx_pkts += tx_buffer->gso_segs;
+ total_tx_bytes += tx_buffer->bytecount;
+
+ while (true) {
+ total_tx_descs++;
+ nbl_unmap_and_free_tx_resource(tx_ring, tx_buffer, true, true);
+ if (tx_desc == end_desc)
+ break;
+ i++;
+ tx_buffer++;
+ tx_desc++;
+ if (unlikely(!i)) {
+ i -= tx_ring->desc_num;
+ tx_buffer = NBL_TX_BUF(tx_ring, 0);
+ tx_desc = NBL_TX_DESC(tx_ring, 0);
+ tx_ring->used_wrap_counter ^= 1;
+ }
+ }
+
+ tx_buffer++;
+ tx_desc++;
+ i++;
+ if (unlikely(!i)) {
+ i -= tx_ring->desc_num;
+ tx_buffer = NBL_TX_BUF(tx_ring, 0);
+ tx_desc = NBL_TX_DESC(tx_ring, 0);
+ tx_ring->used_wrap_counter ^= 1;
+ }
+
+ prefetch(tx_desc);
+
+ } while (--count);
+
+ i += tx_ring->desc_num;
+
+ tx_ring->next_to_clean = i;
+
+ u64_stats_update_begin(&tx_ring->syncp);
+ tx_ring->stats.bytes += total_tx_bytes;
+ tx_ring->stats.packets += total_tx_pkts;
+ tx_ring->stats.descs += total_tx_descs;
+ u64_stats_update_end(&tx_ring->syncp);
+ if (nbl_txrx_within_vsi(&tx_ring->vsi_info[NBL_VSI_DATA], tx_ring->queue_index))
+ netdev_tx_completed_queue(txring_txq(tx_ring), total_tx_pkts, total_tx_bytes);
+
+#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
+ if (unlikely(total_tx_pkts && netif_carrier_ok(tx_ring->netdev) &&
+ nbl_txrx_within_vsi(&tx_ring->vsi_info[NBL_VSI_DATA], tx_ring->queue_index) &&
+ (nbl_unused_tx_desc_count(tx_ring) >= TX_WAKE_THRESHOLD))) {
+ /* Make sure that anybody stopping the queue after this
+ * sees the new next_to_clean.
+ */
+ smp_mb();
+
+ if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index)) {
+ netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index);
+ dev_dbg(NBL_RING_TO_DEV(tx_ring), "wake queue %u\n", tx_ring->queue_index);
+ }
+ }
+
+ return count;
+}
+
+static void nbl_rx_csum(struct nbl_res_rx_ring *rx_ring, struct sk_buff *skb,
+ struct nbl_rx_extend_head *hdr)
+{
+ skb->ip_summed = CHECKSUM_NONE;
+ skb_checksum_none_assert(skb);
+
+ /* if user disable RX Checksum Offload, then stack verify the rx checksum */
+ if (!(rx_ring->netdev->features & NETIF_F_RXCSUM))
+ return;
+
+ if (!hdr->checksum_status)
+ return;
+
+ if (hdr->error_code) {
+ rx_ring->rx_stats.rx_csum_errors++;
+ return;
+ }
+
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ rx_ring->rx_stats.rx_csum_packets++;
+}
+
+static inline void nbl_add_rx_frag(struct nbl_rx_buffer *rx_buffer,
+ struct sk_buff *skb, unsigned int size)
+{
+ page_ref_inc(rx_buffer->di->page);
+ skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->di->page,
+ rx_buffer->offset, size, rx_buffer->size);
+}
+
+static inline int nbl_rx_vlan_pop(struct nbl_res_rx_ring *rx_ring, struct sk_buff *skb)
+{
+ struct vlan_ethhdr *veth = (struct vlan_ethhdr *)skb->data;
+
+ if (!rx_ring->vlan_proto)
+ return 0;
+
+ if (rx_ring->vlan_proto != ntohs(veth->h_vlan_proto) ||
+ (rx_ring->vlan_tci & VLAN_VID_MASK) != (ntohs(veth->h_vlan_TCI) & VLAN_VID_MASK))
+ return 1;
+
+ memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
+ __skb_pull(skb, VLAN_HLEN);
+
+ return 0;
+}
+
+static void nbl_txrx_register_vsi_ring(void *priv, u16 vsi_index, u16 ring_offset, u16 ring_num)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct nbl_txrx_mgt *txrx_mgt = NBL_RES_MGT_TO_TXRX_MGT(res_mgt);
+
+ txrx_mgt->vsi_info[vsi_index].ring_offset = ring_offset;
+ txrx_mgt->vsi_info[vsi_index].ring_num = ring_num;
+}
+
+static void nbl_res_txrx_cfg_txrx_vlan(void *priv, u16 vlan_tci, u16 vlan_proto, u8 vsi_index)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct nbl_txrx_mgt *txrx_mgt = NBL_RES_MGT_TO_TXRX_MGT(res_mgt);
+ struct nbl_txrx_vsi_info *vsi_info = &txrx_mgt->vsi_info[vsi_index];
+ struct nbl_res_tx_ring *tx_ring;
+ struct nbl_res_rx_ring *rx_ring;
+ u16 i;
+
+ if (!txrx_mgt->tx_rings || !txrx_mgt->rx_rings)
+ return;
+
+ for (i = vsi_info->ring_offset; i < vsi_info->ring_offset + vsi_info->ring_num; i++) {
+ tx_ring = txrx_mgt->tx_rings[i];
+ rx_ring = txrx_mgt->rx_rings[i];
+
+ if (tx_ring) {
+ tx_ring->vlan_tci = vlan_tci;
+ tx_ring->vlan_proto = vlan_proto;
+ }
+
+ if (rx_ring) {
+ rx_ring->vlan_tci = vlan_tci;
+ rx_ring->vlan_proto = vlan_proto;
+ }
+ }
+}
+
+/**
+ * Current version support merging multiple descriptor for one packet.
+ */
+static struct sk_buff *nbl_construct_skb(struct nbl_res_rx_ring *rx_ring, struct napi_struct *napi,
+ struct nbl_rx_buffer *rx_buf, unsigned int size)
+{
+ struct sk_buff *skb;
+ char *p, *buf;
+ int tailroom, shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+ unsigned int truesize = rx_buf->size;
+ unsigned int headlen;
+
+ p = page_address(rx_buf->di->page) + rx_buf->offset;
+ buf = p - NBL_RX_PAD;
+ p += NBL_BUFFER_HDR_LEN;
+ tailroom = truesize - size - NBL_RX_PAD;
+ size -= NBL_BUFFER_HDR_LEN;
+
+ if (rx_ring->linear_skb && tailroom >= shinfo_size) {
+ skb = build_skb(buf, truesize);
+ if (unlikely(!skb))
+ return NULL;
+
+ page_ref_inc(rx_buf->di->page);
+ skb_reserve(skb, p - buf);
+ skb_put(skb, size);
+ goto ok;
+ }
+
+ skb = napi_alloc_skb(napi, NBL_RX_HDR_SIZE);
+ if (unlikely(!skb))
+ return NULL;
+
+ headlen = size;
+ if (headlen > NBL_RX_HDR_SIZE)
+ headlen = eth_get_headlen(skb->dev, p, NBL_RX_HDR_SIZE);
+
+ memcpy(__skb_put(skb, headlen), p, ALIGN(headlen, sizeof(long)));
+ size -= headlen;
+ if (size) {
+ page_ref_inc(rx_buf->di->page);
+ skb_add_rx_frag(skb, 0, rx_buf->di->page,
+ rx_buf->offset + NBL_BUFFER_HDR_LEN + headlen,
+ size, truesize);
+ }
+ok:
+ skb_record_rx_queue(skb, rx_ring->queue_index);
+
+ return skb;
+}
+
+static inline struct nbl_rx_buffer *nbl_get_rx_buf(struct nbl_res_rx_ring *rx_ring)
+{
+ struct nbl_rx_buffer *rx_buf;
+
+ rx_buf = NBL_RX_BUF(rx_ring, rx_ring->next_to_clean);
+ prefetchw(rx_buf->di->page);
+
+ dma_sync_single_range_for_cpu(rx_ring->dma_dev, rx_buf->di->addr, rx_buf->offset,
+ rx_ring->buf_len, DMA_FROM_DEVICE);
+
+ return rx_buf;
+}
+
+static inline void nbl_put_rx_buf(struct nbl_res_rx_ring *rx_ring, struct nbl_rx_buffer *rx_buf)
+{
+ u16 ntc = rx_ring->next_to_clean + 1;
+
+ /* if at the end of the ring, reset ntc and flip used wrap bit */
+ if (unlikely(ntc >= rx_ring->desc_num)) {
+ ntc = 0;
+ rx_ring->used_wrap_counter ^= 1;
+ }
+
+ rx_ring->next_to_clean = ntc;
+ prefetch(NBL_RX_DESC(rx_ring, ntc));
+
+ nbl_put_rx_frag(rx_ring, rx_buf, true);
+}
+
+static inline int nbl_maybe_stop_tx(struct nbl_res_tx_ring *tx_ring, unsigned int size)
+{
+ if (likely(nbl_unused_tx_desc_count(tx_ring) >= size))
+ return 0;
+
+ if (!nbl_txrx_within_vsi(&tx_ring->vsi_info[NBL_VSI_DATA], tx_ring->queue_index))
+ return -EBUSY;
+
+ dev_dbg(NBL_RING_TO_DEV(tx_ring), "unused_desc_count:%u, size:%u, stop queue %u\n",
+ nbl_unused_tx_desc_count(tx_ring), size, tx_ring->queue_index);
+ netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
+
+ /* smp_mb */
+ smp_mb();
+
+ if (likely(nbl_unused_tx_desc_count(tx_ring) < size))
+ return -EBUSY;
+
+ dev_dbg(NBL_RING_TO_DEV(tx_ring), "unused_desc_count:%u, size:%u, start queue %u\n",
+ nbl_unused_tx_desc_count(tx_ring), size, tx_ring->queue_index);
+ netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
+
+ return 0;
+}
+
+static int nbl_res_txrx_clean_rx_irq(struct nbl_res_rx_ring *rx_ring,
+ struct napi_struct *napi,
+ int budget)
+{
+ struct nbl_ring_desc *rx_desc;
+ struct nbl_rx_buffer *rx_buf;
+ struct nbl_rx_extend_head *hdr;
+ struct sk_buff *skb = NULL;
+ unsigned int total_rx_pkts = 0;
+ unsigned int total_rx_bytes = 0;
+ unsigned int size;
+ u32 rx_multicast_packets = 0;
+ u32 rx_unicast_packets = 0;
+ u16 desc_count = 0;
+ u16 num_buffers = 0;
+ u16 cleaned_count = nbl_unused_rx_desc_count(rx_ring);
+ bool failure = 0;
+ bool drop = 0;
+
+ while (likely(total_rx_pkts < budget)) {
+ rx_desc = NBL_RX_DESC(rx_ring, rx_ring->next_to_clean);
+ if (!nbl_ring_desc_used(rx_desc, rx_ring->used_wrap_counter))
+ break;
+
+ dma_rmb();
+ size = le32_to_cpu(rx_desc->len);
+ rx_buf = nbl_get_rx_buf(rx_ring);
+
+ desc_count++;
+
+ if (skb) {
+ nbl_add_rx_frag(rx_buf, skb, size);
+ } else {
+ hdr = page_address(rx_buf->di->page) + rx_buf->offset;
+ net_prefetch(hdr);
+ skb = nbl_construct_skb(rx_ring, napi, rx_buf, size);
+ if (unlikely(!skb)) {
+ rx_ring->rx_stats.rx_alloc_buf_err_cnt++;
+ break;
+ }
+
+ num_buffers = le16_to_cpu(hdr->num_buffers);
+ nbl_rx_csum(rx_ring, skb, hdr);
+ drop = nbl_rx_vlan_pop(rx_ring, skb);
+ }
+
+ cleaned_count++;
+ nbl_put_rx_buf(rx_ring, rx_buf);
+ if (desc_count < num_buffers)
+ continue;
+ desc_count = 0;
+
+ if (unlikely(eth_skb_pad(skb))) {
+ skb = NULL;
+ drop = 0;
+ continue;
+ }
+
+ if (unlikely(drop)) {
+ kfree(skb);
+ skb = NULL;
+ drop = 0;
+ continue;
+ }
+
+ total_rx_bytes += skb->len;
+ skb->protocol = eth_type_trans(skb, rx_ring->netdev);
+ if (unlikely(skb->pkt_type == PACKET_BROADCAST ||
+ skb->pkt_type == PACKET_MULTICAST))
+ rx_multicast_packets++;
+ else
+ rx_unicast_packets++;
+
+ napi_gro_receive(napi, skb);
+ skb = NULL;
+ drop = 0;
+ total_rx_pkts++;
+ }
+
+ if (cleaned_count & (~(NBL_MAX_BATCH_DESC - 1)))
+ failure = nbl_alloc_rx_bufs(rx_ring, cleaned_count & (~(NBL_MAX_BATCH_DESC - 1)));
+
+ u64_stats_update_begin(&rx_ring->syncp);
+ rx_ring->stats.packets += total_rx_pkts;
+ rx_ring->stats.bytes += total_rx_bytes;
+ rx_ring->rx_stats.rx_multicast_packets += rx_multicast_packets;
+ rx_ring->rx_stats.rx_unicast_packets += rx_unicast_packets;
+ u64_stats_update_end(&rx_ring->syncp);
+
+ return failure ? budget : total_rx_pkts;
+}
+
+static int nbl_res_napi_poll(struct napi_struct *napi, int budget)
+{
+ struct nbl_napi_struct *nbl_napi = container_of(napi, struct nbl_napi_struct, napi);
+ struct nbl_res_vector *vector = container_of(nbl_napi, struct nbl_res_vector, nbl_napi);
+ struct nbl_res_tx_ring *tx_ring;
+ struct nbl_res_rx_ring *rx_ring;
+ int complete = 1, cleaned = 0, tx_done = 1;
+
+ tx_ring = vector->tx_ring;
+ rx_ring = vector->rx_ring;
+
+ if (vector->started) {
+ tx_done = nbl_res_txrx_clean_tx_irq(tx_ring);
+ cleaned = nbl_res_txrx_clean_rx_irq(rx_ring, napi, budget);
+ }
+ complete = tx_done && (cleaned < budget);
+ if (!complete)
+ return budget;
+
+ if (!napi_complete_done(napi, cleaned))
+ return min_t(int, cleaned, budget - 1);
+
+ /* unmask irq passthrough for performace */
+ if (vector->net_msix_mask_en)
+ writel(vector->irq_data, vector->irq_enable_base);
+
+ return min_t(int, cleaned, budget - 1);
+}
+
+static unsigned int nbl_xmit_desc_count(struct sk_buff *skb)
+{
+ unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
+
+ return nr_frags + 1;
+}
+
+/* set up TSO(TCP Segmentation Offload) */
+static int nbl_tx_tso(struct nbl_tx_buffer *first, struct nbl_tx_hdr_param *hdr_param)
+{
+ struct sk_buff *skb = first->skb;
+ union {
+ struct iphdr *v4;
+ struct ipv6hdr *v6;
+ unsigned char *hdr;
+ } ip;
+ union {
+ struct tcphdr *tcp;
+ struct udphdr *udp;
+ unsigned char *hdr;
+ } l4;
+ u8 l4_start;
+ u32 payload_len;
+ u8 header_len = 0;
+ int err;
+
+ if (skb->ip_summed != CHECKSUM_PARTIAL)
+ return 1;
+
+ if (!skb_is_gso(skb))
+ return 1;
+
+ err = skb_cow_head(skb, 0);
+ if (err < 0)
+ return err;
+
+ ip.hdr = skb_network_header(skb);
+ l4.hdr = skb_transport_header(skb);
+
+ /* initialize IP header fields*/
+ if (ip.v4->version == IP_VERSION_V4) {
+ ip.v4->tot_len = 0;
+ ip.v4->check = 0;
+ } else {
+ ip.v6->payload_len = 0;
+ }
+
+ /* length of (MAC + IP) header */
+ l4_start = (u8)(l4.hdr - skb->data);
+
+ /* l4 packet length */
+ payload_len = skb->len - l4_start;
+
+ /* remove l4 packet length from L4 pseudo-header checksum */
+ if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
+ csum_replace_by_diff(&l4.udp->check, (__force __wsum)htonl(payload_len));
+ /* compute length of UDP segmentation header */
+ header_len = (u8)sizeof(l4.udp) + l4_start;
+ } else {
+ csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(payload_len));
+ /* compute length of TCP segmentation header */
+ header_len = (u8)(l4.tcp->doff * 4 + l4_start);
+ }
+
+ hdr_param->tso = 1;
+ hdr_param->mss = skb_shinfo(skb)->gso_size;
+ hdr_param->total_hlen = header_len;
+
+ first->gso_segs = skb_shinfo(skb)->gso_segs;
+ first->bytecount += (first->gso_segs - 1) * header_len;
+ first->tx_flags = NBL_TX_FLAGS_TSO;
+
+ return first->gso_segs;
+}
+
+/* set up Tx checksum offload */
+static int nbl_tx_csum(struct nbl_tx_buffer *first, struct nbl_tx_hdr_param *hdr_param)
+{
+ struct sk_buff *skb = first->skb;
+ union {
+ struct iphdr *v4;
+ struct ipv6hdr *v6;
+ unsigned char *hdr;
+ } ip;
+ union {
+ struct tcphdr *tcp;
+ struct udphdr *udp;
+ unsigned char *hdr;
+ } l4;
+ __be16 frag_off, protocol;
+ u8 inner_ip_type = 0, l4_type = 0, l4_csum = 0, l4_proto = 0;
+ u32 l2_len = 0, l3_len = 0, l4_len = 0;
+ unsigned char *exthdr;
+ int ret;
+
+ if (skb->ip_summed != CHECKSUM_PARTIAL)
+ return 0;
+
+ ip.hdr = skb_network_header(skb);
+ l4.hdr = skb_transport_header(skb);
+
+ /* compute outer L2 header size */
+ l2_len = ip.hdr - skb->data;
+
+ protocol = vlan_get_protocol(skb);
+
+ if (protocol == htons(ETH_P_IP)) {
+ inner_ip_type = NBL_TX_IIPT_IPV4;
+ l4_proto = ip.v4->protocol;
+ } else if (protocol == htons(ETH_P_IPV6)) {
+ inner_ip_type = NBL_TX_IIPT_IPV6;
+ exthdr = ip.hdr + sizeof(*ip.v6);
+ l4_proto = ip.v6->nexthdr;
+
+ if (l4.hdr != exthdr) {
+ ret = ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_proto, &frag_off);
+ if (ret < 0)
+ return -1;
+ }
+ } else {
+ return -1;
+ }
+
+ l3_len = l4.hdr - ip.hdr;
+
+ switch (l4_proto) {
+ case IPPROTO_TCP:
+ l4_type = NBL_TX_L4T_TCP;
+ l4_len = l4.tcp->doff;
+ l4_csum = 1;
+ break;
+ case IPPROTO_UDP:
+ l4_type = NBL_TX_L4T_UDP;
+ l4_len = (sizeof(struct udphdr) >> 2);
+ l4_csum = 1;
+ break;
+ case IPPROTO_SCTP:
+ if (first->tx_flags & NBL_TX_FLAGS_TSO)
+ return -1;
+ l4_type = NBL_TX_L4T_RSV;
+ l4_len = (sizeof(struct sctphdr) >> 2);
+ l4_csum = 1;
+ break;
+ default:
+ if (first->tx_flags & NBL_TX_FLAGS_TSO)
+ return -2;
+
+ /* unsopported L4 protocol, device cannot offload L4 checksum,
+ * so software compute L4 checskum
+ */
+ skb_checksum_help(skb);
+ return 0;
+ }
+
+ hdr_param->mac_len = l2_len >> 1;
+ hdr_param->ip_len = l3_len >> 2;
+ hdr_param->l4_len = l4_len;
+ hdr_param->l4_type = l4_type;
+ hdr_param->inner_ip_type = inner_ip_type;
+ hdr_param->l3_csum_en = 0;
+ hdr_param->l4_csum_en = l4_csum;
+
+ return 1;
+}
+
+static inline int nbl_tx_fill_desc(struct nbl_res_tx_ring *tx_ring, u64 dma, u32 size,
+ u16 index, bool first, bool page)
+{
+ struct nbl_tx_buffer *tx_buffer = NBL_TX_BUF(tx_ring, index);
+ struct nbl_ring_desc *tx_desc = NBL_TX_DESC(tx_ring, index);
+
+ tx_buffer->dma = dma;
+ tx_buffer->len = size;
+ tx_buffer->page = page;
+ tx_desc->addr = cpu_to_le64(dma);
+ tx_desc->len = size;
+ if (!first)
+ tx_desc->flags = cpu_to_le16(tx_ring->avail_used_flags | NBL_PACKED_DESC_F_NEXT);
+
+ index++;
+ if (index == tx_ring->desc_num) {
+ index = 0;
+ tx_ring->avail_used_flags ^=
+ 1 << NBL_PACKED_DESC_F_AVAIL |
+ 1 << NBL_PACKED_DESC_F_USED;
+ }
+
+ return index;
+}
+
+static int nbl_map_skb(struct nbl_res_tx_ring *tx_ring, struct sk_buff *skb,
+ u16 first, u16 *desc_index)
+{
+ u16 index = *desc_index;
+ const skb_frag_t *frag;
+ unsigned int frag_num = skb_shinfo(skb)->nr_frags;
+ struct device *dma_dev = NBL_RING_TO_DMA_DEV(tx_ring);
+ unsigned int i;
+ unsigned int size;
+ dma_addr_t dma;
+
+ size = skb_headlen(skb);
+ dma = dma_map_single(dma_dev, skb->data, size, DMA_TO_DEVICE);
+ if (dma_mapping_error(dma_dev, dma))
+ return -1;
+
+ index = nbl_tx_fill_desc(tx_ring, dma, size, index, first, 0);
+
+ if (!frag_num) {
+ *desc_index = index;
+ return 0;
+ }
+
+ frag = &skb_shinfo(skb)->frags[0];
+ for (i = 0; i < frag_num; i++) {
+ size = skb_frag_size(frag);
+ dma = skb_frag_dma_map(dma_dev, frag, 0, size, DMA_TO_DEVICE);
+ if (dma_mapping_error(dma_dev, dma)) {
+ *desc_index = index;
+ return -1;
+ }
+
+ index = nbl_tx_fill_desc(tx_ring, dma, size, index, 0, 1);
+ frag++;
+ }
+
+ *desc_index = index;
+ return 0;
+}
+
+static inline void nbl_tx_fill_tx_extend_header_leonis(union nbl_tx_extend_head *pkthdr,
+ struct nbl_tx_hdr_param *param)
+{
+ pkthdr->mac_len = param->mac_len;
+ pkthdr->ip_len = param->ip_len;
+ pkthdr->l4_len = param->l4_len;
+ pkthdr->l4_type = param->l4_type;
+ pkthdr->inner_ip_type = param->inner_ip_type;
+
+ pkthdr->l4s_sid = param->l4s_sid;
+ pkthdr->l4s_sync_ind = param->l4s_sync_ind;
+ pkthdr->l4s_hdl_ind = param->l4s_hdl_ind;
+ pkthdr->l4s_pbrac_mode = param->l4s_pbrac_mode;
+
+ pkthdr->mss = param->mss;
+ pkthdr->tso = param->tso;
+
+ pkthdr->fwd = param->fwd;
+ pkthdr->rss_lag_en = param->rss_lag_en;
+ pkthdr->dport = param->dport;
+ pkthdr->dport_id = param->dport_id;
+
+ pkthdr->l3_csum_en = param->l3_csum_en;
+ pkthdr->l4_csum_en = param->l4_csum_en;
+}
+
+static bool nbl_skb_is_lacp_or_lldp(struct sk_buff *skb)
+{
+ __be16 protocol;
+
+ protocol = vlan_get_protocol(skb);
+ if (protocol == htons(ETH_P_SLOW) || protocol == htons(ETH_P_LLDP))
+ return true;
+
+ return false;
+}
+
+static int nbl_tx_map(struct nbl_res_tx_ring *tx_ring, struct sk_buff *skb,
+ struct nbl_tx_hdr_param *hdr_param)
+{
+ struct device *dma_dev = NBL_RING_TO_DMA_DEV(tx_ring);
+ struct nbl_tx_buffer *first;
+ struct nbl_ring_desc *first_desc;
+ struct nbl_ring_desc *tx_desc;
+ union nbl_tx_extend_head *pkthdr;
+ dma_addr_t hdrdma;
+ int tso, csum;
+ u16 desc_index = tx_ring->next_to_use;
+ u16 head = desc_index;
+ u16 avail_used_flags = tx_ring->avail_used_flags;
+ u32 pkthdr_len;
+ bool can_push;
+ bool doorbell = true;
+
+ first_desc = NBL_TX_DESC(tx_ring, desc_index);
+ first = NBL_TX_BUF(tx_ring, desc_index);
+ first->gso_segs = 1;
+ first->bytecount = skb->len;
+ first->tx_flags = 0;
+ first->skb = skb;
+ skb_tx_timestamp(skb);
+
+ can_push = !skb_header_cloned(skb) && skb_headroom(skb) >= sizeof(*pkthdr);
+
+ if (can_push)
+ pkthdr = (union nbl_tx_extend_head *)(skb->data - sizeof(*pkthdr));
+ else
+ pkthdr = (union nbl_tx_extend_head *)(skb->cb);
+
+ tso = nbl_tx_tso(first, hdr_param);
+ if (tso < 0) {
+ netdev_err(tx_ring->netdev, "tso ret:%d\n", tso);
+ goto out_drop;
+ }
+
+ csum = nbl_tx_csum(first, hdr_param);
+ if (csum < 0) {
+ netdev_err(tx_ring->netdev, "csum ret:%d\n", csum);
+ goto out_drop;
+ }
+
+ memset(pkthdr, 0, sizeof(*pkthdr));
+ switch (tx_ring->product_type) {
+ case NBL_LEONIS_TYPE:
+ nbl_tx_fill_tx_extend_header_leonis(pkthdr, hdr_param);
+ break;
+ default:
+ netdev_err(tx_ring->netdev, "fill tx extend header failed, product type: %d, eth: %u.\n",
+ tx_ring->product_type, hdr_param->dport_id);
+ goto out_drop;
+ }
+
+ pkthdr_len = sizeof(union nbl_tx_extend_head);
+
+ if (can_push) {
+ __skb_push(skb, pkthdr_len);
+ if (nbl_map_skb(tx_ring, skb, 1, &desc_index))
+ goto dma_map_error;
+ __skb_pull(skb, pkthdr_len);
+ } else {
+ hdrdma = dma_map_single(dma_dev, pkthdr, pkthdr_len, DMA_TO_DEVICE);
+ if (dma_mapping_error(dma_dev, hdrdma)) {
+ tx_ring->tx_stats.tx_dma_busy++;
+ return NETDEV_TX_BUSY;
+ }
+
+ first_desc->addr = cpu_to_le64(hdrdma);
+ first_desc->len = pkthdr_len;
+
+ first->dma = hdrdma;
+ first->len = pkthdr_len;
+
+ desc_index++;
+ if (desc_index == tx_ring->desc_num) {
+ desc_index = 0;
+ tx_ring->avail_used_flags ^= 1 << NBL_PACKED_DESC_F_AVAIL |
+ 1 << NBL_PACKED_DESC_F_USED;
+ }
+ if (nbl_map_skb(tx_ring, skb, 0, &desc_index))
+ goto dma_map_error;
+ }
+
+ /* stats */
+ if (is_multicast_ether_addr(skb->data))
+ tx_ring->tx_stats.tx_multicast_packets += tso;
+ else
+ tx_ring->tx_stats.tx_unicast_packets += tso;
+
+ if (tso > 1) {
+ tx_ring->tx_stats.tso_packets++;
+ tx_ring->tx_stats.tso_bytes += skb->len;
+ }
+ tx_ring->tx_stats.tx_csum_packets += csum;
+
+ tx_desc = NBL_TX_DESC(tx_ring, (desc_index == 0 ? tx_ring->desc_num : desc_index) - 1);
+ tx_desc->flags &= cpu_to_le16(~NBL_PACKED_DESC_F_NEXT);
+ first_desc->len += (hdr_param->total_hlen << NBL_TX_TOTAL_HEADERLEN_SHIFT);
+ first_desc->id = cpu_to_le16(skb_shinfo(skb)->gso_size);
+
+ tx_ring->next_to_use = desc_index;
+ nbl_maybe_stop_tx(tx_ring, DESC_NEEDED);
+ if (nbl_txrx_within_vsi(&tx_ring->vsi_info[NBL_VSI_DATA], tx_ring->queue_index))
+ doorbell = __netdev_tx_sent_queue(txring_txq(tx_ring),
+ first->bytecount, netdev_xmit_more());
+ /* wmb */
+ wmb();
+
+ first->next_to_watch = tx_desc;
+ /* first desc last set flag */
+ if (first_desc == tx_desc)
+ first_desc->flags = cpu_to_le16(avail_used_flags);
+ else
+ first_desc->flags = cpu_to_le16(avail_used_flags | NBL_PACKED_DESC_F_NEXT);
+
+ /* kick doorbell passthrough for performace */
+ if (doorbell)
+ writel(tx_ring->notify_qid, tx_ring->notify_addr);
+
+ return NETDEV_TX_OK;
+
+dma_map_error:
+ while (desc_index != head) {
+ if (unlikely(!desc_index))
+ desc_index = tx_ring->desc_num;
+ desc_index--;
+ nbl_unmap_and_free_tx_resource(tx_ring, NBL_TX_BUF(tx_ring, desc_index),
+ false, false);
+ }
+
+ tx_ring->avail_used_flags = avail_used_flags;
+ tx_ring->tx_stats.tx_dma_busy++;
+ return NETDEV_TX_BUSY;
+
+out_drop:
+ netdev_err(tx_ring->netdev, "tx_map, free_skb\n");
+ tx_ring->tx_stats.tx_skb_free++;
+ dev_kfree_skb_any(skb);
+ return NETDEV_TX_OK;
+}
+
+static netdev_tx_t nbl_res_txrx_start_xmit(struct sk_buff *skb,
+ struct net_device *netdev)
+{
+ struct nbl_resource_mgt *res_mgt =
+ NBL_ADAPTER_TO_RES_MGT(NBL_NETDEV_TO_ADAPTER(netdev));
+ struct nbl_txrx_mgt *txrx_mgt = NBL_RES_MGT_TO_TXRX_MGT(res_mgt);
+ struct nbl_res_tx_ring *tx_ring = txrx_mgt->tx_rings[skb_get_queue_mapping(skb)];
+ struct nbl_tx_hdr_param hdr_param = {
+ .mac_len = 14 >> 1,
+ .ip_len = 20 >> 2,
+ .l4_len = 20 >> 2,
+ .mss = 256,
+ };
+ u16 vlan_tci;
+ u16 vlan_proto;
+ unsigned int count;
+ int ret = 0;
+
+ count = nbl_xmit_desc_count(skb);
+ /* we can not tranmit a packet with more than 32 descriptors */
+ WARN_ON(count > MAX_DESC_NUM_PER_PKT);
+ if (unlikely(nbl_maybe_stop_tx(tx_ring, count))) {
+ if (net_ratelimit())
+ dev_dbg(NBL_RING_TO_DEV(tx_ring), "no desc to tx pkt in queue %u\n",
+ tx_ring->queue_index);
+ tx_ring->tx_stats.tx_busy++;
+ return NETDEV_TX_BUSY;
+ }
+
+ if (tx_ring->vlan_proto || skb_vlan_tag_present(skb)) {
+ if (tx_ring->vlan_proto) {
+ vlan_proto = htons(tx_ring->vlan_proto);
+ vlan_tci = tx_ring->vlan_tci;
+ }
+
+ if (skb_vlan_tag_present(skb)) {
+ vlan_proto = skb->vlan_proto;
+ vlan_tci = skb_vlan_tag_get(skb);
+ }
+
+ skb = vlan_insert_tag_set_proto(skb, vlan_proto, vlan_tci);
+ if (!skb)
+ return NETDEV_TX_OK;
+ }
+ /* for dstore and eth, min packet len is 60 */
+ eth_skb_pad(skb);
+
+ hdr_param.dport_id = tx_ring->eth_id;
+ hdr_param.fwd = 1;
+ hdr_param.rss_lag_en = 0;
+
+ if (nbl_skb_is_lacp_or_lldp(skb)) {
+ hdr_param.fwd = NBL_TX_FWD_TYPE_CPU_ASSIGNED;
+ hdr_param.dport = NBL_TX_DPORT_ETH;
+ }
+
+ /* for unicast packet tx_map all */
+ ret = nbl_tx_map(tx_ring, skb, &hdr_param);
+ return ret;
+}
+
+static void nbl_res_txrx_kick_rx_ring(void *priv, u16 index)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct nbl_hw_ops *hw_ops = NBL_RES_MGT_TO_HW_OPS(res_mgt);
+ struct nbl_notify_param notify_param = {0};
+ struct nbl_res_rx_ring *rx_ring = NBL_RES_MGT_TO_RX_RING(res_mgt, index);
+
+ notify_param.notify_qid = rx_ring->notify_qid;
+ notify_param.tail_ptr = rx_ring->tail_ptr;
+ hw_ops->update_tail_ptr(NBL_RES_MGT_TO_HW_PRIV(res_mgt), ¬ify_param);
+}
+
+static struct nbl_napi_struct *nbl_res_txrx_get_vector_napi(void *priv, u16 index)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct nbl_common_info *common = NBL_RES_MGT_TO_COMMON(res_mgt);
+ struct nbl_txrx_mgt *txrx_mgt = res_mgt->txrx_mgt;
+
+ if (!txrx_mgt->vectors || index >= txrx_mgt->rx_ring_num) {
+ nbl_err(common, NBL_DEBUG_RESOURCE, "vectors not allocated\n");
+ return NULL;
+ }
+
+ return &txrx_mgt->vectors[index]->nbl_napi;
+}
+
+static void nbl_res_txrx_set_vector_info(void *priv, u8 *irq_enable_base,
+ u32 irq_data, u16 index, bool mask_en)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct nbl_common_info *common = NBL_RES_MGT_TO_COMMON(res_mgt);
+ struct nbl_txrx_mgt *txrx_mgt = res_mgt->txrx_mgt;
+
+ if (!txrx_mgt->vectors || index >= txrx_mgt->rx_ring_num) {
+ nbl_err(common, NBL_DEBUG_RESOURCE, "vectors not allocated\n");
+ return;
+ }
+
+ txrx_mgt->vectors[index]->irq_enable_base = irq_enable_base;
+ txrx_mgt->vectors[index]->irq_data = irq_data;
+ txrx_mgt->vectors[index]->net_msix_mask_en = mask_en;
+}
+
+static void nbl_res_get_pt_ops(void *priv, struct nbl_resource_pt_ops *pt_ops)
+{
+ pt_ops->start_xmit = nbl_res_txrx_start_xmit;
+ pt_ops->napi_poll = nbl_res_napi_poll;
+}
+
+static u32 nbl_res_txrx_get_tx_headroom(void *priv)
+{
+ return sizeof(union nbl_tx_extend_head);
+}
+
+static bool nbl_res_is_ctrlq(struct nbl_txrx_mgt *txrx_mgt, u16 qid)
+{
+ u16 ring_num = txrx_mgt->vsi_info[NBL_VSI_CTRL].ring_num;
+ u16 ring_offset = txrx_mgt->vsi_info[NBL_VSI_CTRL].ring_offset;
+
+ if (qid >= ring_offset && qid < ring_offset + ring_num)
+ return true;
+
+ return false;
+}
+
+static void nbl_res_txrx_get_net_stats(void *priv, struct nbl_stats *net_stats)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct nbl_txrx_mgt *txrx_mgt = NBL_RES_MGT_TO_TXRX_MGT(res_mgt);
+ struct nbl_res_rx_ring *rx_ring;
+ struct nbl_res_tx_ring *tx_ring;
+ int i;
+ u64 bytes = 0, packets = 0;
+ u64 tso_packets = 0, tso_bytes = 0;
+ u64 tx_csum_packets = 0;
+ u64 rx_csum_packets = 0, rx_csum_errors = 0;
+ u64 tx_multicast_packets = 0, tx_unicast_packets = 0;
+ u64 rx_multicast_packets = 0, rx_unicast_packets = 0;
+ u64 tx_busy = 0, tx_dma_busy = 0;
+ u64 tx_desc_addr_err_cnt = 0;
+ u64 tx_desc_len_err_cnt = 0;
+ u64 rx_desc_addr_err_cnt = 0;
+ u64 rx_alloc_buf_err_cnt = 0;
+ u64 rx_cache_reuse = 0;
+ u64 rx_cache_full = 0;
+ u64 rx_cache_empty = 0;
+ u64 rx_cache_busy = 0;
+ u64 rx_cache_waive = 0;
+ u64 tx_skb_free = 0;
+ unsigned int start;
+
+ rcu_read_lock();
+ for (i = 0; i < txrx_mgt->rx_ring_num; i++) {
+ if (nbl_res_is_ctrlq(txrx_mgt, i))
+ continue;
+
+ rx_ring = NBL_RES_MGT_TO_RX_RING(res_mgt, i);
+ do {
+ start = u64_stats_fetch_begin(&rx_ring->syncp);
+ bytes += rx_ring->stats.bytes;
+ packets += rx_ring->stats.packets;
+ rx_csum_packets += rx_ring->rx_stats.rx_csum_packets;
+ rx_csum_errors += rx_ring->rx_stats.rx_csum_errors;
+ rx_multicast_packets += rx_ring->rx_stats.rx_multicast_packets;
+ rx_unicast_packets += rx_ring->rx_stats.rx_unicast_packets;
+ rx_desc_addr_err_cnt += rx_ring->rx_stats.rx_desc_addr_err_cnt;
+ rx_alloc_buf_err_cnt += rx_ring->rx_stats.rx_alloc_buf_err_cnt;
+ rx_cache_reuse += rx_ring->rx_stats.rx_cache_reuse;
+ rx_cache_full += rx_ring->rx_stats.rx_cache_full;
+ rx_cache_empty += rx_ring->rx_stats.rx_cache_empty;
+ rx_cache_busy += rx_ring->rx_stats.rx_cache_busy;
+ rx_cache_waive += rx_ring->rx_stats.rx_cache_waive;
+ } while (u64_stats_fetch_retry(&rx_ring->syncp, start));
+ }
+
+ net_stats->rx_packets = packets;
+ net_stats->rx_bytes = bytes;
+
+ net_stats->rx_csum_packets = rx_csum_packets;
+ net_stats->rx_csum_errors = rx_csum_errors;
+ net_stats->rx_multicast_packets = rx_multicast_packets;
+ net_stats->rx_unicast_packets = rx_unicast_packets;
+
+ bytes = 0;
+ packets = 0;
+
+ for (i = 0; i < txrx_mgt->tx_ring_num; i++) {
+ if (nbl_res_is_ctrlq(txrx_mgt, i))
+ continue;
+
+ tx_ring = NBL_RES_MGT_TO_TX_RING(res_mgt, i);
+ do {
+ start = u64_stats_fetch_begin(&tx_ring->syncp);
+ bytes += tx_ring->stats.bytes;
+ packets += tx_ring->stats.packets;
+ tso_packets += tx_ring->tx_stats.tso_packets;
+ tso_bytes += tx_ring->tx_stats.tso_bytes;
+ tx_csum_packets += tx_ring->tx_stats.tx_csum_packets;
+ tx_busy += tx_ring->tx_stats.tx_busy;
+ tx_dma_busy += tx_ring->tx_stats.tx_dma_busy;
+ tx_multicast_packets += tx_ring->tx_stats.tx_multicast_packets;
+ tx_unicast_packets += tx_ring->tx_stats.tx_unicast_packets;
+ tx_skb_free += tx_ring->tx_stats.tx_skb_free;
+ tx_desc_addr_err_cnt += tx_ring->tx_stats.tx_desc_addr_err_cnt;
+ tx_desc_len_err_cnt += tx_ring->tx_stats.tx_desc_len_err_cnt;
+ } while (u64_stats_fetch_retry(&tx_ring->syncp, start));
+ }
+
+ rcu_read_unlock();
+
+ net_stats->tx_bytes = bytes;
+ net_stats->tx_packets = packets;
+ net_stats->tso_packets = tso_packets;
+ net_stats->tso_bytes = tso_bytes;
+ net_stats->tx_csum_packets = tx_csum_packets;
+ net_stats->tx_busy = tx_busy;
+ net_stats->tx_dma_busy = tx_dma_busy;
+ net_stats->tx_multicast_packets = tx_multicast_packets;
+ net_stats->tx_unicast_packets = tx_unicast_packets;
+ net_stats->tx_skb_free = tx_skb_free;
+ net_stats->tx_desc_addr_err_cnt = tx_desc_addr_err_cnt;
+ net_stats->tx_desc_len_err_cnt = tx_desc_len_err_cnt;
+ net_stats->rx_desc_addr_err_cnt = rx_desc_addr_err_cnt;
+ net_stats->rx_alloc_buf_err_cnt = rx_alloc_buf_err_cnt;
+ net_stats->rx_cache_reuse = rx_cache_reuse;
+ net_stats->rx_cache_full = rx_cache_full;
+ net_stats->rx_cache_empty = rx_cache_empty;
+ net_stats->rx_cache_busy = rx_cache_busy;
+ net_stats->rx_cache_waive = rx_cache_waive;
+}
+
+static int
+nbl_res_queue_stop_abnormal_sw_queue(void *priv, u16 local_queue_id, int type)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct nbl_res_vector *vector = NULL;
+ struct nbl_res_tx_ring *tx_ring = NBL_RES_MGT_TO_TX_RING(res_mgt, local_queue_id);
+
+ if (type != NBL_TX)
+ return 0;
+
+ if (tx_ring)
+ vector = NBL_RES_MGT_TO_VECTOR(res_mgt, local_queue_id);
+
+ if (!tx_ring->valid)
+ return -EINVAL;
+
+ if (vector && !vector->started)
+ return -EINVAL;
+
+ if (vector) {
+ vector->started = false;
+ napi_synchronize(&vector->nbl_napi.napi);
+ netif_stop_subqueue(tx_ring->netdev, local_queue_id);
+ }
+
+ return 0;
+}
+
+static dma_addr_t nbl_res_txrx_restore_abnormal_ring(void *priv, int ring_index, int type)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct nbl_res_vector *vector = NULL;
+ struct nbl_res_tx_ring *tx_ring = NBL_RES_MGT_TO_TX_RING(res_mgt, ring_index);
+ struct nbl_res_rx_ring *rx_ring = NBL_RES_MGT_TO_RX_RING(res_mgt, ring_index);
+
+ if (tx_ring)
+ vector = NBL_RES_MGT_TO_VECTOR(res_mgt, ring_index);
+
+ switch (type) {
+ case NBL_TX:
+ if (tx_ring && tx_ring->valid) {
+ nbl_res_txrx_stop_tx_ring(res_mgt, ring_index);
+ return nbl_res_txrx_start_tx_ring(res_mgt, ring_index);
+ } else {
+ return (dma_addr_t)NULL;
+ }
+ break;
+ case NBL_RX:
+ if (rx_ring && rx_ring->valid) {
+ nbl_res_txrx_stop_rx_ring(res_mgt, ring_index);
+ return nbl_res_txrx_start_rx_ring(res_mgt, ring_index, true);
+ } else {
+ return (dma_addr_t)NULL;
+ }
+ break;
+ default:
+ break;
+ }
+
+ return (dma_addr_t)NULL;
+}
+
+static int nbl_res_txrx_restart_abnormal_ring(void *priv, int ring_index, int type)
+{
+ struct nbl_resource_mgt *res_mgt = (struct nbl_resource_mgt *)priv;
+ struct nbl_res_tx_ring *tx_ring = NBL_RES_MGT_TO_TX_RING(res_mgt, ring_index);
+ struct nbl_res_rx_ring *rx_ring = NBL_RES_MGT_TO_RX_RING(res_mgt, ring_index);
+ struct nbl_res_vector *vector = NULL;
+ int ret = 0;
+
+ if (tx_ring)
+ vector = NBL_RES_MGT_TO_VECTOR(res_mgt, ring_index);
+
+ switch (type) {
+ case NBL_TX:
+ if (tx_ring && tx_ring->valid) {
+ writel(tx_ring->notify_qid, tx_ring->notify_addr);
+ netif_start_subqueue(tx_ring->netdev, ring_index);
+ } else {
+ ret = -EINVAL;
+ }
+ break;
+ case NBL_RX:
+ if (rx_ring && rx_ring->valid)
+ nbl_res_txrx_kick_rx_ring(res_mgt, ring_index);
+ else
+ ret = -EINVAL;
+ break;
+ default:
+ break;
+ }
+
+ if (vector) {
+ if (vector->net_msix_mask_en)
+ writel(vector->irq_data, vector->irq_enable_base);
+ vector->started = true;
+ }
+
+ return ret;
+}
+
+static int nbl_res_get_max_mtu(void *priv)
+{
+ return NBL_MAX_JUMBO_FRAME_SIZE - NBL_PKT_HDR_PAD;
+}
+
+/* NBL_TXRX_SET_OPS(ops_name, func)
+ *
+ * Use X Macros to reduce setup and remove codes.
+ */
+#define NBL_TXRX_OPS_TBL \
+do { \
+ NBL_TXRX_SET_OPS(get_resource_pt_ops, nbl_res_get_pt_ops); \
+ NBL_TXRX_SET_OPS(alloc_rings, nbl_res_txrx_alloc_rings); \
+ NBL_TXRX_SET_OPS(remove_rings, nbl_res_txrx_remove_rings); \
+ NBL_TXRX_SET_OPS(start_tx_ring, nbl_res_txrx_start_tx_ring); \
+ NBL_TXRX_SET_OPS(stop_tx_ring, nbl_res_txrx_stop_tx_ring); \
+ NBL_TXRX_SET_OPS(start_rx_ring, nbl_res_txrx_start_rx_ring); \
+ NBL_TXRX_SET_OPS(stop_rx_ring, nbl_res_txrx_stop_rx_ring); \
+ NBL_TXRX_SET_OPS(kick_rx_ring, nbl_res_txrx_kick_rx_ring); \
+ NBL_TXRX_SET_OPS(get_vector_napi, nbl_res_txrx_get_vector_napi); \
+ NBL_TXRX_SET_OPS(set_vector_info, nbl_res_txrx_set_vector_info); \
+ NBL_TXRX_SET_OPS(get_tx_headroom, nbl_res_txrx_get_tx_headroom); \
+ NBL_TXRX_SET_OPS(get_net_stats, nbl_res_txrx_get_net_stats); \
+ NBL_TXRX_SET_OPS(stop_abnormal_sw_queue, nbl_res_queue_stop_abnormal_sw_queue); \
+ NBL_TXRX_SET_OPS(restore_abnormal_ring, nbl_res_txrx_restore_abnormal_ring); \
+ NBL_TXRX_SET_OPS(restart_abnormal_ring, nbl_res_txrx_restart_abnormal_ring); \
+ NBL_TXRX_SET_OPS(register_vsi_ring, nbl_txrx_register_vsi_ring); \
+ NBL_TXRX_SET_OPS(cfg_txrx_vlan, nbl_res_txrx_cfg_txrx_vlan); \
+ NBL_TXRX_SET_OPS(get_max_mtu, nbl_res_get_max_mtu); \
+} while (0)
+
+/* Structure starts here, adding an op should not modify anything below */
+static int nbl_txrx_setup_mgt(struct device *dev, struct nbl_txrx_mgt **txrx_mgt)
+{
+ *txrx_mgt = devm_kzalloc(dev, sizeof(struct nbl_txrx_mgt), GFP_KERNEL);
+ if (!*txrx_mgt)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void nbl_txrx_remove_mgt(struct device *dev, struct nbl_txrx_mgt **txrx_mgt)
+{
+ devm_kfree(dev, *txrx_mgt);
+ *txrx_mgt = NULL;
+}
+
+int nbl_txrx_mgt_start(struct nbl_resource_mgt *res_mgt)
+{
+ struct device *dev;
+ struct nbl_txrx_mgt **txrx_mgt;
+
+ dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ txrx_mgt = &NBL_RES_MGT_TO_TXRX_MGT(res_mgt);
+
+ return nbl_txrx_setup_mgt(dev, txrx_mgt);
+}
+
+void nbl_txrx_mgt_stop(struct nbl_resource_mgt *res_mgt)
+{
+ struct device *dev;
+ struct nbl_txrx_mgt **txrx_mgt;
+
+ dev = NBL_RES_MGT_TO_DEV(res_mgt);
+ txrx_mgt = &NBL_RES_MGT_TO_TXRX_MGT(res_mgt);
+
+ if (!(*txrx_mgt))
+ return;
+
+ nbl_txrx_remove_mgt(dev, txrx_mgt);
+}
+
+int nbl_txrx_setup_ops(struct nbl_resource_ops *res_ops)
+{
+#define NBL_TXRX_SET_OPS(name, func) do {res_ops->NBL_NAME(name) = func; ; } while (0)
+ NBL_TXRX_OPS_TBL;
+#undef NBL_TXRX_SET_OPS
+
+ return 0;
+}
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_txrx.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_txrx.h
new file mode 100644
index 000000000000..22b11d33c799
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_txrx.h
@@ -0,0 +1,188 @@
+/* SPDX-License-Identifier: GPL-2.0*/
+/*
+ * Copyright (c) 2025 Nebula Matrix Limited.
+ * Author:
+ */
+
+#ifndef _NBL_TXRX_H_
+#define _NBL_TXRX_H_
+
+#include "nbl_resource.h"
+
+#define NBL_RING_TO_COMMON(ring) ((ring)->common)
+#define NBL_RING_TO_DEV(ring) ((ring)->dma_dev)
+#define NBL_RING_TO_DMA_DEV(ring) ((ring)->dma_dev)
+
+#define NBL_MIN_DESC_NUM 128
+#define NBL_MAX_DESC_NUM 32768
+
+#define NBL_PACKED_DESC_F_NEXT 1
+#define NBL_PACKED_DESC_F_WRITE 2
+#define NBL_PACKED_DESC_F_AVAIL 7
+#define NBL_PACKED_DESC_F_USED 15
+
+#define NBL_TX_DESC(tx_ring, i) (&(((tx_ring)->desc)[i]))
+#define NBL_RX_DESC(rx_ring, i) (&(((rx_ring)->desc)[i]))
+#define NBL_TX_BUF(tx_ring, i) (&(((tx_ring)->tx_bufs)[i]))
+#define NBL_RX_BUF(rx_ring, i) (&(((rx_ring)->rx_bufs)[i]))
+
+#define NBL_RX_BUF_256 256
+#define NBL_RX_HDR_SIZE NBL_RX_BUF_256
+#define NBL_BUFFER_HDR_LEN (sizeof(struct nbl_rx_extend_head))
+#define NBL_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
+#define NBL_RX_BUFSZ (2048)
+#define NBL_RX_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
+
+#define NBL_TX_TOTAL_HEADERLEN_SHIFT 24
+#define DESC_NEEDED (MAX_SKB_FRAGS + 4)
+#define NBL_TX_POLL_WEIGHT 256
+#define NBL_TXD_DATALEN_BITS 16
+#define NBL_TXD_DATALEN_MAX BIT(NBL_TXD_DATALEN_BITS)
+#define MAX_DESC_NUM_PER_PKT (32)
+
+#define NBL_TX_TSO_MSS_MIN (256)
+#define NBL_TX_TSO_MSS_MAX (16383)
+#define NBL_TX_TSO_L2L3L4_HDR_LEN_MIN (42)
+#define NBL_TX_TSO_L2L3L4_HDR_LEN_MAX (128)
+#define NBL_TX_CHECKSUM_OFFLOAD_L2L3L4_HDR_LEN_MAX (255)
+#define IP_VERSION_V4 (4)
+#define NBL_TX_FLAGS_TSO BIT(0)
+
+/* TX inner IP header type */
+enum nbl_tx_iipt {
+ NBL_TX_IIPT_NONE = 0x0,
+ NBL_TX_IIPT_IPV6 = 0x1,
+ NBL_TX_IIPT_IPV4 = 0x2,
+ NBL_TX_IIPT_RSV = 0x3
+};
+
+/* TX L4 packet type */
+enum nbl_tx_l4t {
+ NBL_TX_L4T_NONE = 0x0,
+ NBL_TX_L4T_TCP = 0x1,
+ NBL_TX_L4T_UDP = 0x2,
+ NBL_TX_L4T_RSV = 0x3
+};
+
+struct nbl_tx_hdr_param {
+ u8 l4s_pbrac_mode;
+ u8 l4s_hdl_ind;
+ u8 l4s_sync_ind;
+ u8 tso;
+ u16 l4s_sid;
+ u16 mss;
+ u8 mac_len;
+ u8 ip_len;
+ u8 l4_len;
+ u8 l4_type;
+ u8 inner_ip_type;
+ u8 l3_csum_en;
+ u8 l4_csum_en;
+ u16 total_hlen;
+ u16 dport_id:10;
+ u16 fwd:2;
+ u16 dport:3;
+ u16 rss_lag_en:1;
+};
+
+union nbl_tx_extend_head {
+ struct {
+ /* DW0 */
+ u32 mac_len :5;
+ u32 ip_len :5;
+ u32 l4_len :4;
+ u32 l4_type :2;
+ u32 inner_ip_type :2;
+ u32 external_ip_type :2;
+ u32 external_ip_len :5;
+ u32 l4_tunnel_type :2;
+ u32 l4_tunnel_len :5;
+ /* DW1 */
+ u32 l4s_sid :10;
+ u32 l4s_sync_ind :1;
+ u32 l4s_redun_ind :1;
+ u32 l4s_redun_head_ind :1;
+ u32 l4s_hdl_ind :1;
+ u32 l4s_pbrac_mode :1;
+ u32 rsv0 :2;
+ u32 mss :14;
+ u32 tso :1;
+ /* DW2 */
+ /* if dport = NBL_TX_DPORT_ETH; dport_info = 0
+ * if dport = NBL_TX_DPORT_HOST; dport_info = host queue id
+ * if dport = NBL_TX_DPORT_ECPU; dport_info = ecpu queue_id
+ */
+ u32 dport_info :11;
+ /* if dport = NBL_TX_DPORT_ETH; dport_id[3:0] = eth port id, dport_id[9:4] = lag id
+ * if dport = NBL_TX_DPORT_HOST; dport_id[9:0] = host vsi_id
+ * if dport = NBL_TX_DPORT_ECPU; dport_id[9:0] = ecpu vsi_id
+ */
+ u32 dport_id :10;
+#define NBL_TX_DPORT_ID_LAG_OFFSET (4)
+ u32 dport :3;
+#define NBL_TX_DPORT_ETH (0)
+#define NBL_TX_DPORT_HOST (1)
+#define NBL_TX_DPORT_ECPU (2)
+#define NBL_TX_DPORT_EMP (3)
+#define NBL_TX_DPORT_BMC (4)
+ u32 fwd :2;
+#define NBL_TX_FWD_TYPE_DROP (0)
+#define NBL_TX_FWD_TYPE_NORMAL (1)
+#define NBL_TX_FWD_TYPE_RSV (2)
+#define NBL_TX_FWD_TYPE_CPU_ASSIGNED (3)
+ u32 rss_lag_en :1;
+ u32 l4_csum_en :1;
+ u32 l3_csum_en :1;
+ u32 rsv1 :3;
+ };
+ u32 dw[3];
+};
+
+struct nbl_rx_extend_head {
+ /* DW0 */
+ /* 0x0:eth, 0x1:host, 0x2:ecpu, 0x3:emp, 0x4:bcm */
+ uint32_t sport :3;
+ uint32_t dport_info :11;
+ /* sport = 0, sport_id[3:0] = eth id,
+ * sport = 1, sport_id[9:0] = host vsi_id,
+ * sport = 2, sport_id[9:0] = ecpu vsi_id,
+ */
+ uint32_t sport_id :10;
+ /* 0x0:drop, 0x1:normal, 0x2:cpu upcall */
+ uint32_t fwd :2;
+ uint32_t rsv0 :6;
+ /* DW1 */
+ uint32_t error_code :6;
+ uint32_t ptype :10;
+ uint32_t profile_id :4;
+ uint32_t checksum_status :1;
+ uint32_t rsv1 :1;
+ uint32_t l4s_sid :10;
+ /* DW2 */
+ uint32_t rsv3 :2;
+ uint32_t l4s_hdl_ind :1;
+ uint32_t l4s_tcp_offset :14;
+ uint32_t l4s_resync_ind :1;
+ uint32_t l4s_check_ind :1;
+ uint32_t l4s_dec_ind :1;
+ uint32_t rsv2 :4;
+ uint32_t num_buffers :8;
+} __packed;
+
+static inline u16 nbl_unused_rx_desc_count(struct nbl_res_rx_ring *ring)
+{
+ u16 ntc = ring->next_to_clean;
+ u16 ntu = ring->next_to_use;
+
+ return ((ntc > ntu) ? 0 : ring->desc_num) + ntc - ntu - 1;
+}
+
+static inline u16 nbl_unused_tx_desc_count(struct nbl_res_tx_ring *ring)
+{
+ u16 ntc = ring->next_to_clean;
+ u16 ntu = ring->next_to_use;
+
+ return ((ntc > ntu) ? 0 : ring->desc_num) + ntc - ntu - 1;
+}
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h
index b0bd29dcb0d1..13ec3ce3c7f2 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h
@@ -88,6 +88,9 @@ struct nbl_hw_ops {
void (*update_adminq_queue_tail_ptr)(void *priv, u16 tail_ptr, u8 txrx);
bool (*check_adminq_dma_err)(void *priv, bool tx);
+ void (*update_tail_ptr)(void *priv, struct nbl_notify_param *param);
+ u8* (*get_tail_ptr)(void *priv);
+
int (*set_spoof_check_addr)(void *priv, u16 vsi_id, u8 *mac);
int (*set_spoof_check_enable)(void *priv, u16 vsi_id, u8 enable);
int (*set_vsi_mtu)(void *priv, u16 vsi_id, u16 mtu_sel);
--
2.43.0
Powered by blists - more mailing lists