| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Thu, 13 Jun 2024 11:22:02 +0300
From: Omer Shpigelman <oshpigelman@...ana.ai>
To: linux-kernel@...r.kernel.org, linux-rdma@...r.kernel.org,
netdev@...r.kernel.org, dri-devel@...ts.freedesktop.org
Cc: ogabbay@...nel.org, oshpigelman@...ana.ai, zyehudai@...ana.ai
Subject: [PATCH 09/15] net: hbl_en: add habanalabs Ethernet driver
This ethernet driver is initialized via auxiliary bus by the hbl_cn
driver.
It serves mainly for control operations that are needed for AI scaling.
Signed-off-by: Omer Shpigelman <oshpigelman@...ana.ai>
Co-developed-by: Abhilash K V <kvabhilash@...ana.ai>
Signed-off-by: Abhilash K V <kvabhilash@...ana.ai>
Co-developed-by: Andrey Agranovich <aagranovich@...ana.ai>
Signed-off-by: Andrey Agranovich <aagranovich@...ana.ai>
Co-developed-by: Bharat Jauhari <bjauhari@...ana.ai>
Signed-off-by: Bharat Jauhari <bjauhari@...ana.ai>
Co-developed-by: David Meriin <dmeriin@...ana.ai>
Signed-off-by: David Meriin <dmeriin@...ana.ai>
Co-developed-by: Sagiv Ozeri <sozeri@...ana.ai>
Signed-off-by: Sagiv Ozeri <sozeri@...ana.ai>
Co-developed-by: Zvika Yehudai <zyehudai@...ana.ai>
Signed-off-by: Zvika Yehudai <zyehudai@...ana.ai>
---
MAINTAINERS | 9 +
drivers/net/ethernet/intel/Kconfig | 18 +
drivers/net/ethernet/intel/Makefile | 1 +
drivers/net/ethernet/intel/hbl_en/Makefile | 9 +
.../net/ethernet/intel/hbl_en/common/Makefile | 3 +
.../net/ethernet/intel/hbl_en/common/hbl_en.c | 1168 +++++++++++++++++
.../net/ethernet/intel/hbl_en/common/hbl_en.h | 206 +++
.../intel/hbl_en/common/hbl_en_dcbnl.c | 101 ++
.../ethernet/intel/hbl_en/common/hbl_en_drv.c | 211 +++
.../intel/hbl_en/common/hbl_en_ethtool.c | 452 +++++++
10 files changed, 2178 insertions(+)
create mode 100644 drivers/net/ethernet/intel/hbl_en/Makefile
create mode 100644 drivers/net/ethernet/intel/hbl_en/common/Makefile
create mode 100644 drivers/net/ethernet/intel/hbl_en/common/hbl_en.c
create mode 100644 drivers/net/ethernet/intel/hbl_en/common/hbl_en.h
create mode 100644 drivers/net/ethernet/intel/hbl_en/common/hbl_en_dcbnl.c
create mode 100644 drivers/net/ethernet/intel/hbl_en/common/hbl_en_drv.c
create mode 100644 drivers/net/ethernet/intel/hbl_en/common/hbl_en_ethtool.c
diff --git a/MAINTAINERS b/MAINTAINERS
index 096439a62129..7301f38e9cfb 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -9617,6 +9617,15 @@ F: include/linux/habanalabs/
F: include/linux/net/intel/cn*
F: include/linux/net/intel/gaudi2*
+HABANALABS ETHERNET DRIVER
+M: Omer Shpigelman <oshpigelman@...ana.ai>
+L: netdev@...r.kernel.org
+S: Supported
+W: https://www.habana.ai
+F: Documentation/networking/device_drivers/ethernet/intel/hbl.rst
+F: drivers/net/ethernet/intel/hbl_en/
+F: include/linux/net/intel/cn*
+
HACKRF MEDIA DRIVER
L: linux-media@...r.kernel.org
S: Orphan
diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig
index 0d1b8a2bae99..5d07349348a0 100644
--- a/drivers/net/ethernet/intel/Kconfig
+++ b/drivers/net/ethernet/intel/Kconfig
@@ -417,4 +417,22 @@ config HABANA_CN
To compile this driver as a module, choose M here. The module
will be called habanalabs_cn.
+config HABANA_EN
+ tristate "HabanaLabs (an Intel Company) Ethernet driver"
+ depends on NETDEVICES && ETHERNET && INET
+ select HABANA_CN
+ help
+ This driver enables Ethernet functionality for the network interfaces
+ that are part of the GAUDI ASIC family of AI Accelerators.
+ For more information on how to identify your adapter, go to the
+ Adapter & Driver ID Guide that can be located at:
+
+ <http://support.intel.com>
+
+ More specific information on configuring the driver is in
+ <file:Documentation/networking/device_drivers/ethernet/intel/hbl.rst>.
+
+ To compile this driver as a module, choose M here. The module
+ will be called habanalabs_en.
+
endif # NET_VENDOR_INTEL
diff --git a/drivers/net/ethernet/intel/Makefile b/drivers/net/ethernet/intel/Makefile
index 10049a28e336..ec62a0227897 100644
--- a/drivers/net/ethernet/intel/Makefile
+++ b/drivers/net/ethernet/intel/Makefile
@@ -20,3 +20,4 @@ obj-$(CONFIG_FM10K) += fm10k/
obj-$(CONFIG_ICE) += ice/
obj-$(CONFIG_IDPF) += idpf/
obj-$(CONFIG_HABANA_CN) += hbl_cn/
+obj-$(CONFIG_HABANA_EN) += hbl_en/
diff --git a/drivers/net/ethernet/intel/hbl_en/Makefile b/drivers/net/ethernet/intel/hbl_en/Makefile
new file mode 100644
index 000000000000..695497ab93b6
--- /dev/null
+++ b/drivers/net/ethernet/intel/hbl_en/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for HabanaLabs (an Intel Company) Ethernet network driver
+#
+
+obj-$(CONFIG_HABANA_EN) := habanalabs_en.o
+
+include $(src)/common/Makefile
+habanalabs_en-y += $(HBL_EN_COMMON_FILES)
diff --git a/drivers/net/ethernet/intel/hbl_en/common/Makefile b/drivers/net/ethernet/intel/hbl_en/common/Makefile
new file mode 100644
index 000000000000..a3ccb5dbf4a6
--- /dev/null
+++ b/drivers/net/ethernet/intel/hbl_en/common/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0-only
+HBL_EN_COMMON_FILES := common/hbl_en_drv.o common/hbl_en.o \
+ common/hbl_en_ethtool.o common/hbl_en_dcbnl.o
diff --git a/drivers/net/ethernet/intel/hbl_en/common/hbl_en.c b/drivers/net/ethernet/intel/hbl_en/common/hbl_en.c
new file mode 100644
index 000000000000..066be5ac2d84
--- /dev/null
+++ b/drivers/net/ethernet/intel/hbl_en/common/hbl_en.c
@@ -0,0 +1,1168 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2020-2024 HabanaLabs, Ltd.
+ * Copyright (C) 2023-2024, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+#include "hbl_en.h"
+#include <linux/inetdevice.h>
+
+#define TX_TIMEOUT (5 * HZ)
+#define PORT_RESET_TIMEOUT_MSEC (60 * 1000ull) /* 60s */
+
+/**
+ * struct hbl_en_tx_pkt_work - used to schedule a work of a Tx packet.
+ * @tx_work: workqueue object to run when packet needs to be sent.
+ * @port: pointer to current port structure.
+ * @skb: copy of the packet to send.
+ */
+struct hbl_en_tx_pkt_work {
+ struct work_struct tx_work;
+ struct hbl_en_port *port;
+ struct sk_buff *skb;
+};
+
+static int hbl_en_napi_poll(struct napi_struct *napi, int budget);
+static int hbl_en_port_open(struct hbl_en_port *port);
+
+static int hbl_en_ports_reopen(struct hbl_aux_dev *aux_dev)
+{
+ struct hbl_en_device *hdev = aux_dev->priv;
+ struct hbl_en_port *port;
+ int rc = 0, i;
+
+ for (i = 0; i < hdev->max_num_of_ports; i++) {
+ if (!(hdev->ports_mask & BIT(i)))
+ continue;
+
+ port = &hdev->ports[i];
+
+ /* It could be that the port was shutdown by 'ip link set down' and there is no need
+ * in reopening it.
+ * Since we mark the ports as in reset even if they are disabled, we clear the flag
+ * here anyway.
+ * See hbl_en_ports_stop_prepare() for more info.
+ */
+ if (!netif_running(port->ndev)) {
+ atomic_set(&port->in_reset, 0);
+ continue;
+ }
+
+ rc = hbl_en_port_open(port);
+
+ atomic_set(&port->in_reset, 0);
+
+ if (rc)
+ break;
+ }
+
+ hdev->in_reset = false;
+
+ return rc;
+}
+
+static void hbl_en_port_fini(struct hbl_en_port *port)
+{
+ if (port->rx_wq)
+ destroy_workqueue(port->rx_wq);
+}
+
+static int hbl_en_port_init(struct hbl_en_port *port)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ u32 port_idx = port->idx;
+ char wq_name[32];
+ int rc;
+
+ if (hdev->poll_enable) {
+ memset(wq_name, 0, sizeof(wq_name));
+ snprintf(wq_name, sizeof(wq_name) - 1, "hbl%u-port%d-rx-wq", hdev->core_dev_id,
+ port_idx);
+ port->rx_wq = alloc_ordered_workqueue(wq_name, 0);
+ if (!port->rx_wq) {
+ dev_err(hdev->dev, "Failed to allocate Rx WQ\n");
+ rc = -ENOMEM;
+ goto fail;
+ }
+ }
+
+ hbl_en_ethtool_init_coalesce(port);
+
+ return 0;
+
+fail:
+ hbl_en_port_fini(port);
+
+ return rc;
+}
+
+static void _hbl_en_set_port_status(struct hbl_en_port *port, bool up)
+{
+ struct net_device *ndev = port->ndev;
+ u32 port_idx = port->idx;
+
+ if (up) {
+ netif_carrier_on(ndev);
+ netif_wake_queue(ndev);
+ } else {
+ netif_carrier_off(ndev);
+ netif_stop_queue(ndev);
+ }
+
+ /* Unless link events are getting through the EQ, no need to print about link down events
+ * during port reset
+ */
+ if (port->hdev->has_eq || up || !atomic_read(&port->in_reset))
+ netdev_info(port->ndev, "link %s, port %d\n", up ? "up" : "down", port_idx);
+}
+
+static void hbl_en_set_port_status(struct hbl_aux_dev *aux_dev, u32 port_idx, bool up)
+{
+ struct hbl_en_port *port = HBL_EN_PORT(aux_dev, port_idx);
+
+ _hbl_en_set_port_status(port, up);
+}
+
+static bool hbl_en_is_port_open(struct hbl_aux_dev *aux_dev, u32 port_idx)
+{
+ struct hbl_en_port *port = HBL_EN_PORT(aux_dev, port_idx);
+
+ return port->is_initialized;
+}
+
+/* get the src IP as it is done in devinet_ioctl() */
+static int hbl_en_get_src_ip(struct hbl_aux_dev *aux_dev, u32 port_idx, u32 *src_ip)
+{
+ struct hbl_en_port *port = HBL_EN_PORT(aux_dev, port_idx);
+ struct net_device *ndev = port->ndev;
+ struct in_device *in_dev;
+ struct in_ifaddr *ifa;
+ int rc = 0;
+
+ /* for the case where no src IP is configured */
+ *src_ip = 0;
+
+ /* rtnl lock should be acquired in relevant flows before taking configuration lock */
+ if (!rtnl_is_locked()) {
+ netdev_err(port->ndev, "Rtnl lock is not acquired, can't proceed\n");
+ rc = -EFAULT;
+ goto out;
+ }
+
+ in_dev = __in_dev_get_rtnl(ndev);
+ if (!in_dev) {
+ netdev_err(port->ndev, "Failed to get IPv4 struct\n");
+ rc = -EFAULT;
+ goto out;
+ }
+
+ ifa = rtnl_dereference(in_dev->ifa_list);
+
+ while (ifa) {
+ if (!strcmp(ndev->name, ifa->ifa_label)) {
+ /* convert the BE to native and later on it will be
+ * written to the HW as LE in QPC_SET
+ */
+ *src_ip = be32_to_cpu(ifa->ifa_local);
+ break;
+ }
+ ifa = rtnl_dereference(ifa->ifa_next);
+ }
+out:
+ return rc;
+}
+
+static void hbl_en_reset_stats(struct hbl_aux_dev *aux_dev, u32 port_idx)
+{
+ struct hbl_en_port *port = HBL_EN_PORT(aux_dev, port_idx);
+
+ port->net_stats.rx_packets = 0;
+ port->net_stats.tx_packets = 0;
+ port->net_stats.rx_bytes = 0;
+ port->net_stats.tx_bytes = 0;
+ port->net_stats.tx_errors = 0;
+ atomic64_set(&port->net_stats.rx_dropped, 0);
+ atomic64_set(&port->net_stats.tx_dropped, 0);
+}
+
+static u32 hbl_en_get_mtu(struct hbl_aux_dev *aux_dev, u32 port_idx)
+{
+ struct hbl_en_port *port = HBL_EN_PORT(aux_dev, port_idx);
+ struct net_device *ndev = port->ndev;
+ u32 mtu;
+
+ if (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ netdev_err(ndev, "port is in reset, can't get MTU\n");
+ return 0;
+ }
+
+ mtu = ndev->mtu;
+
+ atomic_set(&port->in_reset, 0);
+
+ return mtu;
+}
+
+static u32 hbl_en_get_pflags(struct hbl_aux_dev *aux_dev, u32 port_idx)
+{
+ struct hbl_en_port *port = HBL_EN_PORT(aux_dev, port_idx);
+
+ return port->pflags;
+}
+
+static void hbl_en_set_dev_lpbk(struct hbl_aux_dev *aux_dev, u32 port_idx, bool enable)
+{
+ struct hbl_en_port *port = HBL_EN_PORT(aux_dev, port_idx);
+ struct net_device *ndev = port->ndev;
+
+ if (enable)
+ ndev->features |= NETIF_F_LOOPBACK;
+ else
+ ndev->features &= ~NETIF_F_LOOPBACK;
+}
+
+/* This function should be called after ctrl_lock was taken */
+static int hbl_en_port_open_locked(struct hbl_en_port *port)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ struct net_device *ndev = port->ndev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+ int rc;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ if (port->is_initialized)
+ return 0;
+
+ if (!hdev->poll_enable)
+ netif_napi_add(ndev, &port->napi, hbl_en_napi_poll);
+
+ rc = aux_ops->port_hw_init(aux_dev, port_idx);
+ if (rc) {
+ netdev_err(ndev, "Failed to configure the HW, rc %d\n", rc);
+ goto hw_init_fail;
+ }
+
+ if (!hdev->poll_enable)
+ napi_enable(&port->napi);
+
+ rc = hdev->asic_funcs.eth_port_open(port);
+ if (rc) {
+ netdev_err(ndev, "Failed to init H/W, rc %d\n", rc);
+ goto port_open_fail;
+ }
+
+ rc = aux_ops->update_mtu(aux_dev, port_idx, ndev->mtu);
+ if (rc) {
+ netdev_err(ndev, "MTU update failed, rc %d\n", rc);
+ goto update_mtu_fail;
+ }
+
+ rc = aux_ops->phy_init(aux_dev, port_idx);
+ if (rc) {
+ netdev_err(ndev, "PHY init failed, rc %d\n", rc);
+ goto phy_init_fail;
+ }
+
+ netif_start_queue(ndev);
+
+ port->is_initialized = true;
+
+ return 0;
+
+phy_init_fail:
+ /* no need to revert the MTU change, it will be updated on next port open */
+update_mtu_fail:
+ hdev->asic_funcs.eth_port_close(port);
+port_open_fail:
+ if (!hdev->poll_enable)
+ napi_disable(&port->napi);
+
+ aux_ops->port_hw_fini(aux_dev, port_idx);
+hw_init_fail:
+ if (!hdev->poll_enable)
+ netif_napi_del(&port->napi);
+
+ return rc;
+}
+
+static int hbl_en_port_open(struct hbl_en_port *port)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+ int rc;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ aux_ops->ctrl_lock(aux_dev, port_idx);
+ rc = hbl_en_port_open_locked(port);
+ aux_ops->ctrl_unlock(aux_dev, port_idx);
+
+ return rc;
+}
+
+static int hbl_en_open(struct net_device *netdev)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(netdev);
+ int rc;
+
+ if (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ netdev_err(netdev, "port is in reset, can't open it\n");
+ return -EBUSY;
+ }
+
+ rc = hbl_en_port_open(port);
+
+ atomic_set(&port->in_reset, 0);
+
+ return rc;
+}
+
+/* This function should be called after ctrl_lock was taken */
+static void hbl_en_port_close_locked(struct hbl_en_port *port)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ if (!port->is_initialized)
+ return;
+
+ port->is_initialized = false;
+
+ /* verify that the port is marked as closed before continuing */
+ mb();
+
+ /* Print if not in hard reset flow e.g. from ip cmd */
+ if (!hdev->in_reset && netif_carrier_ok(port->ndev))
+ netdev_info(port->ndev, "port was closed\n");
+
+ /* disable the PHY here so no link changes will occur from this point forward */
+ aux_ops->phy_fini(aux_dev, port_idx);
+
+ /* disable Tx SW flow */
+ netif_carrier_off(port->ndev);
+ netif_tx_disable(port->ndev);
+
+ /* stop Tx/Rx HW */
+ aux_ops->port_hw_fini(aux_dev, port_idx);
+
+ /* disable Tx/Rx QPs */
+ hdev->asic_funcs.eth_port_close(port);
+
+ /* stop Rx SW flow */
+ if (hdev->poll_enable) {
+ hbl_en_rx_poll_stop(port);
+ } else {
+ napi_disable(&port->napi);
+ netif_napi_del(&port->napi);
+ }
+
+ /* Explicitly count the port close operations as we don't get a link event for this.
+ * Upon port open we receive a link event, hence no additional action required.
+ */
+ aux_ops->port_toggle_count(aux_dev, port_idx);
+}
+
+static void hbl_en_port_close(struct hbl_en_port *port)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ aux_ops->ctrl_lock(aux_dev, port_idx);
+ hbl_en_port_close_locked(port);
+ aux_ops->ctrl_unlock(aux_dev, port_idx);
+}
+
+/* This function should be called after ctrl_lock was taken */
+static int __hbl_en_port_reset_locked(struct hbl_en_port *port)
+{
+ hbl_en_port_close_locked(port);
+
+ return hbl_en_port_open_locked(port);
+}
+
+/* This function should be called after ctrl_lock was taken */
+int hbl_en_port_reset_locked(struct hbl_aux_dev *aux_dev, u32 port_idx)
+{
+ struct hbl_en_port *port = HBL_EN_PORT(aux_dev, port_idx);
+
+ return __hbl_en_port_reset_locked(port);
+}
+
+int hbl_en_port_reset(struct hbl_en_port *port)
+{
+ hbl_en_port_close(port);
+
+ /* Sleep in order to let obsolete events to be dropped before re-opening the port */
+ msleep(20);
+
+ return hbl_en_port_open(port);
+}
+
+static int hbl_en_close(struct net_device *netdev)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(netdev);
+ struct hbl_en_device *hdev = port->hdev;
+ ktime_t timeout;
+
+ /* Looks like the return value of this function is not checked, so we can't just return
+ * EBUSY if the port is under reset. We need to wait until the reset is finished and then
+ * close the port. Otherwise the netdev will set the port as closed although port_close()
+ * wasn't called. Only if we waited long enough and the reset hasn't finished, we can return
+ * an error without actually closing the port as it is a fatal flow anyway.
+ */
+ timeout = ktime_add_ms(ktime_get(), PORT_RESET_TIMEOUT_MSEC);
+ while (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ /* If this is called from unregister_netdev() then the port was already closed and
+ * hence we can safely return.
+ * We could have just check the port_open boolean, but that might hide some future
+ * bugs. Hence it is better to use a dedicated flag for that.
+ */
+ if (READ_ONCE(hdev->in_teardown))
+ return 0;
+
+ usleep_range(50, 200);
+ if (ktime_compare(ktime_get(), timeout) > 0) {
+ netdev_crit(netdev,
+ "Timeout while waiting for port to finish reset, can't close it\n"
+ );
+ return -EBUSY;
+ }
+ }
+
+ hbl_en_port_close(port);
+
+ atomic_set(&port->in_reset, 0);
+
+ return 0;
+}
+
+/**
+ * hbl_en_ports_stop_prepare() - stop the Rx and Tx and synchronize with other reset flows.
+ * @aux_dev: habanalabs auxiliary device structure.
+ *
+ * This function makes sure that during the reset no packets will be processed and that
+ * ndo_open/ndo_close do not open/close the ports.
+ * A hard reset might occur right after the driver was loaded, which means before the ports
+ * initialization was finished. Therefore, even if the ports are not yet open, we mark it as in
+ * reset in order to avoid races. We clear the in reset flag later on when reopening the ports.
+ */
+static void hbl_en_ports_stop_prepare(struct hbl_aux_dev *aux_dev)
+{
+ struct hbl_en_device *hdev = aux_dev->priv;
+ struct hbl_en_port *port;
+ ktime_t timeout;
+ int i;
+
+ /* Check if the ports where initialized. If not, we shouldn't mark them as in reset because
+ * they will fail to get opened.
+ */
+ if (!hdev->is_initialized || hdev->in_reset)
+ return;
+
+ for (i = 0; i < hdev->max_num_of_ports; i++) {
+ if (!(hdev->ports_mask & BIT(i)))
+ continue;
+
+ port = &hdev->ports[i];
+
+ /* This function is competing with reset from ethtool/ip, so try to take the
+ * in_reset atomic and if we are already in a middle of reset, wait until reset
+ * function is finished.
+ * Reset function is designed to always finish (could take up to a few seconds in
+ * worst case).
+ * We mark also closed ports as in reset so they won't be able to get opened while
+ * the device in under reset.
+ */
+
+ timeout = ktime_add_ms(ktime_get(), PORT_RESET_TIMEOUT_MSEC);
+ while (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ usleep_range(50, 200);
+ if (ktime_compare(ktime_get(), timeout) > 0) {
+ netdev_crit(port->ndev,
+ "Timeout while waiting for port %d to finish reset\n",
+ port->idx);
+ break;
+ }
+ }
+ }
+
+ hdev->in_reset = true;
+}
+
+static void hbl_en_ports_stop(struct hbl_aux_dev *aux_dev)
+{
+ struct hbl_en_device *hdev = aux_dev->priv;
+ struct hbl_en_port *port;
+ int i;
+
+ for (i = 0; i < hdev->max_num_of_ports; i++) {
+ if (!(hdev->ports_mask & BIT(i)))
+ continue;
+
+ port = &hdev->ports[i];
+
+ if (netif_running(port->ndev))
+ hbl_en_port_close(port);
+ }
+}
+
+static int hbl_en_change_mtu(struct net_device *netdev, int new_mtu)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(netdev);
+ int rc = 0;
+
+ if (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ netdev_err(netdev, "port is in reset, can't change MTU\n");
+ return -EBUSY;
+ }
+
+ if (netif_running(port->ndev)) {
+ hbl_en_port_close(port);
+
+ /* Sleep in order to let obsolete events to be dropped before re-opening the port */
+ msleep(20);
+
+ netdev->mtu = new_mtu;
+
+ rc = hbl_en_port_open(port);
+ if (rc)
+ netdev_err(netdev, "Failed to reinit port for MTU change, rc %d\n", rc);
+ } else {
+ netdev->mtu = new_mtu;
+ }
+
+ atomic_set(&port->in_reset, 0);
+
+ return rc;
+}
+
+/* Swap source and destination MAC addresses */
+static inline void swap_l2(char *buf)
+{
+ u16 *eth_hdr, tmp;
+
+ eth_hdr = (u16 *)buf;
+ tmp = eth_hdr[0];
+ eth_hdr[0] = eth_hdr[3];
+ eth_hdr[3] = tmp;
+ tmp = eth_hdr[1];
+ eth_hdr[1] = eth_hdr[4];
+ eth_hdr[4] = tmp;
+ tmp = eth_hdr[2];
+ eth_hdr[2] = eth_hdr[5];
+ eth_hdr[5] = tmp;
+}
+
+/* Swap source and destination IP addresses
+ */
+static inline void swap_l3(char *buf)
+{
+ u32 tmp;
+
+ /* skip the Ethernet header and the IP header till source IP address */
+ buf += ETH_HLEN + 12;
+ tmp = ((u32 *)buf)[0];
+ ((u32 *)buf)[0] = ((u32 *)buf)[1];
+ ((u32 *)buf)[1] = tmp;
+}
+
+static void do_tx_swap(struct hbl_en_port *port, struct sk_buff *skb)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ u16 *tmp_buff = (u16 *)skb->data;
+ u32 port_idx = port->idx;
+
+ /* First, let's print the SKB we got */
+ dev_dbg_ratelimited(hdev->dev,
+ "Send [P%d]: dst-mac:%04x%04x%04x, src-mac:%04x%04x%04x, eth-type:%04x, len:%u\n",
+ port_idx, swab16(tmp_buff[0]), swab16(tmp_buff[1]), swab16(tmp_buff[2]),
+ swab16(tmp_buff[3]), swab16(tmp_buff[4]), swab16(tmp_buff[5]),
+ swab16(tmp_buff[6]), skb->len);
+
+ /* Before submit it to HW, in case this is ipv4 pkt, swap eth/ip addresses.
+ * that way, we may send ECMP (ping) to ourselves in LB cases.
+ */
+ swap_l2(skb->data);
+ if (swab16(tmp_buff[6]) == ETH_P_IP)
+ swap_l3(skb->data);
+}
+
+static bool is_pkt_swap_enabled(struct hbl_en_device *hdev)
+{
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ return aux_ops->is_eth_lpbk(aux_dev);
+}
+
+static bool is_tx_disabled(struct hbl_en_port *port)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ return aux_ops->get_mac_lpbk(aux_dev, port_idx) && !is_pkt_swap_enabled(hdev);
+}
+
+static netdev_tx_t hbl_en_handle_tx(struct hbl_en_port *port, struct sk_buff *skb)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ netdev_tx_t ret;
+
+ if (skb->len <= 0 || is_tx_disabled(port))
+ goto free_skb;
+
+ if (skb->len > hdev->max_frm_len) {
+ netdev_err(port->ndev, "Tx pkt size %uB exceeds maximum of %uB\n", skb->len,
+ hdev->max_frm_len);
+ goto free_skb;
+ }
+
+ if (is_pkt_swap_enabled(hdev))
+ do_tx_swap(port, skb);
+
+ /* Pad the ethernet packets to the minimum frame size as the NIC hw doesn't do it.
+ * eth_skb_pad() frees the packet on failure, so just increment the dropped counter and
+ * return as success to avoid a retry.
+ */
+ if (skb_put_padto(skb, hdev->pad_size)) {
+ dev_err_ratelimited(hdev->dev, "Padding failed, the skb is dropped\n");
+ atomic64_inc(&port->net_stats.tx_dropped);
+ return NETDEV_TX_OK;
+ }
+
+ ret = hdev->asic_funcs.write_pkt_to_hw(port, skb);
+ if (ret == NETDEV_TX_OK) {
+ port->net_stats.tx_packets++;
+ port->net_stats.tx_bytes += skb->len;
+ }
+
+ return ret;
+
+free_skb:
+ dev_kfree_skb_any(skb);
+ return NETDEV_TX_OK;
+}
+
+static netdev_tx_t hbl_en_start_xmit(struct sk_buff *skb, struct net_device *netdev)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(netdev);
+ struct hbl_en_device *hdev;
+
+ hdev = port->hdev;
+
+ return hbl_en_handle_tx(port, skb);
+}
+
+static int hbl_en_set_port_mac_loopback(struct hbl_en_port *port, bool enable)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ struct net_device *ndev = port->ndev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+ int rc;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ rc = aux_ops->set_mac_lpbk(aux_dev, port_idx, enable);
+ if (rc)
+ return rc;
+
+ netdev_info(ndev, "port %u: mac loopback is %s\n", port_idx,
+ enable ? "enabled" : "disabled");
+
+ if (netif_running(ndev)) {
+ rc = hbl_en_port_reset(port);
+ if (rc) {
+ netdev_err(ndev, "Failed to reset port %u, rc %d\n", port_idx, rc);
+ return rc;
+ }
+ }
+
+ return 0;
+}
+
+static int hbl_en_set_features(struct net_device *netdev, netdev_features_t features)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(netdev);
+ netdev_features_t changed;
+ int rc = 0;
+
+ if (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ netdev_err(netdev, "port %d is in reset, can't update settings", port->idx);
+ return -EBUSY;
+ }
+
+ changed = netdev->features ^ features;
+
+ if (changed & NETIF_F_LOOPBACK)
+ rc = hbl_en_set_port_mac_loopback(port, !!(features & NETIF_F_LOOPBACK));
+
+ atomic_set(&port->in_reset, 0);
+
+ return rc;
+}
+
+static void hbl_en_handle_tx_timeout(struct net_device *netdev, unsigned int txqueue)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(netdev);
+
+ port->net_stats.tx_errors++;
+ atomic64_inc(&port->net_stats.tx_dropped);
+}
+
+static void hbl_en_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(dev);
+
+ stats->rx_bytes = port->net_stats.rx_bytes;
+ stats->tx_bytes = port->net_stats.tx_bytes;
+ stats->rx_packets = port->net_stats.rx_packets;
+ stats->tx_packets = port->net_stats.tx_packets;
+ stats->tx_errors = port->net_stats.tx_errors;
+ stats->tx_dropped = (u64)atomic64_read(&port->net_stats.tx_dropped);
+ stats->rx_dropped = (u64)atomic64_read(&port->net_stats.rx_dropped);
+}
+
+static const struct net_device_ops hbl_en_netdev_ops = {
+ .ndo_open = hbl_en_open,
+ .ndo_stop = hbl_en_close,
+ .ndo_start_xmit = hbl_en_start_xmit,
+ .ndo_validate_addr = eth_validate_addr,
+ .ndo_change_mtu = hbl_en_change_mtu,
+ .ndo_set_features = hbl_en_set_features,
+ .ndo_get_stats64 = hbl_en_get_stats64,
+ .ndo_tx_timeout = hbl_en_handle_tx_timeout,
+};
+
+static void hbl_en_set_ops(struct net_device *ndev)
+{
+ ndev->netdev_ops = &hbl_en_netdev_ops;
+ ndev->ethtool_ops = hbl_en_ethtool_get_ops(ndev);
+#ifdef CONFIG_DCB
+ ndev->dcbnl_ops = &hbl_en_dcbnl_ops;
+#endif
+}
+
+static int hbl_en_port_register(struct hbl_en_port *port)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+ struct hbl_en_port **ptr;
+ struct net_device *ndev;
+ int rc;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ ndev = alloc_etherdev(sizeof(struct hbl_en_port *));
+ if (!ndev) {
+ dev_err(hdev->dev, "netdevice %d alloc failed\n", port_idx);
+ return -ENOMEM;
+ }
+
+ port->ndev = ndev;
+ SET_NETDEV_DEV(ndev, &hdev->pdev->dev);
+ ptr = netdev_priv(ndev);
+ *ptr = port;
+
+ /* necessary for creating multiple interfaces */
+ ndev->dev_port = port_idx;
+
+ hbl_en_set_ops(ndev);
+
+ ndev->watchdog_timeo = TX_TIMEOUT;
+ ndev->min_mtu = hdev->min_raw_mtu;
+ ndev->max_mtu = hdev->max_raw_mtu;
+
+ /* Add loopback capability to the device. */
+ ndev->hw_features |= NETIF_F_LOOPBACK;
+
+ /* If this port was set to loopback, set it also to the ndev features */
+ if (aux_ops->get_mac_lpbk(aux_dev, port_idx))
+ ndev->features |= NETIF_F_LOOPBACK;
+
+ eth_hw_addr_set(ndev, port->mac_addr);
+
+ /* It's more an intelligent poll wherein, we enable the Rx completion EQE event and then
+ * start the poll from there.
+ * Inside the polling thread, we read packets from hardware and then reschedule the poll
+ * only if there are more packets to be processed. Else we re-enable the CQ Arm interrupt
+ * and exit the poll.
+ */
+ if (hdev->poll_enable)
+ hbl_en_rx_poll_trigger_init(port);
+
+ netif_carrier_off(ndev);
+
+ rc = register_netdev(ndev);
+ if (rc) {
+ dev_err(hdev->dev, "Could not register netdevice %d\n", port_idx);
+ goto err;
+ }
+
+ return 0;
+
+err:
+ if (ndev) {
+ free_netdev(ndev);
+ port->ndev = NULL;
+ }
+
+ return rc;
+}
+
+static void dump_swap_pkt(struct hbl_en_port *port, struct sk_buff *skb)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ u16 *tmp_buff = (u16 *)skb->data;
+ u32 port_idx = port->idx;
+
+ /* The SKB is ready now (before stripping-out the L2), print its content */
+ dev_dbg_ratelimited(hdev->dev,
+ "Recv [P%d]: dst-mac:%04x%04x%04x, src-mac:%04x%04x%04x, eth-type:%04x, len:%u\n",
+ port_idx, swab16(tmp_buff[0]), swab16(tmp_buff[1]), swab16(tmp_buff[2]),
+ swab16(tmp_buff[3]), swab16(tmp_buff[4]), swab16(tmp_buff[5]),
+ swab16(tmp_buff[6]), skb->len);
+}
+
+int hbl_en_handle_rx(struct hbl_en_port *port, int budget)
+{
+ struct hbl_en_device *hdev = port->hdev;
+ enum hbl_en_eth_pkt_status pkt_status;
+ struct net_device *ndev = port->ndev;
+ int rc, pkt_count = 0;
+ struct sk_buff *skb;
+ void *pkt_addr;
+ u32 pkt_size;
+
+ if (!netif_carrier_ok(ndev))
+ return 0;
+
+ while (pkt_count < budget) {
+ pkt_status = hdev->asic_funcs.read_pkt_from_hw(port, &pkt_addr, &pkt_size);
+
+ if (pkt_status == ETH_PKT_NONE)
+ break;
+
+ pkt_count++;
+
+ if (pkt_status == ETH_PKT_DROP) {
+ atomic64_inc(&port->net_stats.rx_dropped);
+ continue;
+ }
+
+ if (hdev->poll_enable)
+ skb = __netdev_alloc_skb_ip_align(ndev, pkt_size, GFP_KERNEL);
+ else
+ skb = napi_alloc_skb(&port->napi, pkt_size);
+
+ if (!skb) {
+ atomic64_inc(&port->net_stats.rx_dropped);
+ break;
+ }
+
+ skb_copy_to_linear_data(skb, pkt_addr, pkt_size);
+ skb_put(skb, pkt_size);
+
+ if (is_pkt_swap_enabled(hdev))
+ dump_swap_pkt(port, skb);
+
+ skb->protocol = eth_type_trans(skb, ndev);
+
+ /* Zero the packet buffer memory to avoid leak in case of wrong
+ * size is used when next packet populates the same memory
+ */
+ memset(pkt_addr, 0, pkt_size);
+
+ /* polling is done in thread context and hence BH should be disabled */
+ if (hdev->poll_enable)
+ local_bh_disable();
+
+ rc = netif_receive_skb(skb);
+
+ if (hdev->poll_enable)
+ local_bh_enable();
+
+ if (rc == NET_RX_SUCCESS) {
+ port->net_stats.rx_packets++;
+ port->net_stats.rx_bytes += pkt_size;
+ } else {
+ atomic64_inc(&port->net_stats.rx_dropped);
+ }
+ }
+
+ return pkt_count;
+}
+
+static bool __hbl_en_rx_poll_schedule(struct hbl_en_port *port, unsigned long delay)
+{
+ return queue_delayed_work(port->rx_wq, &port->rx_poll_work, delay);
+}
+
+static void hbl_en_rx_poll_work(struct work_struct *work)
+{
+ struct hbl_en_port *port = container_of(work, struct hbl_en_port, rx_poll_work.work);
+ struct hbl_en_device *hdev = port->hdev;
+ int pkt_count;
+
+ pkt_count = hbl_en_handle_rx(port, NAPI_POLL_WEIGHT);
+
+ /* Reschedule the poll if we have consumed budget which means we still have packets to
+ * process. Else re-enable the Rx IRQs and exit the work.
+ */
+ if (pkt_count < NAPI_POLL_WEIGHT)
+ hdev->asic_funcs.reenable_rx_irq(port);
+ else
+ __hbl_en_rx_poll_schedule(port, 0);
+}
+
+/* Rx poll init and trigger routines are used in event-driven setups where
+ * Rx polling is initialized once during init or open and started/triggered by the event handler.
+ */
+void hbl_en_rx_poll_trigger_init(struct hbl_en_port *port)
+{
+ INIT_DELAYED_WORK(&port->rx_poll_work, hbl_en_rx_poll_work);
+}
+
+bool hbl_en_rx_poll_start(struct hbl_en_port *port)
+{
+ return __hbl_en_rx_poll_schedule(port, msecs_to_jiffies(1));
+}
+
+void hbl_en_rx_poll_stop(struct hbl_en_port *port)
+{
+ cancel_delayed_work_sync(&port->rx_poll_work);
+}
+
+static int hbl_en_napi_poll(struct napi_struct *napi, int budget)
+{
+ struct hbl_en_port *port = container_of(napi, struct hbl_en_port, napi);
+ struct hbl_en_device *hdev = port->hdev;
+ int pkt_count;
+
+ /* exit if we are called by netpoll as we free the Tx ring via EQ (if enabled) */
+ if (!budget)
+ return 0;
+
+ pkt_count = hbl_en_handle_rx(port, budget);
+
+ /* If budget not fully consumed, exit the polling mode */
+ if (pkt_count < budget) {
+ napi_complete_done(napi, pkt_count);
+ hdev->asic_funcs.reenable_rx_irq(port);
+ }
+
+ return pkt_count;
+}
+
+static void hbl_en_port_unregister(struct hbl_en_port *port)
+{
+ struct net_device *ndev = port->ndev;
+
+ unregister_netdev(ndev);
+ free_netdev(ndev);
+ port->ndev = NULL;
+}
+
+static int hbl_en_set_asic_funcs(struct hbl_en_device *hdev)
+{
+ switch (hdev->asic_type) {
+ case HBL_ASIC_GAUDI2:
+ default:
+ dev_err(hdev->dev, "Unrecognized ASIC type %d\n", hdev->asic_type);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void hbl_en_handle_eqe(struct hbl_aux_dev *aux_dev, u32 port, struct hbl_cn_eqe *eqe)
+{
+ struct hbl_en_device *hdev = aux_dev->priv;
+
+ hdev->asic_funcs.handle_eqe(aux_dev, port, eqe);
+}
+
+static void hbl_en_set_aux_ops(struct hbl_en_device *hdev, bool enable)
+{
+ struct hbl_en_aux_ops *aux_ops = hdev->aux_dev->aux_ops;
+
+ if (enable) {
+ aux_ops->ports_reopen = hbl_en_ports_reopen;
+ aux_ops->ports_stop_prepare = hbl_en_ports_stop_prepare;
+ aux_ops->ports_stop = hbl_en_ports_stop;
+ aux_ops->set_port_status = hbl_en_set_port_status;
+ aux_ops->is_port_open = hbl_en_is_port_open;
+ aux_ops->get_src_ip = hbl_en_get_src_ip;
+ aux_ops->reset_stats = hbl_en_reset_stats;
+ aux_ops->get_mtu = hbl_en_get_mtu;
+ aux_ops->get_pflags = hbl_en_get_pflags;
+ aux_ops->set_dev_lpbk = hbl_en_set_dev_lpbk;
+ aux_ops->handle_eqe = hbl_en_handle_eqe;
+ } else {
+ aux_ops->ports_reopen = NULL;
+ aux_ops->ports_stop_prepare = NULL;
+ aux_ops->ports_stop = NULL;
+ aux_ops->set_port_status = NULL;
+ aux_ops->is_port_open = NULL;
+ aux_ops->get_src_ip = NULL;
+ aux_ops->reset_stats = NULL;
+ aux_ops->get_mtu = NULL;
+ aux_ops->get_pflags = NULL;
+ aux_ops->set_dev_lpbk = NULL;
+ aux_ops->handle_eqe = NULL;
+ }
+}
+
+int hbl_en_dev_init(struct hbl_en_device *hdev)
+{
+ struct hbl_en_asic_funcs *asic_funcs = &hdev->asic_funcs;
+ struct hbl_en_port *port;
+ int rc, i, port_cnt = 0;
+
+ /* must be called before the call to dev_init() */
+ rc = hbl_en_set_asic_funcs(hdev);
+ if (rc) {
+ dev_err(hdev->dev, "failed to set aux ops\n");
+ return rc;
+ }
+
+ rc = asic_funcs->dev_init(hdev);
+ if (rc) {
+ dev_err(hdev->dev, "device init failed\n");
+ return rc;
+ }
+
+ /* init the function pointers here before calling hbl_en_port_register which sets up
+ * net_device_ops, and its ops might start getting called.
+ * If any failure is encountered, these will be made NULL and the core driver won't call
+ * them.
+ */
+ hbl_en_set_aux_ops(hdev, true);
+
+ /* Port register depends on the above initialization so it must be called here and not
+ * before that.
+ */
+ for (i = 0; i < hdev->max_num_of_ports; i++, port_cnt++) {
+ if (!(hdev->ports_mask & BIT(i)))
+ continue;
+
+ port = &hdev->ports[i];
+
+ rc = hbl_en_port_init(port);
+ if (rc) {
+ dev_err(hdev->dev, "port init failed\n");
+ goto unregister_ports;
+ }
+
+ rc = hbl_en_port_register(port);
+ if (rc) {
+ dev_err(hdev->dev, "port register failed\n");
+
+ hbl_en_port_fini(port);
+ goto unregister_ports;
+ }
+ }
+
+ hdev->is_initialized = true;
+
+ return 0;
+
+unregister_ports:
+ for (i = 0; i < port_cnt; i++) {
+ if (!(hdev->ports_mask & BIT(i)))
+ continue;
+
+ port = &hdev->ports[i];
+
+ hbl_en_port_unregister(port);
+ hbl_en_port_fini(port);
+ }
+
+ hbl_en_set_aux_ops(hdev, false);
+
+ asic_funcs->dev_fini(hdev);
+
+ return rc;
+}
+
+void hbl_en_dev_fini(struct hbl_en_device *hdev)
+{
+ struct hbl_en_asic_funcs *asic_funcs = &hdev->asic_funcs;
+ struct hbl_en_port *port;
+ int i;
+
+ hdev->in_teardown = true;
+
+ if (!hdev->is_initialized)
+ return;
+
+ hdev->is_initialized = false;
+
+ for (i = 0; i < hdev->max_num_of_ports; i++) {
+ if (!(hdev->ports_mask & BIT(i)))
+ continue;
+
+ port = &hdev->ports[i];
+
+ /* It could be this cleanup flow is called after a failed init flow.
+ * Hence we need to check that we indeed have a netdev to unregister.
+ */
+ if (!port->ndev)
+ continue;
+
+ hbl_en_port_unregister(port);
+ hbl_en_port_fini(port);
+ }
+
+ hbl_en_set_aux_ops(hdev, false);
+
+ asic_funcs->dev_fini(hdev);
+}
+
+dma_addr_t hbl_en_dma_map(struct hbl_en_device *hdev, void *addr, int len)
+{
+ dma_addr_t dma_addr;
+
+ if (hdev->dma_map_support)
+ dma_addr = dma_map_single(&hdev->pdev->dev, addr, len, DMA_TO_DEVICE);
+ else
+ dma_addr = virt_to_phys(addr);
+
+ return dma_addr;
+}
+
+void hbl_en_dma_unmap(struct hbl_en_device *hdev, dma_addr_t dma_addr, int len)
+{
+ if (hdev->dma_map_support)
+ dma_unmap_single(&hdev->pdev->dev, dma_addr, len, DMA_TO_DEVICE);
+}
diff --git a/drivers/net/ethernet/intel/hbl_en/common/hbl_en.h b/drivers/net/ethernet/intel/hbl_en/common/hbl_en.h
new file mode 100644
index 000000000000..15504c1f3cfb
--- /dev/null
+++ b/drivers/net/ethernet/intel/hbl_en/common/hbl_en.h
@@ -0,0 +1,206 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2020-2024 HabanaLabs, Ltd.
+ * Copyright (C) 2023-2024, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+#ifndef HABANALABS_EN_H_
+#define HABANALABS_EN_H_
+
+#include <linux/net/intel/cn.h>
+
+#include <linux/netdevice.h>
+#include <linux/pci.h>
+
+#define HBL_EN_NAME "habanalabs_en"
+
+#define HBL_EN_PORT(aux_dev, idx) (&(((struct hbl_en_device *)(aux_dev)->priv)->ports[(idx)]))
+
+#define hbl_netdev_priv(ndev) \
+({ \
+ typecheck(struct net_device *, ndev); \
+ *(struct hbl_en_port **)netdev_priv(ndev); \
+})
+
+/**
+ * enum hbl_en_eth_pkt_status - status of Rx Ethernet packet.
+ * ETH_PKT_OK: packet was received successfully.
+ * ETH_PKT_DROP: packet should be dropped.
+ * ETH_PKT_NONE: no available packet.
+ */
+enum hbl_en_eth_pkt_status {
+ ETH_PKT_OK,
+ ETH_PKT_DROP,
+ ETH_PKT_NONE
+};
+
+/**
+ * struct hbl_en_net_stats - stats of Ethernet interface.
+ * rx_packets: number of packets received.
+ * tx_packets: number of packets sent.
+ * rx_bytes: total bytes of data received.
+ * tx_bytes: total bytes of data sent.
+ * tx_errors: number of errors in the TX.
+ * rx_dropped: number of packets dropped by the RX.
+ * tx_dropped: number of packets dropped by the TX.
+ */
+struct hbl_en_net_stats {
+ u64 rx_packets;
+ u64 tx_packets;
+ u64 rx_bytes;
+ u64 tx_bytes;
+ u64 tx_errors;
+ atomic64_t rx_dropped;
+ atomic64_t tx_dropped;
+};
+
+/**
+ * struct hbl_en_port - manage port common structure.
+ * @hdev: habanalabs Ethernet device structure.
+ * @ndev: network device.
+ * @rx_wq: WQ for Rx poll when we cannot schedule NAPI poll.
+ * @mac_addr: HW MAC addresses.
+ * @asic_specific: ASIC specific port structure.
+ * @napi: New API structure.
+ * @rx_poll_work: Rx work for polling mode.
+ * @net_stats: statistics of the ethernet interface.
+ * @in_reset: true if the NIC was marked as in reset, false otherwise. Used to avoid an additional
+ * stopping of the NIC if a hard reset was re-initiated.
+ * @pflags: ethtool private flags bit mask.
+ * @idx: index of this specific port.
+ * @rx_max_coalesced_frames: Maximum number of packets to receive before an RX interrupt.
+ * @tx_max_coalesced_frames: Maximum number of packets to be sent before a TX interrupt.
+ * @rx_coalesce_usecs: How many usecs to delay an RX interrupt after a packet arrives.
+ * @is_initialized: true if the port H/W is initialized, false otherwise.
+ * @pfc_enable: true if this port supports Priority Flow Control, false otherwise.
+ * @auto_neg_enable: is autoneg enabled.
+ * @auto_neg_resolved: was autoneg phase finished successfully.
+ */
+struct hbl_en_port {
+ struct hbl_en_device *hdev;
+ struct net_device *ndev;
+ struct workqueue_struct *rx_wq;
+ char *mac_addr;
+ void *asic_specific;
+ struct napi_struct napi;
+ struct delayed_work rx_poll_work;
+ struct hbl_en_net_stats net_stats;
+ atomic_t in_reset;
+ u32 pflags;
+ u32 idx;
+ u32 rx_max_coalesced_frames;
+ u32 tx_max_coalesced_frames;
+ u16 rx_coalesce_usecs;
+ u8 is_initialized;
+ u8 pfc_enable;
+ u8 auto_neg_enable;
+ u8 auto_neg_resolved;
+};
+
+/**
+ * struct hbl_en_asic_funcs - ASIC specific Ethernet functions.
+ * @dev_init: device init.
+ * @dev_fini: device cleanup.
+ * @reenable_rx_irq: re-enable Rx interrupts.
+ * @eth_port_open: initialize and open the Ethernet port.
+ * @eth_port_close: close the Ethernet port.
+ * @write_pkt_to_hw: write skb to HW.
+ * @read_pkt_from_hw: read pkt from HW.
+ * @get_pfc_cnts: get PFC counters.
+ * @set_coalesce: set Tx/Rx coalesce config in HW.
+ * @get_rx_ring size: get max number of elements the Rx ring can contain.
+ * @handle_eqe: Handle a received event.
+ */
+struct hbl_en_asic_funcs {
+ int (*dev_init)(struct hbl_en_device *hdev);
+ void (*dev_fini)(struct hbl_en_device *hdev);
+ void (*reenable_rx_irq)(struct hbl_en_port *port);
+ int (*eth_port_open)(struct hbl_en_port *port);
+ void (*eth_port_close)(struct hbl_en_port *port);
+ netdev_tx_t (*write_pkt_to_hw)(struct hbl_en_port *port, struct sk_buff *skb);
+ int (*read_pkt_from_hw)(struct hbl_en_port *port, void **pkt_addr, u32 *pkt_size);
+ void (*get_pfc_cnts)(struct hbl_en_port *port, void *ptr);
+ int (*set_coalesce)(struct hbl_en_port *port);
+ int (*get_rx_ring_size)(struct hbl_en_port *port);
+ void (*handle_eqe)(struct hbl_aux_dev *aux_dev, u32 port_idx, struct hbl_cn_eqe *eqe);
+};
+
+/**
+ * struct hbl_en_device - habanalabs Ethernet device structure.
+ * @pdev: pointer to PCI device.
+ * @dev: related kernel basic device structure.
+ * @ports: array of all ports manage common structures.
+ * @aux_dev: pointer to auxiliary device.
+ * @asic_specific: ASIC specific device structure.
+ * @fw_ver: FW version.
+ * @qsfp_eeprom: QSFPD EEPROM info.
+ * @mac_addr: array of all MAC addresses.
+ * @asic_funcs: ASIC specific Ethernet functions.
+ * @asic_type: ASIC specific type.
+ * @ports_mask: mask of available ports.
+ * @auto_neg_mask: mask of port with Autonegotiation enabled.
+ * @port_reset_timeout: max time in seconds for a port reset flow to finish.
+ * @pending_reset_long_timeout: long timeout for pending hard reset to finish in seconds.
+ * @max_frm_len: maximum allowed frame length.
+ * @raw_elem_size: size of element in raw buffers.
+ * @max_raw_mtu: maximum MTU size for raw packets.
+ * @min_raw_mtu: minimum MTU size for raw packets.
+ * @pad_size: the pad size in bytes for the skb to transmit.
+ * @core_dev_id: core device ID.
+ * @max_num_of_ports: max number of available ports;
+ * @in_reset: is the entire NIC currently under reset.
+ * @poll_enable: Enable Rx polling rather than IRQ + NAPI.
+ * @in_teardown: true if the NIC is in teardown (during device remove).
+ * @is_initialized: was the device initialized successfully.
+ * @has_eq: true if event queue is supported.
+ * @dma_map_support: HW supports DMA mapping.
+ */
+struct hbl_en_device {
+ struct pci_dev *pdev;
+ struct device *dev;
+ struct hbl_en_port *ports;
+ struct hbl_aux_dev *aux_dev;
+ void *asic_specific;
+ char *fw_ver;
+ char *qsfp_eeprom;
+ char *mac_addr;
+ struct hbl_en_asic_funcs asic_funcs;
+ enum hbl_cn_asic_type asic_type;
+ u64 ports_mask;
+ u64 auto_neg_mask;
+ u32 port_reset_timeout;
+ u32 pending_reset_long_timeout;
+ u32 max_frm_len;
+ u32 raw_elem_size;
+ u16 max_raw_mtu;
+ u16 min_raw_mtu;
+ u16 pad_size;
+ u16 core_dev_id;
+ u8 max_num_of_ports;
+ u8 in_reset;
+ u8 poll_enable;
+ u8 in_teardown;
+ u8 is_initialized;
+ u8 has_eq;
+ u8 dma_map_support;
+};
+
+int hbl_en_dev_init(struct hbl_en_device *hdev);
+void hbl_en_dev_fini(struct hbl_en_device *hdev);
+
+const struct ethtool_ops *hbl_en_ethtool_get_ops(struct net_device *ndev);
+void hbl_en_ethtool_init_coalesce(struct hbl_en_port *port);
+
+extern const struct dcbnl_rtnl_ops hbl_en_dcbnl_ops;
+
+bool hbl_en_rx_poll_start(struct hbl_en_port *port);
+void hbl_en_rx_poll_stop(struct hbl_en_port *port);
+void hbl_en_rx_poll_trigger_init(struct hbl_en_port *port);
+int hbl_en_port_reset(struct hbl_en_port *port);
+int hbl_en_port_reset_locked(struct hbl_aux_dev *aux_dev, u32 port_idx);
+int hbl_en_handle_rx(struct hbl_en_port *port, int budget);
+dma_addr_t hbl_en_dma_map(struct hbl_en_device *hdev, void *addr, int len);
+void hbl_en_dma_unmap(struct hbl_en_device *hdev, dma_addr_t dma_addr, int len);
+
+#endif /* HABANALABS_EN_H_ */
diff --git a/drivers/net/ethernet/intel/hbl_en/common/hbl_en_dcbnl.c b/drivers/net/ethernet/intel/hbl_en/common/hbl_en_dcbnl.c
new file mode 100644
index 000000000000..5d718579a2b6
--- /dev/null
+++ b/drivers/net/ethernet/intel/hbl_en/common/hbl_en_dcbnl.c
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2020-2024 HabanaLabs, Ltd.
+ * Copyright (C) 2023-2024, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+#include "hbl_en.h"
+
+#define PFC_PRIO_MASK_ALL GENMASK(HBL_EN_PFC_PRIO_NUM - 1, 0)
+#define PFC_PRIO_MASK_NONE 0
+
+#ifdef CONFIG_DCB
+static int hbl_en_dcbnl_ieee_getpfc(struct net_device *netdev, struct ieee_pfc *pfc)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(netdev);
+ struct hbl_en_device *hdev;
+ u32 port_idx;
+
+ hdev = port->hdev;
+ port_idx = port->idx;
+
+ if (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ dev_dbg_ratelimited(hdev->dev, "port %d is in reset, can't get PFC", port_idx);
+ return -EBUSY;
+ }
+
+ pfc->pfc_en = port->pfc_enable ? PFC_PRIO_MASK_ALL : PFC_PRIO_MASK_NONE;
+ pfc->pfc_cap = HBL_EN_PFC_PRIO_NUM;
+
+ hdev->asic_funcs.get_pfc_cnts(port, pfc);
+
+ atomic_set(&port->in_reset, 0);
+
+ return 0;
+}
+
+static int hbl_en_dcbnl_ieee_setpfc(struct net_device *netdev, struct ieee_pfc *pfc)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(netdev);
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ struct hbl_en_device *hdev;
+ u8 curr_pfc_en;
+ u32 port_idx;
+ int rc = 0;
+
+ hdev = port->hdev;
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+ port_idx = port->idx;
+
+ if (pfc->pfc_en & ~PFC_PRIO_MASK_ALL) {
+ dev_dbg_ratelimited(hdev->dev, "PFC supports %d priorities only, port %d\n",
+ HBL_EN_PFC_PRIO_NUM, port_idx);
+ return -EINVAL;
+ }
+
+ if (pfc->pfc_en != PFC_PRIO_MASK_NONE && pfc->pfc_en != PFC_PRIO_MASK_ALL) {
+ dev_dbg_ratelimited(hdev->dev,
+ "PFC should be enabled/disabled on all priorities, port %d\n",
+ port_idx);
+ return -EINVAL;
+ }
+
+ if (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ dev_dbg_ratelimited(hdev->dev, "port %d is in reset, can't set PFC", port_idx);
+ return -EBUSY;
+ }
+
+ curr_pfc_en = port->pfc_enable ? PFC_PRIO_MASK_ALL : PFC_PRIO_MASK_NONE;
+
+ if (pfc->pfc_en == curr_pfc_en)
+ goto out;
+
+ port->pfc_enable = !port->pfc_enable;
+
+ rc = aux_ops->set_pfc(aux_dev, port_idx, port->pfc_enable);
+
+out:
+ atomic_set(&port->in_reset, 0);
+
+ return rc;
+}
+
+static u8 hbl_en_dcbnl_getdcbx(struct net_device *netdev)
+{
+ return DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE;
+}
+
+static u8 hbl_en_dcbnl_setdcbx(struct net_device *netdev, u8 mode)
+{
+ return !(mode == (DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE));
+}
+
+const struct dcbnl_rtnl_ops hbl_en_dcbnl_ops = {
+ .ieee_getpfc = hbl_en_dcbnl_ieee_getpfc,
+ .ieee_setpfc = hbl_en_dcbnl_ieee_setpfc,
+ .getdcbx = hbl_en_dcbnl_getdcbx,
+ .setdcbx = hbl_en_dcbnl_setdcbx
+};
+#endif
diff --git a/drivers/net/ethernet/intel/hbl_en/common/hbl_en_drv.c b/drivers/net/ethernet/intel/hbl_en/common/hbl_en_drv.c
new file mode 100644
index 000000000000..23a87d36ded5
--- /dev/null
+++ b/drivers/net/ethernet/intel/hbl_en/common/hbl_en_drv.c
@@ -0,0 +1,211 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2020-2024 HabanaLabs, Ltd.
+ * Copyright (C) 2023-2024, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+#define pr_fmt(fmt) "habanalabs_en: " fmt
+
+#include "hbl_en.h"
+
+#include <linux/module.h>
+#include <linux/auxiliary_bus.h>
+
+#define HBL_DRIVER_AUTHOR "HabanaLabs Kernel Driver Team"
+
+#define HBL_DRIVER_DESC "HabanaLabs AI accelerators Ethernet driver"
+
+MODULE_AUTHOR(HBL_DRIVER_AUTHOR);
+MODULE_DESCRIPTION(HBL_DRIVER_DESC);
+MODULE_LICENSE("GPL");
+
+static bool poll_enable;
+
+module_param(poll_enable, bool, 0444);
+MODULE_PARM_DESC(poll_enable,
+ "Enable Rx polling rather than IRQ + NAPI (0 = no, 1 = yes, default: no)");
+
+static int hdev_init(struct hbl_aux_dev *aux_dev)
+{
+ struct hbl_en_aux_data *aux_data = aux_dev->aux_data;
+ struct hbl_en_port *ports, *port;
+ struct hbl_en_device *hdev;
+ int rc, i;
+
+ hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
+ if (!hdev)
+ return -ENOMEM;
+
+ ports = kcalloc(aux_data->max_num_of_ports, sizeof(*ports), GFP_KERNEL);
+ if (!ports) {
+ rc = -ENOMEM;
+ goto ports_alloc_fail;
+ }
+
+ aux_dev->priv = hdev;
+ hdev->aux_dev = aux_dev;
+ hdev->ports = ports;
+ hdev->pdev = aux_data->pdev;
+ hdev->dev = aux_data->dev;
+ hdev->ports_mask = aux_data->ports_mask;
+ hdev->auto_neg_mask = aux_data->auto_neg_mask;
+ hdev->max_num_of_ports = aux_data->max_num_of_ports;
+ hdev->core_dev_id = aux_data->id;
+ hdev->fw_ver = aux_data->fw_ver;
+ hdev->qsfp_eeprom = aux_data->qsfp_eeprom;
+ hdev->asic_type = aux_data->asic_type;
+ hdev->pending_reset_long_timeout = aux_data->pending_reset_long_timeout;
+ hdev->max_frm_len = aux_data->max_frm_len;
+ hdev->raw_elem_size = aux_data->raw_elem_size;
+ hdev->max_raw_mtu = aux_data->max_raw_mtu;
+ hdev->min_raw_mtu = aux_data->min_raw_mtu;
+ hdev->pad_size = ETH_ZLEN;
+ hdev->has_eq = aux_data->has_eq;
+ hdev->dma_map_support = true;
+ hdev->poll_enable = poll_enable;
+
+ for (i = 0; i < hdev->max_num_of_ports; i++) {
+ if (!(hdev->ports_mask & BIT(i)))
+ continue;
+
+ port = &hdev->ports[i];
+ port->hdev = hdev;
+ port->idx = i;
+ port->pfc_enable = true;
+ port->pflags = PFLAGS_PCS_LINK_CHECK | PFLAGS_PHY_AUTO_NEG_LPBK;
+ port->mac_addr = aux_data->mac_addr[i];
+ port->auto_neg_enable = !!(aux_data->auto_neg_mask & BIT(i));
+ }
+
+ return 0;
+
+ports_alloc_fail:
+ kfree(hdev);
+
+ return rc;
+}
+
+static void hdev_fini(struct hbl_aux_dev *aux_dev)
+{
+ struct hbl_en_device *hdev = aux_dev->priv;
+
+ kfree(hdev->ports);
+ kfree(hdev);
+ aux_dev->priv = NULL;
+}
+
+static const struct auxiliary_device_id hbl_en_id_table[] = {
+ { .name = "habanalabs_cn.en", },
+ {},
+};
+
+MODULE_DEVICE_TABLE(auxiliary, hbl_en_id_table);
+
+static int hbl_en_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id)
+{
+ struct hbl_aux_dev *aux_dev = container_of(adev, struct hbl_aux_dev, adev);
+ struct hbl_en_aux_ops *aux_ops = aux_dev->aux_ops;
+ struct hbl_en_device *hdev;
+ ktime_t timeout;
+ int rc;
+
+ rc = hdev_init(aux_dev);
+ if (rc) {
+ dev_err(&aux_dev->adev.dev, "Failed to init hdev\n");
+ return -EIO;
+ }
+
+ hdev = aux_dev->priv;
+
+ /* don't allow module unloading while it is attached */
+ if (!try_module_get(THIS_MODULE)) {
+ dev_err(hdev->dev, "Failed to increment %s module refcount\n", HBL_EN_NAME);
+ rc = -EIO;
+ goto module_get_err;
+ }
+
+ timeout = ktime_add_ms(ktime_get(), hdev->pending_reset_long_timeout * MSEC_PER_SEC);
+ while (1) {
+ aux_ops->hw_access_lock(aux_dev);
+
+ /* if the device is operational, proceed to actual init while holding the lock in
+ * order to prevent concurrent hard reset
+ */
+ if (aux_ops->device_operational(aux_dev))
+ break;
+
+ aux_ops->hw_access_unlock(aux_dev);
+
+ if (ktime_compare(ktime_get(), timeout) > 0) {
+ dev_err(hdev->dev, "Timeout while waiting for hard reset to finish\n");
+ rc = -EBUSY;
+ goto timeout_err;
+ }
+
+ dev_notice_once(hdev->dev, "Waiting for hard reset to finish before probing en\n");
+
+ msleep_interruptible(MSEC_PER_SEC);
+ }
+
+ rc = hbl_en_dev_init(hdev);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to init en device\n");
+ goto dev_init_err;
+ }
+
+ aux_ops->hw_access_unlock(aux_dev);
+
+ return 0;
+
+dev_init_err:
+ aux_ops->hw_access_unlock(aux_dev);
+timeout_err:
+ module_put(THIS_MODULE);
+module_get_err:
+ hdev_fini(aux_dev);
+
+ return rc;
+}
+
+/* This function can be called only from the CN driver when deleting the aux bus, because we
+ * incremented the module refcount on probing. Hence no need to protect here from hard reset.
+ */
+static void hbl_en_remove(struct auxiliary_device *adev)
+{
+ struct hbl_aux_dev *aux_dev = container_of(adev, struct hbl_aux_dev, adev);
+ struct hbl_en_device *hdev = aux_dev->priv;
+
+ if (!hdev)
+ return;
+
+ hbl_en_dev_fini(hdev);
+
+ /* allow module unloading as now it is detached */
+ module_put(THIS_MODULE);
+
+ hdev_fini(aux_dev);
+}
+
+static struct auxiliary_driver hbl_en_driver = {
+ .name = "eth",
+ .probe = hbl_en_probe,
+ .remove = hbl_en_remove,
+ .id_table = hbl_en_id_table,
+};
+
+static int __init hbl_en_init(void)
+{
+ pr_info("loading driver\n");
+
+ return auxiliary_driver_register(&hbl_en_driver);
+}
+
+static void __exit hbl_en_exit(void)
+{
+ auxiliary_driver_unregister(&hbl_en_driver);
+
+ pr_info("driver removed\n");
+}
+
+module_init(hbl_en_init);
+module_exit(hbl_en_exit);
diff --git a/drivers/net/ethernet/intel/hbl_en/common/hbl_en_ethtool.c b/drivers/net/ethernet/intel/hbl_en/common/hbl_en_ethtool.c
new file mode 100644
index 000000000000..1d14d283409b
--- /dev/null
+++ b/drivers/net/ethernet/intel/hbl_en/common/hbl_en_ethtool.c
@@ -0,0 +1,452 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2020-2024 HabanaLabs, Ltd.
+ * Copyright (C) 2023-2024, Intel Corporation.
+ * All Rights Reserved.
+ */
+
+#include "hbl_en.h"
+#include <linux/ethtool.h>
+
+#define RX_COALESCED_FRAMES_MIN 1
+#define TX_COALESCED_FRAMES_MIN 1
+#define TX_COALESCED_FRAMES_MAX 10
+
+static const char pflags_str[][ETH_GSTRING_LEN] = {
+ "pcs-link-check",
+ "phy-auto-neg-lpbk",
+};
+
+#define NIC_STAT(m) {#m, offsetof(struct hbl_en_port, net_stats.m)}
+
+static struct hbl_cn_stat netdev_eth_stats[] = {
+ NIC_STAT(rx_packets),
+ NIC_STAT(tx_packets),
+ NIC_STAT(rx_bytes),
+ NIC_STAT(tx_bytes),
+ NIC_STAT(tx_errors),
+ NIC_STAT(rx_dropped),
+ NIC_STAT(tx_dropped)
+};
+
+static size_t pflags_str_len = ARRAY_SIZE(pflags_str);
+static size_t netdev_eth_stats_len = ARRAY_SIZE(netdev_eth_stats);
+
+static void hbl_en_ethtool_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *drvinfo)
+{
+ struct hbl_en_device *hdev;
+ struct hbl_en_port *port;
+
+ port = hbl_netdev_priv(ndev);
+ hdev = port->hdev;
+
+ strscpy(drvinfo->driver, HBL_EN_NAME, sizeof(drvinfo->driver));
+ strscpy(drvinfo->fw_version, hdev->fw_ver, sizeof(drvinfo->fw_version));
+ strscpy(drvinfo->bus_info, pci_name(hdev->pdev), sizeof(drvinfo->bus_info));
+}
+
+static int hbl_en_ethtool_get_module_info(struct net_device *ndev, struct ethtool_modinfo *modinfo)
+{
+ modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN;
+ modinfo->type = ETH_MODULE_SFF_8636;
+
+ return 0;
+}
+
+static int hbl_en_ethtool_get_module_eeprom(struct net_device *ndev, struct ethtool_eeprom *ee,
+ u8 *data)
+{
+ struct hbl_en_device *hdev;
+ struct hbl_en_port *port;
+ u32 first, last, len;
+ u8 *qsfp_eeprom;
+
+ port = hbl_netdev_priv(ndev);
+ hdev = port->hdev;
+ qsfp_eeprom = hdev->qsfp_eeprom;
+
+ if (ee->len == 0)
+ return -EINVAL;
+
+ first = ee->offset;
+ last = ee->offset + ee->len;
+
+ if (first < ETH_MODULE_SFF_8636_LEN) {
+ len = min_t(unsigned int, last, ETH_MODULE_SFF_8079_LEN);
+ len -= first;
+
+ memcpy(data, qsfp_eeprom + first, len);
+ }
+
+ return 0;
+}
+
+static u32 hbl_en_ethtool_get_priv_flags(struct net_device *ndev)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(ndev);
+
+ return port->pflags;
+}
+
+static int hbl_en_ethtool_set_priv_flags(struct net_device *ndev, u32 priv_flags)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(ndev);
+
+ port->pflags = priv_flags;
+
+ return 0;
+}
+
+static int hbl_en_ethtool_get_link_ksettings(struct net_device *ndev,
+ struct ethtool_link_ksettings *cmd)
+{
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ struct hbl_en_device *hdev;
+ struct hbl_en_port *port;
+ u32 port_idx, speed;
+
+ port = hbl_netdev_priv(ndev);
+ hdev = port->hdev;
+ port_idx = port->idx;
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+ speed = aux_ops->get_speed(aux_dev, port_idx);
+
+ cmd->base.speed = speed;
+ cmd->base.duplex = DUPLEX_FULL;
+
+ ethtool_link_ksettings_zero_link_mode(cmd, supported);
+ ethtool_link_ksettings_zero_link_mode(cmd, advertising);
+
+ switch (speed) {
+ case SPEED_100000:
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 100000baseCR4_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 100000baseSR4_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 100000baseKR4_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 100000baseLR4_ER4_Full);
+
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 100000baseCR4_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 100000baseSR4_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 100000baseKR4_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 100000baseLR4_ER4_Full);
+
+ cmd->base.port = PORT_FIBRE;
+
+ ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE);
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE);
+
+ ethtool_link_ksettings_add_link_mode(cmd, supported, Backplane);
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, Backplane);
+ break;
+ case SPEED_50000:
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 50000baseSR2_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 50000baseCR2_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 50000baseKR2_Full);
+
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 50000baseSR2_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 50000baseCR2_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 50000baseKR2_Full);
+ break;
+ case SPEED_25000:
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 25000baseCR_Full);
+
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 25000baseCR_Full);
+ break;
+ case SPEED_200000:
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 200000baseCR4_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 200000baseKR4_Full);
+
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 200000baseCR4_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 200000baseKR4_Full);
+ break;
+ case SPEED_400000:
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 400000baseCR4_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, supported, 400000baseKR4_Full);
+
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 400000baseCR4_Full);
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, 400000baseKR4_Full);
+ break;
+ default:
+ netdev_err(port->ndev, "unknown speed %d\n", speed);
+ return -EFAULT;
+ }
+
+ ethtool_link_ksettings_add_link_mode(cmd, supported, Autoneg);
+
+ if (port->auto_neg_enable) {
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, Autoneg);
+ cmd->base.autoneg = AUTONEG_ENABLE;
+ if (port->auto_neg_resolved)
+ ethtool_link_ksettings_add_link_mode(cmd, lp_advertising, Autoneg);
+ } else {
+ cmd->base.autoneg = AUTONEG_DISABLE;
+ }
+
+ ethtool_link_ksettings_add_link_mode(cmd, supported, Pause);
+
+ if (port->pfc_enable)
+ ethtool_link_ksettings_add_link_mode(cmd, advertising, Pause);
+
+ return 0;
+}
+
+/* only autoneg is mutable */
+static bool check_immutable_ksettings(const struct ethtool_link_ksettings *old_cmd,
+ const struct ethtool_link_ksettings *new_cmd)
+{
+ return (old_cmd->base.speed == new_cmd->base.speed) &&
+ (old_cmd->base.duplex == new_cmd->base.duplex) &&
+ (old_cmd->base.port == new_cmd->base.port) &&
+ (old_cmd->base.phy_address == new_cmd->base.phy_address) &&
+ (old_cmd->base.eth_tp_mdix_ctrl == new_cmd->base.eth_tp_mdix_ctrl) &&
+ bitmap_equal(old_cmd->link_modes.advertising, new_cmd->link_modes.advertising,
+ __ETHTOOL_LINK_MODE_MASK_NBITS);
+}
+
+static int
+hbl_en_ethtool_set_link_ksettings(struct net_device *ndev, const struct ethtool_link_ksettings *cmd)
+{
+ struct ethtool_link_ksettings curr_cmd;
+ struct hbl_en_device *hdev;
+ struct hbl_en_port *port;
+ bool auto_neg;
+ u32 port_idx;
+ int rc;
+
+ port = hbl_netdev_priv(ndev);
+ hdev = port->hdev;
+ port_idx = port->idx;
+
+ memset(&curr_cmd, 0, sizeof(struct ethtool_link_ksettings));
+
+ rc = hbl_en_ethtool_get_link_ksettings(ndev, &curr_cmd);
+ if (rc)
+ return rc;
+
+ if (!check_immutable_ksettings(&curr_cmd, cmd))
+ return -EOPNOTSUPP;
+
+ auto_neg = cmd->base.autoneg == AUTONEG_ENABLE;
+
+ if (port->auto_neg_enable == auto_neg)
+ return 0;
+
+ if (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ netdev_err(port->ndev, "port is in reset, can't update settings\n");
+ return -EBUSY;
+ }
+
+ if (auto_neg && !(hdev->auto_neg_mask & BIT(port_idx))) {
+ netdev_err(port->ndev, "port autoneg is disabled by BMC\n");
+ rc = -EFAULT;
+ goto out;
+ }
+
+ port->auto_neg_enable = auto_neg;
+
+ if (netif_running(port->ndev)) {
+ rc = hbl_en_port_reset(port);
+ if (rc)
+ netdev_err(port->ndev, "Failed to reset port for settings update, rc %d\n",
+ rc);
+ }
+
+out:
+ atomic_set(&port->in_reset, 0);
+
+ return rc;
+}
+
+static int hbl_en_ethtool_get_sset_count(struct net_device *ndev, int sset)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(ndev);
+ struct hbl_en_device *hdev = port->hdev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ switch (sset) {
+ case ETH_SS_STATS:
+ return netdev_eth_stats_len + aux_ops->get_cnts_num(aux_dev, port_idx);
+ case ETH_SS_PRIV_FLAGS:
+ return pflags_str_len;
+ default:
+ return -EOPNOTSUPP;
+ }
+}
+
+static void hbl_en_ethtool_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(ndev);
+ struct hbl_en_device *hdev = port->hdev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+ int i;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ switch (stringset) {
+ case ETH_SS_STATS:
+ for (i = 0; i < netdev_eth_stats_len; i++)
+ ethtool_puts(&data, netdev_eth_stats[i].str);
+
+ aux_ops->get_cnts_names(aux_dev, port_idx, data);
+ break;
+ case ETH_SS_PRIV_FLAGS:
+ for (i = 0; i < pflags_str_len; i++)
+ ethtool_puts(&data, pflags_str[i]);
+ break;
+ }
+}
+
+static void hbl_en_ethtool_get_ethtool_stats(struct net_device *ndev,
+ __always_unused struct ethtool_stats *stats, u64 *data)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(ndev);
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ struct hbl_en_device *hdev;
+ u32 port_idx;
+ char *p;
+ int i;
+
+ hdev = port->hdev;
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+ port_idx = port->idx;
+
+ if (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ dev_info_ratelimited(hdev->dev, "port %d is in reset, can't get ethtool stats",
+ port_idx);
+ return;
+ }
+
+ /* Even though the Ethernet Rx/Tx flow might update the stats in parallel, there is not an
+ * absolute need for synchronisation. This is because, missing few counts of these stats is
+ * much better than adding a lock to synchronize and increase the overhead of the Rx/Tx
+ * flows. In worst case scenario, reader will get stale stats. He will receive updated
+ * stats in next read.
+ */
+ for (i = 0; i < netdev_eth_stats_len; i++) {
+ p = (char *)port + netdev_eth_stats[i].lo_offset;
+ data[i] = *(u32 *)p;
+ }
+
+ data += i;
+
+ aux_ops->get_cnts_values(aux_dev, port_idx, data);
+
+ atomic_set(&port->in_reset, 0);
+}
+
+static int hbl_en_ethtool_get_coalesce(struct net_device *ndev,
+ struct ethtool_coalesce *coal,
+ struct kernel_ethtool_coalesce *kernel_coal,
+ struct netlink_ext_ack *extack)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(ndev);
+ struct hbl_en_device *hdev = port->hdev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ aux_ops->ctrl_lock(aux_dev, port_idx);
+
+ coal->tx_max_coalesced_frames = port->tx_max_coalesced_frames;
+ coal->rx_coalesce_usecs = port->rx_coalesce_usecs;
+ coal->rx_max_coalesced_frames = port->rx_max_coalesced_frames;
+
+ aux_ops->ctrl_unlock(aux_dev, port_idx);
+
+ return 0;
+}
+
+static int hbl_en_ethtool_set_coalesce(struct net_device *ndev,
+ struct ethtool_coalesce *coal,
+ struct kernel_ethtool_coalesce *kernel_coal,
+ struct netlink_ext_ack *extack)
+{
+ struct hbl_en_port *port = hbl_netdev_priv(ndev);
+ struct hbl_en_device *hdev = port->hdev;
+ struct hbl_en_aux_ops *aux_ops;
+ struct hbl_aux_dev *aux_dev;
+ u32 port_idx = port->idx;
+ int rc, rx_ring_size;
+
+ aux_dev = hdev->aux_dev;
+ aux_ops = aux_dev->aux_ops;
+
+ if (atomic_cmpxchg(&port->in_reset, 0, 1)) {
+ netdev_err(port->ndev, "port is in reset, can't update settings\n");
+ return -EBUSY;
+ }
+
+ if (coal->tx_max_coalesced_frames < TX_COALESCED_FRAMES_MIN ||
+ coal->tx_max_coalesced_frames > TX_COALESCED_FRAMES_MAX) {
+ netdev_err(ndev, "tx max_coalesced_frames should be between %d and %d\n",
+ TX_COALESCED_FRAMES_MIN, TX_COALESCED_FRAMES_MAX);
+ rc = -EINVAL;
+ goto atomic_out;
+ }
+
+ rx_ring_size = hdev->asic_funcs.get_rx_ring_size(port);
+ if (coal->rx_max_coalesced_frames < RX_COALESCED_FRAMES_MIN ||
+ coal->rx_max_coalesced_frames >= rx_ring_size) {
+ netdev_err(ndev, "rx max_coalesced_frames should be between %d and %d\n",
+ RX_COALESCED_FRAMES_MIN, rx_ring_size);
+ rc = -EINVAL;
+ goto atomic_out;
+ }
+
+ aux_ops->ctrl_lock(aux_dev, port_idx);
+
+ port->tx_max_coalesced_frames = coal->tx_max_coalesced_frames;
+ port->rx_coalesce_usecs = coal->rx_coalesce_usecs;
+ port->rx_max_coalesced_frames = coal->rx_max_coalesced_frames;
+
+ rc = hdev->asic_funcs.set_coalesce(port);
+
+ aux_ops->ctrl_unlock(aux_dev, port_idx);
+
+atomic_out:
+ atomic_set(&port->in_reset, 0);
+ return rc;
+}
+
+void hbl_en_ethtool_init_coalesce(struct hbl_en_port *port)
+{
+ port->rx_coalesce_usecs = CQ_ARM_TIMEOUT_USEC;
+ port->rx_max_coalesced_frames = 1;
+ port->tx_max_coalesced_frames = 1;
+}
+
+static const struct ethtool_ops hbl_en_ethtool_ops_coalesce = {
+ .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS | ETHTOOL_COALESCE_RX_MAX_FRAMES |
+ ETHTOOL_COALESCE_TX_MAX_FRAMES,
+ .get_drvinfo = hbl_en_ethtool_get_drvinfo,
+ .get_link = ethtool_op_get_link,
+ .get_module_info = hbl_en_ethtool_get_module_info,
+ .get_module_eeprom = hbl_en_ethtool_get_module_eeprom,
+ .get_priv_flags = hbl_en_ethtool_get_priv_flags,
+ .set_priv_flags = hbl_en_ethtool_set_priv_flags,
+ .get_link_ksettings = hbl_en_ethtool_get_link_ksettings,
+ .set_link_ksettings = hbl_en_ethtool_set_link_ksettings,
+ .get_sset_count = hbl_en_ethtool_get_sset_count,
+ .get_strings = hbl_en_ethtool_get_strings,
+ .get_ethtool_stats = hbl_en_ethtool_get_ethtool_stats,
+ .get_coalesce = hbl_en_ethtool_get_coalesce,
+ .set_coalesce = hbl_en_ethtool_set_coalesce,
+};
+
+const struct ethtool_ops *hbl_en_ethtool_get_ops(struct net_device *ndev)
+{
+ return &hbl_en_ethtool_ops_coalesce;
+}
--
2.34.1
Powered by blists - more mailing lists