Message-ID: <20230217225558.19837-11-shannon.nelson@amd.com>
Date:   Fri, 17 Feb 2023 14:55:54 -0800
From:   Shannon Nelson <shannon.nelson@....com>
To:     <netdev@...r.kernel.org>, <davem@...emloft.net>, <kuba@...nel.org>
CC:     <drivers@...sando.io>, <brett.creeley@....com>,
        Shannon Nelson <shannon.nelson@....com>
Subject: [PATCH v3 net-next 10/14] pds_core: add auxiliary_bus devices

An auxiliary_bus device is created for each VF, and the device
name is made up of the PF driver name, VIF name, and PCI BDF
of the VF.

Signed-off-by: Shannon Nelson <shannon.nelson@....com>
---
 drivers/net/ethernet/amd/pds_core/Makefile |   1 +
 drivers/net/ethernet/amd/pds_core/auxbus.c | 123 +++++++++++++++++++++
 drivers/net/ethernet/amd/pds_core/core.h   |   4 +
 drivers/net/ethernet/amd/pds_core/main.c   |  43 +++++++
 include/linux/pds/pds_auxbus.h             |  35 ++++++
 5 files changed, 206 insertions(+)
 create mode 100644 drivers/net/ethernet/amd/pds_core/auxbus.c
 create mode 100644 include/linux/pds/pds_auxbus.h

diff --git a/drivers/net/ethernet/amd/pds_core/Makefile b/drivers/net/ethernet/amd/pds_core/Makefile
index 6d1d6c58a1fa..0abc33ce826c 100644
--- a/drivers/net/ethernet/amd/pds_core/Makefile
+++ b/drivers/net/ethernet/amd/pds_core/Makefile
@@ -5,6 +5,7 @@ obj-$(CONFIG_PDS_CORE) := pds_core.o
 
 pds_core-y := main.o \
 	      devlink.o \
+	      auxbus.o \
 	      dev.o \
 	      adminq.o \
 	      core.o \
diff --git a/drivers/net/ethernet/amd/pds_core/auxbus.c b/drivers/net/ethernet/amd/pds_core/auxbus.c
new file mode 100644
index 000000000000..dc36fc98de52
--- /dev/null
+++ b/drivers/net/ethernet/amd/pds_core/auxbus.c
@@ -0,0 +1,123 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+
+#include "core.h"
+
+#include <linux/pds/pds_adminq.h>
+#include <linux/pds/pds_auxbus.h>
+
+static void pdsc_auxbus_dev_release(struct device *dev)
+{
+	struct pds_auxiliary_dev *padev =
+		container_of(dev, struct pds_auxiliary_dev, aux_dev.dev);
+
+	devm_kfree(dev->parent, padev);
+}
+
+static struct pds_auxiliary_dev *pdsc_auxbus_dev_register(struct pdsc *pdsc,
+							  char *name, u32 id,
+							  struct pci_dev *client_dev)
+{
+	struct auxiliary_device *aux_dev;
+	struct pds_auxiliary_dev *padev;
+	int err;
+
+	padev = devm_kzalloc(pdsc->dev, sizeof(*padev), GFP_KERNEL);
+	if (!padev)
+		return NULL;
+
+	padev->pcidev = client_dev;
+
+	aux_dev = &padev->aux_dev;
+	aux_dev->name = name;
+	aux_dev->id = id;
+	padev->id = id;
+	aux_dev->dev.parent = pdsc->dev;
+	aux_dev->dev.release = pdsc_auxbus_dev_release;
+
+	err = auxiliary_device_init(aux_dev);
+	if (err < 0) {
+		dev_warn(pdsc->dev, "auxiliary_device_init of %s id %d failed: %pe\n",
+			 name, id, ERR_PTR(err));
+		goto err_out;
+	}
+
+	err = auxiliary_device_add(aux_dev);
+	if (err) {
+		auxiliary_device_uninit(aux_dev);
+		dev_warn(pdsc->dev, "auxiliary_device_add of %s id %d failed: %pe\n",
+			 name, id, ERR_PTR(err));
+		goto err_out;
+	}
+
+	dev_dbg(pdsc->dev, "%s: name %s id %d pdsc %p\n",
+		__func__, padev->aux_dev.name, id, pdsc);
+
+	return padev;
+
+err_out:
+	devm_kfree(pdsc->dev, padev);
+	return NULL;
+}
+
+int pdsc_auxbus_dev_add_vf(struct pdsc *pdsc, int vf_id)
+{
+	struct pds_auxiliary_dev *padev;
+	enum pds_core_vif_types vt;
+	int err = 0;
+
+	if (!pdsc->vfs)
+		return -ENOTTY;
+
+	if (vf_id >= pdsc->num_vfs)
+		return -ERANGE;
+
+	if (pdsc->vfs[vf_id].padev) {
+		dev_info(pdsc->dev, "%s: vfid %d already running\n", __func__, vf_id);
+		return -ENODEV;
+	}
+
+	for (vt = 0; vt < PDS_DEV_TYPE_MAX; vt++) {
+		u16 vt_support;
+		u32 id;
+
+		/* Verify that the type supported and enabled */
+		vt_support = !!le16_to_cpu(pdsc->dev_ident.vif_types[vt]);
+		if (!(vt_support &&
+		      pdsc->viftype_status[vt].supported &&
+		      pdsc->viftype_status[vt].enabled))
+			continue;
+
+		id = PCI_DEVID(pdsc->pdev->bus->number,
+			       pci_iov_virtfn_devfn(pdsc->pdev, vf_id));
+		padev = pdsc_auxbus_dev_register(pdsc, pdsc->viftype_status[vt].name, id,
+						 pdsc->pdev);
+		pdsc->vfs[vf_id].padev = padev;
+
+		/* We only support a single type per VF, so jump out here */
+		break;
+	}
+
+	return err;
+}
+
+int pdsc_auxbus_dev_del_vf(struct pdsc *pdsc, int vf_id)
+{
+	struct pds_auxiliary_dev *padev;
+
+	dev_info(pdsc->dev, "%s: vfid %d\n", __func__, vf_id);
+
+	padev = pdsc->vfs[vf_id].padev;
+	pdsc->vfs[vf_id].padev = NULL;
+	if (padev) {
+		auxiliary_device_delete(&padev->aux_dev);
+		auxiliary_device_uninit(&padev->aux_dev);
+	}
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/amd/pds_core/core.h b/drivers/net/ethernet/amd/pds_core/core.h
index 1885976c6486..7abd2cc4efc7 100644
--- a/drivers/net/ethernet/amd/pds_core/core.h
+++ b/drivers/net/ethernet/amd/pds_core/core.h
@@ -188,6 +188,7 @@ struct pdsc {
 	dma_addr_t phy_db_pages;
 	u64 __iomem *kern_dbpage;
 
+	struct notifier_block nb;
 	struct pdsc_qcq adminqcq;
 	struct pdsc_qcq notifyqcq;
 	u64 last_eid;
@@ -302,6 +303,9 @@ int pdsc_start(struct pdsc *pdsc);
 void pdsc_stop(struct pdsc *pdsc);
 void pdsc_health_thread(struct work_struct *work);
 
+int pdsc_auxbus_dev_add_vf(struct pdsc *pdsc, int vf_id);
+int pdsc_auxbus_dev_del_vf(struct pdsc *pdsc, int vf_id);
+
 void pdsc_process_adminq(struct pdsc_qcq *qcq);
 void pdsc_work_thread(struct work_struct *work);
 irqreturn_t pdsc_adminq_isr(int irq, void *data);
diff --git a/drivers/net/ethernet/amd/pds_core/main.c b/drivers/net/ethernet/amd/pds_core/main.c
index 1376dec84756..160b38ba674c 100644
--- a/drivers/net/ethernet/amd/pds_core/main.c
+++ b/drivers/net/ethernet/amd/pds_core/main.c
@@ -170,6 +170,7 @@ static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
 	struct pdsc *pdsc = pci_get_drvdata(pdev);
 	struct device *dev = pdsc->dev;
 	int ret = 0;
+	int i;
 
 	if (num_vfs > 0) {
 		pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf), GFP_KERNEL);
@@ -187,6 +188,8 @@ static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
 	}
 
 no_vfs:
+	for (i = pdsc->num_vfs - 1; i >= 0; i--)
+		pdsc_auxbus_dev_del_vf(pdsc, i);
 	pci_disable_sriov(pdev);
 
 	kfree(pdsc->vfs);
@@ -196,6 +199,36 @@ static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
 	return ret;
 }
 
+static int pdsc_pci_bus_notifier(struct notifier_block *nb,
+				 unsigned long action, void *data)
+{
+	struct pdsc *pdsc = container_of(nb, struct pdsc, nb);
+	struct device *dev = data;
+	struct pci_dev *physfn;
+	struct pci_dev *pdev;
+
+	pdev = to_pci_dev(dev);
+	physfn = pci_physfn(pdev);
+
+	if (!(pdev->is_virtfn && physfn == pdsc->pdev))
+		return 0;
+
+	switch (action) {
+	case BUS_NOTIFY_BOUND_DRIVER:
+		dev_dbg(pdsc->dev, "BOUND_DRIVER %s vf %d\n",
+			pci_name(pdev), pci_iov_vf_id(pdev));
+		pdsc_auxbus_dev_add_vf(pdsc, pci_iov_vf_id(pdev));
+		break;
+	case BUS_NOTIFY_UNBIND_DRIVER:
+		dev_dbg(pdsc->dev, "UNBIND_DRIVER %s vf %d\n",
+			pci_name(pdev), pci_iov_vf_id(pdev));
+		pdsc_auxbus_dev_del_vf(pdsc, pci_iov_vf_id(pdev));
+		break;
+	}
+
+	return 0;
+}
+
 static DEFINE_IDA(pdsc_pf_ida);
 
 #define PDSC_WQ_NAME_LEN 24
@@ -285,6 +318,11 @@ static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	mutex_unlock(&pdsc->config_lock);
 
+	pdsc->nb.notifier_call = pdsc_pci_bus_notifier;
+	err = bus_register_notifier(&pci_bus_type, &pdsc->nb);
+	if (err)
+		dev_warn(dev, "Cannot register bus notifier: %pe\n", ERR_PTR(err));
+
 	pdsc->fw_generation = PDS_CORE_FW_STS_F_GENERATION &
 			      ioread8(&pdsc->info_regs->fw_status);
 	/* Lastly, start the health check timer */
@@ -330,8 +368,13 @@ static void pdsc_remove(struct pci_dev *pdev)
 	/* Undo the devlink registration now to be sure there
 	 * are no requests while we're stopping.
 	 */
+	bus_unregister_notifier(&pci_bus_type, &pdsc->nb);
 	pdsc_dl_unregister(pdsc);
 
+	/* Remove the VFs and their aux_bus connections before other
+	 * cleanup so that the clients can use the AdminQ to cleanly
+	 * shut themselves down.
+	 */
 	pdsc_sriov_configure(pdev, 0);
 
 	/* Now we can lock it up and tear it down */
diff --git a/include/linux/pds/pds_auxbus.h b/include/linux/pds/pds_auxbus.h
new file mode 100644
index 000000000000..737fd4dbbf5a
--- /dev/null
+++ b/include/linux/pds/pds_auxbus.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2023 Advanced Micro Devices, Inc */
+
+#ifndef _PDSC_AUXBUS_H_
+#define _PDSC_AUXBUS_H_
+
+#include <linux/auxiliary_bus.h>
+
+struct pds_auxiliary_dev;
+
+struct pds_auxiliary_drv {
+	/* .event_handler() - callback for receiving events
+	 * padev:  ptr to the client device info
+	 * event:  ptr to event data
+	 * The client can provide an event handler callback that can
+	 * receive DSC events.  The Core driver may generate its
+	 * own events which can notify the client of changes in the
+	 * DSC status, such as a RESET event generated when the Core
+	 * has lost contact with the FW - in this case the event.eid
+	 * field will be 0.
+	 */
+	void (*event_handler)(struct pds_auxiliary_dev *padev,
+			      union pds_core_notifyq_comp *event);
+};
+
+struct pds_auxiliary_dev {
+	struct auxiliary_device aux_dev;
+	struct pci_dev *pcidev;
+	u32 id;
+	u16 client_id;
+	void (*event_handler)(struct pds_auxiliary_dev *padev,
+			      union pds_core_notifyq_comp *event);
+	void *priv;
+};
+#endif /* _PDSC_AUXBUS_H_ */
-- 
2.17.1

Powered by blists - more mailing lists