Message-ID: <3528db67-86b4-47af-b002-87a75bbdc4df@onemain.com>
Date: Wed, 23 Jul 2025 11:11:46 -0700
From: Shannon Nelson <sln@...main.com>
To: Abhijit Gangurde <abhijit.gangurde@....com>, shannon.nelson@....com,
 brett.creeley@....com, davem@...emloft.net, edumazet@...gle.com,
 kuba@...nel.org, pabeni@...hat.com, corbet@....net, jgg@...pe.ca,
 leon@...nel.org, andrew+netdev@...n.ch
Cc: allen.hubbe@....com, nikhil.agarwal@....com, linux-rdma@...r.kernel.org,
 netdev@...r.kernel.org, linux-doc@...r.kernel.org,
 linux-kernel@...r.kernel.org
Subject: Re: [PATCH v4 01/14] net: ionic: Create an auxiliary device for rdma
 driver

On 7/23/25 10:31 AM, Abhijit Gangurde wrote:
> To support RDMA capable ethernet device, create an auxiliary device in
> the ionic Ethernet driver. The RDMA device is modeled as an auxiliary
> device to the Ethernet device.
>
> Reviewed-by: Shannon Nelson <shannon.nelson@....com>
> Signed-off-by: Abhijit Gangurde <abhijit.gangurde@....com>
> ---
>   drivers/net/ethernet/pensando/Kconfig         |  1 +
>   drivers/net/ethernet/pensando/ionic/Makefile  |  2 +-
>   .../net/ethernet/pensando/ionic/ionic_api.h   | 21 ++++
>   .../net/ethernet/pensando/ionic/ionic_aux.c   | 95 +++++++++++++++++++
>   .../net/ethernet/pensando/ionic/ionic_aux.h   | 10 ++
>   .../ethernet/pensando/ionic/ionic_bus_pci.c   |  5 +
>   .../net/ethernet/pensando/ionic/ionic_lif.c   |  7 ++
>   .../net/ethernet/pensando/ionic/ionic_lif.h   |  3 +
>   8 files changed, 143 insertions(+), 1 deletion(-)
>   create mode 100644 drivers/net/ethernet/pensando/ionic/ionic_api.h
>   create mode 100644 drivers/net/ethernet/pensando/ionic/ionic_aux.c
>   create mode 100644 drivers/net/ethernet/pensando/ionic/ionic_aux.h

Hi Abhijit,

It occurred to me that there should be some discussion added into the 
ionic.rst file about this new auxiliary_device support.  You might look 
in the amd/pds_core.rst and mellanox/mlx5/switchdev.rst files for 
similar examples.  This perhaps isn't a blocker for this patchset, but 
should be planned soon, or added if you do another rev for other reasons.

(In a similar vein, the pds_core.rst should be updated for the new 
pds_core.fwctl auxiliary_device, but that's a different problem)

sln


>
> diff --git a/drivers/net/ethernet/pensando/Kconfig b/drivers/net/ethernet/pensando/Kconfig
> index 01fe76786f77..c99758adf3ad 100644
> --- a/drivers/net/ethernet/pensando/Kconfig
> +++ b/drivers/net/ethernet/pensando/Kconfig
> @@ -24,6 +24,7 @@ config IONIC
>   	select NET_DEVLINK
>   	select DIMLIB
>   	select PAGE_POOL
> +	select AUXILIARY_BUS
>   	help
>   	  This enables the support for the Pensando family of Ethernet
>   	  adapters.  More specific information on this driver can be
> diff --git a/drivers/net/ethernet/pensando/ionic/Makefile b/drivers/net/ethernet/pensando/ionic/Makefile
> index 4e7642a2d25f..a598972fef41 100644
> --- a/drivers/net/ethernet/pensando/ionic/Makefile
> +++ b/drivers/net/ethernet/pensando/ionic/Makefile
> @@ -5,5 +5,5 @@ obj-$(CONFIG_IONIC) := ionic.o
>   
>   ionic-y := ionic_main.o ionic_bus_pci.o ionic_devlink.o ionic_dev.o \
>   	   ionic_debugfs.o ionic_lif.o ionic_rx_filter.o ionic_ethtool.o \
> -	   ionic_txrx.o ionic_stats.o ionic_fw.o
> +	   ionic_txrx.o ionic_stats.o ionic_fw.o ionic_aux.o
>   ionic-$(CONFIG_PTP_1588_CLOCK) += ionic_phc.o
> diff --git a/drivers/net/ethernet/pensando/ionic/ionic_api.h b/drivers/net/ethernet/pensando/ionic/ionic_api.h
> new file mode 100644
> index 000000000000..f9fcd1b67b35
> --- /dev/null
> +++ b/drivers/net/ethernet/pensando/ionic/ionic_api.h
> @@ -0,0 +1,21 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
> +
> +#ifndef _IONIC_API_H_
> +#define _IONIC_API_H_
> +
> +#include <linux/auxiliary_bus.h>
> +
> +/**
> + * struct ionic_aux_dev - Auxiliary device information
> + * @lif:        Logical interface
> + * @idx:        Index identifier
> + * @adev:       Auxiliary device
> + */
> +struct ionic_aux_dev {
> +	struct ionic_lif *lif;
> +	int idx;
> +	struct auxiliary_device adev;
> +};
> +
> +#endif /* _IONIC_API_H_ */
> diff --git a/drivers/net/ethernet/pensando/ionic/ionic_aux.c b/drivers/net/ethernet/pensando/ionic/ionic_aux.c
> new file mode 100644
> index 000000000000..781218c3feba
> --- /dev/null
> +++ b/drivers/net/ethernet/pensando/ionic/ionic_aux.c
> @@ -0,0 +1,95 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
> +
> +#include <linux/kernel.h>
> +#include "ionic.h"
> +#include "ionic_lif.h"
> +#include "ionic_aux.h"
> +
> +static DEFINE_IDA(aux_ida);
> +
> +static void ionic_auxbus_release(struct device *dev)
> +{
> +	struct ionic_aux_dev *ionic_adev;
> +
> +	ionic_adev = container_of(dev, struct ionic_aux_dev, adev.dev);
> +	kfree(ionic_adev);
> +}
> +
> +int ionic_auxbus_register(struct ionic_lif *lif)
> +{
> +	struct ionic_aux_dev *ionic_adev;
> +	struct auxiliary_device *aux_dev;
> +	int err, id;
> +
> +	if (!(le64_to_cpu(lif->ionic->ident.lif.capabilities) & IONIC_LIF_CAP_RDMA))
> +		return 0;
> +
> +	ionic_adev = kzalloc(sizeof(*ionic_adev), GFP_KERNEL);
> +	if (!ionic_adev)
> +		return -ENOMEM;
> +
> +	aux_dev = &ionic_adev->adev;
> +
> +	id = ida_alloc_range(&aux_ida, 0, INT_MAX, GFP_KERNEL);
> +	if (id < 0) {
> +		dev_err(lif->ionic->dev, "Failed to allocate aux id: %d\n",
> +			id);
> +		err = id;
> +		goto err_adev_free;
> +	}
> +
> +	aux_dev->id = id;
> +	aux_dev->name = "rdma";
> +	aux_dev->dev.parent = &lif->ionic->pdev->dev;
> +	aux_dev->dev.release = ionic_auxbus_release;
> +	ionic_adev->lif = lif;
> +	err = auxiliary_device_init(aux_dev);
> +	if (err) {
> +		dev_err(lif->ionic->dev, "Failed to initialize %s aux device: %d\n",
> +			aux_dev->name, err);
> +		goto err_ida_free;
> +	}
> +
> +	err = auxiliary_device_add(aux_dev);
> +	if (err) {
> +		dev_err(lif->ionic->dev, "Failed to add %s aux device: %d\n",
> +			aux_dev->name, err);
> +		goto err_aux_uninit;
> +	}
> +
> +	lif->ionic_adev = ionic_adev;
> +
> +	return 0;
> +
> +err_aux_uninit:
> +	auxiliary_device_uninit(aux_dev);
> +err_ida_free:
> +	ida_free(&aux_ida, id);
> +err_adev_free:
> +	kfree(ionic_adev);
> +
> +	return err;
> +}
> +
> +void ionic_auxbus_unregister(struct ionic_lif *lif)
> +{
> +	struct auxiliary_device *aux_dev;
> +	int id;
> +
> +	mutex_lock(&lif->adev_lock);
> +	if (!lif->ionic_adev)
> +		goto out;
> +
> +	aux_dev = &lif->ionic_adev->adev;
> +	id = aux_dev->id;
> +
> +	auxiliary_device_delete(aux_dev);
> +	auxiliary_device_uninit(aux_dev);
> +	ida_free(&aux_ida, id);
> +
> +	lif->ionic_adev = NULL;
> +
> +out:
> +	mutex_unlock(&lif->adev_lock);
> +}
> diff --git a/drivers/net/ethernet/pensando/ionic/ionic_aux.h b/drivers/net/ethernet/pensando/ionic/ionic_aux.h
> new file mode 100644
> index 000000000000..f5528a9f187d
> --- /dev/null
> +++ b/drivers/net/ethernet/pensando/ionic/ionic_aux.h
> @@ -0,0 +1,10 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
> +
> +#ifndef _IONIC_AUX_H_
> +#define _IONIC_AUX_H_
> +
> +int ionic_auxbus_register(struct ionic_lif *lif);
> +void ionic_auxbus_unregister(struct ionic_lif *lif);
> +
> +#endif /* _IONIC_AUX_H_ */
> diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
> index 4c377bdc62c8..bb75044dfb82 100644
> --- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
> +++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
> @@ -9,6 +9,7 @@
>   #include "ionic.h"
>   #include "ionic_bus.h"
>   #include "ionic_lif.h"
> +#include "ionic_aux.h"
>   #include "ionic_debugfs.h"
>   
>   /* Supported devices */
> @@ -375,6 +376,8 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
>   		goto err_out_deregister_devlink;
>   	}
>   
> +	ionic_auxbus_register(ionic->lif);
> +
>   	mod_timer(&ionic->watchdog_timer,
>   		  round_jiffies(jiffies + ionic->watchdog_period));
>   	ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
> @@ -415,6 +418,7 @@ static void ionic_remove(struct pci_dev *pdev)
>   
>   		if (ionic->lif->doorbell_wa)
>   			cancel_delayed_work_sync(&ionic->doorbell_check_dwork);
> +		ionic_auxbus_unregister(ionic->lif);
>   		ionic_lif_unregister(ionic->lif);
>   		ionic_devlink_unregister(ionic);
>   		ionic_lif_deinit(ionic->lif);
> @@ -444,6 +448,7 @@ static void ionic_reset_prepare(struct pci_dev *pdev)
>   	timer_delete_sync(&ionic->watchdog_timer);
>   	cancel_work_sync(&lif->deferred.work);
>   
> +	ionic_auxbus_unregister(ionic->lif);
>   	mutex_lock(&lif->queue_lock);
>   	ionic_stop_queues_reconfig(lif);
>   	ionic_txrx_free(lif);
> diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
> index 7707a9e53c43..146659f6862a 100644
> --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
> +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
> @@ -19,6 +19,7 @@
>   #include "ionic_bus.h"
>   #include "ionic_dev.h"
>   #include "ionic_lif.h"
> +#include "ionic_aux.h"
>   #include "ionic_txrx.h"
>   #include "ionic_ethtool.h"
>   #include "ionic_debugfs.h"
> @@ -3293,6 +3294,7 @@ int ionic_lif_alloc(struct ionic *ionic)
>   
>   	mutex_init(&lif->queue_lock);
>   	mutex_init(&lif->config_lock);
> +	mutex_init(&lif->adev_lock);
>   
>   	spin_lock_init(&lif->adminq_lock);
>   
> @@ -3349,6 +3351,7 @@ int ionic_lif_alloc(struct ionic *ionic)
>   	lif->info = NULL;
>   	lif->info_pa = 0;
>   err_out_free_mutex:
> +	mutex_destroy(&lif->adev_lock);
>   	mutex_destroy(&lif->config_lock);
>   	mutex_destroy(&lif->queue_lock);
>   err_out_free_netdev:
> @@ -3384,6 +3387,7 @@ static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
>   
>   	netif_device_detach(lif->netdev);
>   
> +	ionic_auxbus_unregister(ionic->lif);
>   	mutex_lock(&lif->queue_lock);
>   	if (test_bit(IONIC_LIF_F_UP, lif->state)) {
>   		dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
> @@ -3446,6 +3450,8 @@ int ionic_restart_lif(struct ionic_lif *lif)
>   	netif_device_attach(lif->netdev);
>   	ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
>   
> +	ionic_auxbus_register(ionic->lif);
> +
>   	return 0;
>   
>   err_txrx_free:
> @@ -3532,6 +3538,7 @@ void ionic_lif_free(struct ionic_lif *lif)
>   
>   	mutex_destroy(&lif->config_lock);
>   	mutex_destroy(&lif->queue_lock);
> +	mutex_destroy(&lif->adev_lock);
>   
>   	/* free netdev & lif */
>   	ionic_debugfs_del_lif(lif);
> diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
> index e01756fb7fdd..43bdd0fb8733 100644
> --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.h
> +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
> @@ -10,6 +10,7 @@
>   #include <linux/dim.h>
>   #include <linux/pci.h>
>   #include "ionic_rx_filter.h"
> +#include "ionic_api.h"
>   
>   #define IONIC_ADMINQ_LENGTH	16	/* must be a power of two */
>   #define IONIC_NOTIFYQ_LENGTH	64	/* must be a power of two */
> @@ -225,6 +226,8 @@ struct ionic_lif {
>   	dma_addr_t info_pa;
>   	u32 info_sz;
>   	struct ionic_qtype_info qtype_info[IONIC_QTYPE_MAX];
> +	struct ionic_aux_dev *ionic_adev;
> +	struct mutex adev_lock;	/* lock for aux_dev actions */
>   
>   	u8 rss_hash_key[IONIC_RSS_HASH_KEY_SIZE];
>   	u8 *rss_ind_tbl;

Powered by blists - more mailing lists