| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20200520070415.3392210-16-jeffrey.t.kirsher@intel.com>
Date: Wed, 20 May 2020 00:04:14 -0700
From: Jeff Kirsher <jeffrey.t.kirsher@...el.com>
To: dledford@...hat.com, jgg@...lanox.com, davem@...emloft.net,
gregkh@...uxfoundation.org
Cc: Shiraz Saleem <shiraz.saleem@...el.com>,
linux-rdma@...r.kernel.org, netdev@...r.kernel.org,
nhorman@...hat.com, sassmann@...hat.com, poswald@...e.com
Subject: [RDMA RFC v6 15/16] RDMA/irdma: Add irdma Kconfig/Makefile and remove i40iw
From: Shiraz Saleem <shiraz.saleem@...el.com>
Add Kconfig and Makefile to build irdma driver.
Remove i40iw driver and add an alias in irdma.
irdma is the replacement driver that supports X722.
Signed-off-by: Shiraz Saleem <shiraz.saleem@...el.com>
---
.../ABI/stable/sysfs-class-infiniband | 18 -
MAINTAINERS | 8 -
drivers/infiniband/Kconfig | 2 +-
drivers/infiniband/hw/Makefile | 2 +-
drivers/infiniband/hw/i40iw/Kconfig | 9 -
drivers/infiniband/hw/i40iw/Makefile | 9 -
drivers/infiniband/hw/i40iw/i40iw.h | 622 --
drivers/infiniband/hw/i40iw/i40iw_cm.c | 4414 --------------
drivers/infiniband/hw/i40iw/i40iw_cm.h | 462 --
drivers/infiniband/hw/i40iw/i40iw_ctrl.c | 5294 -----------------
drivers/infiniband/hw/i40iw/i40iw_d.h | 1757 ------
drivers/infiniband/hw/i40iw/i40iw_hmc.c | 821 ---
drivers/infiniband/hw/i40iw/i40iw_hmc.h | 241 -
drivers/infiniband/hw/i40iw/i40iw_hw.c | 852 ---
drivers/infiniband/hw/i40iw/i40iw_main.c | 2070 -------
drivers/infiniband/hw/i40iw/i40iw_osdep.h | 217 -
drivers/infiniband/hw/i40iw/i40iw_p.h | 129 -
drivers/infiniband/hw/i40iw/i40iw_pble.c | 612 --
drivers/infiniband/hw/i40iw/i40iw_pble.h | 131 -
drivers/infiniband/hw/i40iw/i40iw_puda.c | 1493 -----
drivers/infiniband/hw/i40iw/i40iw_puda.h | 188 -
drivers/infiniband/hw/i40iw/i40iw_register.h | 1030 ----
drivers/infiniband/hw/i40iw/i40iw_status.h | 102 -
drivers/infiniband/hw/i40iw/i40iw_type.h | 1375 -----
drivers/infiniband/hw/i40iw/i40iw_uk.c | 1232 ----
drivers/infiniband/hw/i40iw/i40iw_user.h | 430 --
drivers/infiniband/hw/i40iw/i40iw_utils.c | 1557 -----
drivers/infiniband/hw/i40iw/i40iw_verbs.c | 2791 ---------
drivers/infiniband/hw/i40iw/i40iw_verbs.h | 179 -
drivers/infiniband/hw/i40iw/i40iw_vf.c | 85 -
drivers/infiniband/hw/i40iw/i40iw_vf.h | 62 -
drivers/infiniband/hw/i40iw/i40iw_virtchnl.c | 756 ---
drivers/infiniband/hw/i40iw/i40iw_virtchnl.h | 124 -
drivers/infiniband/hw/irdma/Kconfig | 11 +
drivers/infiniband/hw/irdma/Makefile | 28 +
drivers/infiniband/hw/irdma/main.c | 1 +
include/uapi/rdma/i40iw-abi.h | 107 -
37 files changed, 42 insertions(+), 29179 deletions(-)
delete mode 100644 drivers/infiniband/hw/i40iw/Kconfig
delete mode 100644 drivers/infiniband/hw/i40iw/Makefile
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_cm.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_cm.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_ctrl.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_d.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_hmc.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_hmc.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_hw.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_main.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_osdep.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_p.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_pble.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_pble.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_puda.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_puda.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_register.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_status.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_type.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_uk.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_user.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_utils.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_verbs.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_verbs.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_vf.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_vf.h
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_virtchnl.c
delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_virtchnl.h
create mode 100644 drivers/infiniband/hw/irdma/Kconfig
create mode 100644 drivers/infiniband/hw/irdma/Makefile
delete mode 100644 include/uapi/rdma/i40iw-abi.h
diff --git a/Documentation/ABI/stable/sysfs-class-infiniband b/Documentation/ABI/stable/sysfs-class-infiniband
index 96dfe1926b76..7dcb45ab3641 100644
--- a/Documentation/ABI/stable/sysfs-class-infiniband
+++ b/Documentation/ABI/stable/sysfs-class-infiniband
@@ -717,24 +717,6 @@ Description:
is the irq number of "sdma3", and M is irq number of "sdma4" in
the /proc/interrupts file.
-
-sysfs interface for Intel(R) X722 iWARP i40iw driver
-----------------------------------------------------
-
-What: /sys/class/infiniband/i40iwX/hw_rev
-What: /sys/class/infiniband/i40iwX/hca_type
-What: /sys/class/infiniband/i40iwX/board_id
-Date: Jan, 2016
-KernelVersion: v4.10
-Contact: linux-rdma@...r.kernel.org
-Description:
- hw_rev: (RO) Hardware revision number
-
- hca_type: (RO) Show HCA type (I40IW)
-
- board_id: (RO) I40IW board ID
-
-
sysfs interface for QLogic qedr NIC Driver
------------------------------------------
diff --git a/MAINTAINERS b/MAINTAINERS
index 853d6bba8d78..598d0e1b3501 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -8745,14 +8745,6 @@ L: linux-pm@...r.kernel.org
S: Supported
F: drivers/cpufreq/intel_pstate.c
-INTEL RDMA RNIC DRIVER
-M: Faisal Latif <faisal.latif@...el.com>
-M: Shiraz Saleem <shiraz.saleem@...el.com>
-L: linux-rdma@...r.kernel.org
-S: Supported
-F: drivers/infiniband/hw/i40iw/
-F: include/uapi/rdma/i40iw-abi.h
-
INTEL SPEED SELECT TECHNOLOGY
M: Srinivas Pandruvada <srinivas.pandruvada@...ux.intel.com>
L: platform-driver-x86@...r.kernel.org
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index ade86388434f..f4cdb865de81 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -85,7 +85,7 @@ source "drivers/infiniband/hw/mthca/Kconfig"
source "drivers/infiniband/hw/qib/Kconfig"
source "drivers/infiniband/hw/cxgb4/Kconfig"
source "drivers/infiniband/hw/efa/Kconfig"
-source "drivers/infiniband/hw/i40iw/Kconfig"
+source "drivers/infiniband/hw/irdma/Kconfig"
source "drivers/infiniband/hw/mlx4/Kconfig"
source "drivers/infiniband/hw/mlx5/Kconfig"
source "drivers/infiniband/hw/ocrdma/Kconfig"
diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile
index 0aeccd984889..fba0b3be903e 100644
--- a/drivers/infiniband/hw/Makefile
+++ b/drivers/infiniband/hw/Makefile
@@ -3,7 +3,7 @@ obj-$(CONFIG_INFINIBAND_MTHCA) += mthca/
obj-$(CONFIG_INFINIBAND_QIB) += qib/
obj-$(CONFIG_INFINIBAND_CXGB4) += cxgb4/
obj-$(CONFIG_INFINIBAND_EFA) += efa/
-obj-$(CONFIG_INFINIBAND_I40IW) += i40iw/
+obj-$(CONFIG_INFINIBAND_IRDMA) += irdma/
obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/
obj-$(CONFIG_MLX5_INFINIBAND) += mlx5/
obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/
diff --git a/drivers/infiniband/hw/i40iw/Kconfig b/drivers/infiniband/hw/i40iw/Kconfig
deleted file mode 100644
index e4b45f4cd8f8..000000000000
--- a/drivers/infiniband/hw/i40iw/Kconfig
+++ /dev/null
@@ -1,9 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config INFINIBAND_I40IW
- tristate "Intel(R) Ethernet X722 iWARP Driver"
- depends on INET && I40E
- depends on IPV6 || !IPV6
- depends on PCI
- select GENERIC_ALLOCATOR
- ---help---
- Intel(R) Ethernet X722 iWARP Driver
diff --git a/drivers/infiniband/hw/i40iw/Makefile b/drivers/infiniband/hw/i40iw/Makefile
deleted file mode 100644
index 34da9eba8a7c..000000000000
--- a/drivers/infiniband/hw/i40iw/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-
-obj-$(CONFIG_INFINIBAND_I40IW) += i40iw.o
-
-i40iw-objs :=\
- i40iw_cm.o i40iw_ctrl.o \
- i40iw_hmc.o i40iw_hw.o i40iw_main.o \
- i40iw_pble.o i40iw_puda.o i40iw_uk.o i40iw_utils.o \
- i40iw_verbs.o i40iw_virtchnl.o i40iw_vf.o
diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h
deleted file mode 100644
index 1ba7561f9cbb..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw.h
+++ /dev/null
@@ -1,622 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_IW_H
-#define I40IW_IW_H
-#include <linux/netdevice.h>
-#include <linux/inetdevice.h>
-#include <linux/spinlock.h>
-#include <linux/kernel.h>
-#include <linux/delay.h>
-#include <linux/pci.h>
-#include <linux/dma-mapping.h>
-#include <linux/workqueue.h>
-#include <linux/slab.h>
-#include <linux/io.h>
-#include <linux/crc32c.h>
-#include <linux/net/intel/i40e_client.h>
-#include <rdma/ib_smi.h>
-#include <rdma/ib_verbs.h>
-#include <rdma/ib_pack.h>
-#include <rdma/rdma_cm.h>
-#include <rdma/iw_cm.h>
-#include <crypto/hash.h>
-
-#include "i40iw_status.h"
-#include "i40iw_osdep.h"
-#include "i40iw_d.h"
-#include "i40iw_hmc.h"
-
-#include "i40iw_type.h"
-#include "i40iw_p.h"
-#include <rdma/i40iw-abi.h>
-#include "i40iw_pble.h"
-#include "i40iw_verbs.h"
-#include "i40iw_cm.h"
-#include "i40iw_user.h"
-#include "i40iw_puda.h"
-
-#define I40IW_FW_VER_DEFAULT 2
-#define I40IW_HW_VERSION 2
-
-#define I40IW_ARP_ADD 1
-#define I40IW_ARP_DELETE 2
-#define I40IW_ARP_RESOLVE 3
-
-#define I40IW_MACIP_ADD 1
-#define I40IW_MACIP_DELETE 2
-
-#define IW_CCQ_SIZE (I40IW_CQP_SW_SQSIZE_2048 + 1)
-#define IW_CEQ_SIZE 2048
-#define IW_AEQ_SIZE 2048
-
-#define RX_BUF_SIZE (1536 + 8)
-#define IW_REG0_SIZE (4 * 1024)
-#define IW_TX_TIMEOUT (6 * HZ)
-#define IW_FIRST_QPN 1
-#define IW_SW_CONTEXT_ALIGN 1024
-
-#define MAX_DPC_ITERATIONS 128
-
-#define I40IW_EVENT_TIMEOUT 100000
-#define I40IW_VCHNL_EVENT_TIMEOUT 100000
-
-#define I40IW_NO_VLAN 0xffff
-#define I40IW_NO_QSET 0xffff
-
-/* access to mcast filter list */
-#define IW_ADD_MCAST false
-#define IW_DEL_MCAST true
-
-#define I40IW_DRV_OPT_ENABLE_MPA_VER_0 0x00000001
-#define I40IW_DRV_OPT_DISABLE_MPA_CRC 0x00000002
-#define I40IW_DRV_OPT_DISABLE_FIRST_WRITE 0x00000004
-#define I40IW_DRV_OPT_DISABLE_INTF 0x00000008
-#define I40IW_DRV_OPT_ENABLE_MSI 0x00000010
-#define I40IW_DRV_OPT_DUAL_LOGICAL_PORT 0x00000020
-#define I40IW_DRV_OPT_NO_INLINE_DATA 0x00000080
-#define I40IW_DRV_OPT_DISABLE_INT_MOD 0x00000100
-#define I40IW_DRV_OPT_DISABLE_VIRT_WQ 0x00000200
-#define I40IW_DRV_OPT_ENABLE_PAU 0x00000400
-#define I40IW_DRV_OPT_MCAST_LOGPORT_MAP 0x00000800
-
-#define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types)
-#define IW_CFG_FPM_QP_COUNT 32768
-#define I40IW_MAX_PAGES_PER_FMR 512
-#define I40IW_MIN_PAGES_PER_FMR 1
-#define I40IW_CQP_COMPL_RQ_WQE_FLUSHED 2
-#define I40IW_CQP_COMPL_SQ_WQE_FLUSHED 3
-#define I40IW_CQP_COMPL_RQ_SQ_WQE_FLUSHED 4
-
-struct i40iw_cqp_compl_info {
- u32 op_ret_val;
- u16 maj_err_code;
- u16 min_err_code;
- bool error;
- u8 op_code;
-};
-
-#define i40iw_pr_err(fmt, args ...) pr_err("%s: "fmt, __func__, ## args)
-
-#define i40iw_pr_info(fmt, args ...) pr_info("%s: " fmt, __func__, ## args)
-
-#define i40iw_pr_warn(fmt, args ...) pr_warn("%s: " fmt, __func__, ## args)
-
-struct i40iw_cqp_request {
- struct cqp_commands_info info;
- wait_queue_head_t waitq;
- struct list_head list;
- atomic_t refcount;
- void (*callback_fcn)(struct i40iw_cqp_request*, u32);
- void *param;
- struct i40iw_cqp_compl_info compl_info;
- bool waiting;
- bool request_done;
- bool dynamic;
-};
-
-struct i40iw_cqp {
- struct i40iw_sc_cqp sc_cqp;
- spinlock_t req_lock; /*cqp request list */
- wait_queue_head_t waitq;
- struct i40iw_dma_mem sq;
- struct i40iw_dma_mem host_ctx;
- u64 *scratch_array;
- struct i40iw_cqp_request *cqp_requests;
- struct list_head cqp_avail_reqs;
- struct list_head cqp_pending_reqs;
-};
-
-struct i40iw_device;
-
-struct i40iw_ccq {
- struct i40iw_sc_cq sc_cq;
- spinlock_t lock; /* ccq control */
- wait_queue_head_t waitq;
- struct i40iw_dma_mem mem_cq;
- struct i40iw_dma_mem shadow_area;
-};
-
-struct i40iw_ceq {
- struct i40iw_sc_ceq sc_ceq;
- struct i40iw_dma_mem mem;
- u32 irq;
- u32 msix_idx;
- struct i40iw_device *iwdev;
- struct tasklet_struct dpc_tasklet;
-};
-
-struct i40iw_aeq {
- struct i40iw_sc_aeq sc_aeq;
- struct i40iw_dma_mem mem;
-};
-
-struct i40iw_arp_entry {
- u32 ip_addr[4];
- u8 mac_addr[ETH_ALEN];
-};
-
-enum init_completion_state {
- INVALID_STATE = 0,
- INITIAL_STATE,
- CQP_CREATED,
- HMC_OBJS_CREATED,
- PBLE_CHUNK_MEM,
- CCQ_CREATED,
- AEQ_CREATED,
- CEQ_CREATED,
- ILQ_CREATED,
- IEQ_CREATED,
- IP_ADDR_REGISTERED,
- RDMA_DEV_REGISTERED
-};
-
-struct i40iw_msix_vector {
- u32 idx;
- u32 irq;
- u32 cpu_affinity;
- u32 ceq_id;
- cpumask_t mask;
-};
-
-struct l2params_work {
- struct work_struct work;
- struct i40iw_device *iwdev;
- struct i40iw_l2params l2params;
-};
-
-#define I40IW_MSIX_TABLE_SIZE 65
-
-struct virtchnl_work {
- struct work_struct work;
- union {
- struct i40iw_cqp_request *cqp_request;
- struct i40iw_virtchnl_work_info work_info;
- };
-};
-
-struct i40e_qvlist_info;
-
-struct i40iw_device {
- struct i40iw_ib_device *iwibdev;
- struct net_device *netdev;
- wait_queue_head_t vchnl_waitq;
- struct i40iw_sc_dev sc_dev;
- struct i40iw_sc_vsi vsi;
- struct i40iw_handler *hdl;
- struct i40e_info *ldev;
- struct i40e_client *client;
- struct i40iw_hw hw;
- struct i40iw_cm_core cm_core;
- u8 *mem_resources;
- unsigned long *allocated_qps;
- unsigned long *allocated_cqs;
- unsigned long *allocated_mrs;
- unsigned long *allocated_pds;
- unsigned long *allocated_arps;
- struct i40iw_qp **qp_table;
- bool msix_shared;
- u32 msix_count;
- struct i40iw_msix_vector *iw_msixtbl;
- struct i40e_qvlist_info *iw_qvlist;
-
- struct i40iw_hmc_pble_rsrc *pble_rsrc;
- struct i40iw_arp_entry *arp_table;
- struct i40iw_cqp cqp;
- struct i40iw_ccq ccq;
- u32 ceqs_count;
- struct i40iw_ceq *ceqlist;
- struct i40iw_aeq aeq;
- u32 arp_table_size;
- u32 next_arp_index;
- spinlock_t resource_lock; /* hw resource access */
- spinlock_t qptable_lock;
- u32 vendor_id;
- u32 vendor_part_id;
- u32 of_device_registered;
-
- u32 device_cap_flags;
- unsigned long db_start;
- u8 resource_profile;
- u8 max_rdma_vfs;
- u8 max_enabled_vfs;
- u8 max_sge;
- u8 iw_status;
- u8 send_term_ok;
- bool push_mode; /* Initialized from parameter passed to driver */
-
- /* x710 specific */
- struct mutex pbl_mutex;
- struct tasklet_struct dpc_tasklet;
- struct workqueue_struct *virtchnl_wq;
- struct virtchnl_work virtchnl_w[I40IW_MAX_PE_ENABLED_VF_COUNT];
- struct i40iw_dma_mem obj_mem;
- struct i40iw_dma_mem obj_next;
- u8 *hmc_info_mem;
- u32 sd_type;
- struct workqueue_struct *param_wq;
- atomic_t params_busy;
- enum init_completion_state init_state;
- u16 mac_ip_table_idx;
- atomic_t vchnl_msgs;
- u32 max_mr;
- u32 max_qp;
- u32 max_cq;
- u32 max_pd;
- u32 next_qp;
- u32 next_cq;
- u32 next_pd;
- u32 max_mr_size;
- u32 max_qp_wr;
- u32 max_cqe;
- u32 mr_stagmask;
- u32 mpa_version;
- bool dcb;
- bool closing;
- bool reset;
- u32 used_pds;
- u32 used_cqs;
- u32 used_mrs;
- u32 used_qps;
- wait_queue_head_t close_wq;
- atomic64_t use_count;
-};
-
-struct i40iw_ib_device {
- struct ib_device ibdev;
- struct i40iw_device *iwdev;
-};
-
-struct i40iw_handler {
- struct list_head list;
- struct i40e_client *client;
- struct i40iw_device device;
- struct i40e_info ldev;
-};
-
-/**
- * i40iw_fw_major_ver - get firmware major version
- * @dev: iwarp device
- **/
-static inline u64 i40iw_fw_major_ver(struct i40iw_sc_dev *dev)
-{
- return RS_64(dev->feature_info[I40IW_FEATURE_FW_INFO],
- I40IW_FW_VER_MAJOR);
-}
-
-/**
- * i40iw_fw_minor_ver - get firmware minor version
- * @dev: iwarp device
- **/
-static inline u64 i40iw_fw_minor_ver(struct i40iw_sc_dev *dev)
-{
- return RS_64(dev->feature_info[I40IW_FEATURE_FW_INFO],
- I40IW_FW_VER_MINOR);
-}
-
-/**
- * to_iwdev - get device
- * @ibdev: ib device
- **/
-static inline struct i40iw_device *to_iwdev(struct ib_device *ibdev)
-{
- return container_of(ibdev, struct i40iw_ib_device, ibdev)->iwdev;
-}
-
-/**
- * to_ucontext - get user context
- * @ibucontext: ib user context
- **/
-static inline struct i40iw_ucontext *to_ucontext(struct ib_ucontext *ibucontext)
-{
- return container_of(ibucontext, struct i40iw_ucontext, ibucontext);
-}
-
-/**
- * to_iwpd - get protection domain
- * @ibpd: ib pd
- **/
-static inline struct i40iw_pd *to_iwpd(struct ib_pd *ibpd)
-{
- return container_of(ibpd, struct i40iw_pd, ibpd);
-}
-
-/**
- * to_iwmr - get device memory region
- * @ibdev: ib memory region
- **/
-static inline struct i40iw_mr *to_iwmr(struct ib_mr *ibmr)
-{
- return container_of(ibmr, struct i40iw_mr, ibmr);
-}
-
-/**
- * to_iwmr_from_ibfmr - get device memory region
- * @ibfmr: ib fmr
- **/
-static inline struct i40iw_mr *to_iwmr_from_ibfmr(struct ib_fmr *ibfmr)
-{
- return container_of(ibfmr, struct i40iw_mr, ibfmr);
-}
-
-/**
- * to_iwmw - get device memory window
- * @ibmw: ib memory window
- **/
-static inline struct i40iw_mr *to_iwmw(struct ib_mw *ibmw)
-{
- return container_of(ibmw, struct i40iw_mr, ibmw);
-}
-
-/**
- * to_iwcq - get completion queue
- * @ibcq: ib cqdevice
- **/
-static inline struct i40iw_cq *to_iwcq(struct ib_cq *ibcq)
-{
- return container_of(ibcq, struct i40iw_cq, ibcq);
-}
-
-/**
- * to_iwqp - get device qp
- * @ibqp: ib qp
- **/
-static inline struct i40iw_qp *to_iwqp(struct ib_qp *ibqp)
-{
- return container_of(ibqp, struct i40iw_qp, ibqp);
-}
-
-/* i40iw.c */
-void i40iw_add_ref(struct ib_qp *);
-void i40iw_rem_ref(struct ib_qp *);
-struct ib_qp *i40iw_get_qp(struct ib_device *, int);
-
-void i40iw_flush_wqes(struct i40iw_device *iwdev,
- struct i40iw_qp *qp);
-
-void i40iw_manage_arp_cache(struct i40iw_device *iwdev,
- unsigned char *mac_addr,
- u32 *ip_addr,
- bool ipv4,
- u32 action);
-
-int i40iw_manage_apbvt(struct i40iw_device *iwdev,
- u16 accel_local_port,
- bool add_port);
-
-struct i40iw_cqp_request *i40iw_get_cqp_request(struct i40iw_cqp *cqp, bool wait);
-void i40iw_free_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request);
-void i40iw_put_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request);
-
-/**
- * i40iw_alloc_resource - allocate a resource
- * @iwdev: device pointer
- * @resource_array: resource bit array:
- * @max_resources: maximum resource number
- * @req_resources_num: Allocated resource number
- * @next: next free id
- **/
-static inline int i40iw_alloc_resource(struct i40iw_device *iwdev,
- unsigned long *resource_array,
- u32 max_resources,
- u32 *req_resource_num,
- u32 *next)
-{
- u32 resource_num;
- unsigned long flags;
-
- spin_lock_irqsave(&iwdev->resource_lock, flags);
- resource_num = find_next_zero_bit(resource_array, max_resources, *next);
- if (resource_num >= max_resources) {
- resource_num = find_first_zero_bit(resource_array, max_resources);
- if (resource_num >= max_resources) {
- spin_unlock_irqrestore(&iwdev->resource_lock, flags);
- return -EOVERFLOW;
- }
- }
- set_bit(resource_num, resource_array);
- *next = resource_num + 1;
- if (*next == max_resources)
- *next = 0;
- *req_resource_num = resource_num;
- spin_unlock_irqrestore(&iwdev->resource_lock, flags);
-
- return 0;
-}
-
-/**
- * i40iw_is_resource_allocated - detrmine if resource is
- * allocated
- * @iwdev: device pointer
- * @resource_array: resource array for the resource_num
- * @resource_num: resource number to check
- **/
-static inline bool i40iw_is_resource_allocated(struct i40iw_device *iwdev,
- unsigned long *resource_array,
- u32 resource_num)
-{
- bool bit_is_set;
- unsigned long flags;
-
- spin_lock_irqsave(&iwdev->resource_lock, flags);
-
- bit_is_set = test_bit(resource_num, resource_array);
- spin_unlock_irqrestore(&iwdev->resource_lock, flags);
-
- return bit_is_set;
-}
-
-/**
- * i40iw_free_resource - free a resource
- * @iwdev: device pointer
- * @resource_array: resource array for the resource_num
- * @resource_num: resource number to free
- **/
-static inline void i40iw_free_resource(struct i40iw_device *iwdev,
- unsigned long *resource_array,
- u32 resource_num)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&iwdev->resource_lock, flags);
- clear_bit(resource_num, resource_array);
- spin_unlock_irqrestore(&iwdev->resource_lock, flags);
-}
-
-/**
- * to_iwhdl - Get the handler from the device pointer
- * @iwdev: device pointer
- **/
-static inline struct i40iw_handler *to_iwhdl(struct i40iw_device *iw_dev)
-{
- return container_of(iw_dev, struct i40iw_handler, device);
-}
-
-struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev);
-
-/**
- * iw_init_resources -
- */
-u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev);
-
-int i40iw_register_rdma_device(struct i40iw_device *iwdev);
-void i40iw_port_ibevent(struct i40iw_device *iwdev);
-void i40iw_cm_disconn(struct i40iw_qp *iwqp);
-void i40iw_cm_disconn_worker(void *);
-int mini_cm_recv_pkt(struct i40iw_cm_core *, struct i40iw_device *,
- struct sk_buff *);
-
-enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev,
- struct i40iw_cqp_request *cqp_request);
-enum i40iw_status_code i40iw_add_mac_addr(struct i40iw_device *iwdev,
- u8 *mac_addr, u8 *mac_index);
-int i40iw_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *);
-void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq);
-
-void i40iw_cleanup_pending_cqp_op(struct i40iw_device *iwdev);
-void i40iw_rem_pdusecount(struct i40iw_pd *iwpd, struct i40iw_device *iwdev);
-void i40iw_add_pdusecount(struct i40iw_pd *iwpd);
-void i40iw_rem_devusecount(struct i40iw_device *iwdev);
-void i40iw_add_devusecount(struct i40iw_device *iwdev);
-void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp,
- struct i40iw_modify_qp_info *info, bool wait);
-
-void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev,
- struct i40iw_sc_qp *qp,
- bool suspend);
-enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev,
- struct i40iw_cm_info *cminfo,
- enum i40iw_quad_entry_type etype,
- enum i40iw_quad_hash_manage_type mtype,
- void *cmnode,
- bool wait);
-void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf);
-void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp);
-void i40iw_free_qp_resources(struct i40iw_device *iwdev,
- struct i40iw_qp *iwqp,
- u32 qp_num);
-enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev,
- struct i40iw_dma_mem *memptr,
- u32 size, u32 mask);
-
-void i40iw_request_reset(struct i40iw_device *iwdev);
-void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev);
-int i40iw_setup_cm_core(struct i40iw_device *iwdev);
-void i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core);
-void i40iw_process_ceq(struct i40iw_device *, struct i40iw_ceq *iwceq);
-void i40iw_process_aeq(struct i40iw_device *);
-void i40iw_next_iw_state(struct i40iw_qp *iwqp,
- u8 state, u8 del_hash,
- u8 term, u8 term_len);
-int i40iw_send_syn(struct i40iw_cm_node *cm_node, u32 sendack);
-int i40iw_send_reset(struct i40iw_cm_node *cm_node);
-struct i40iw_cm_node *i40iw_find_node(struct i40iw_cm_core *cm_core,
- u16 rem_port,
- u32 *rem_addr,
- u16 loc_port,
- u32 *loc_addr,
- bool add_refcnt,
- bool accelerated_list);
-
-enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev,
- struct i40iw_sc_qp *qp,
- struct i40iw_qp_flush_info *info,
- bool wait);
-
-void i40iw_gen_ae(struct i40iw_device *iwdev,
- struct i40iw_sc_qp *qp,
- struct i40iw_gen_ae_info *info,
- bool wait);
-
-void i40iw_copy_ip_ntohl(u32 *dst, __be32 *src);
-struct ib_mr *i40iw_reg_phys_mr(struct ib_pd *ib_pd,
- u64 addr,
- u64 size,
- int acc,
- u64 *iova_start);
-
-int i40iw_inetaddr_event(struct notifier_block *notifier,
- unsigned long event,
- void *ptr);
-int i40iw_inet6addr_event(struct notifier_block *notifier,
- unsigned long event,
- void *ptr);
-int i40iw_net_event(struct notifier_block *notifier,
- unsigned long event,
- void *ptr);
-int i40iw_netdevice_event(struct notifier_block *notifier,
- unsigned long event,
- void *ptr);
-
-#endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c
deleted file mode 100644
index fa7a5ff498c7..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.c
+++ /dev/null
@@ -1,4414 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include <linux/atomic.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/init.h>
-#include <linux/if_arp.h>
-#include <linux/if_vlan.h>
-#include <linux/notifier.h>
-#include <linux/net.h>
-#include <linux/types.h>
-#include <linux/timer.h>
-#include <linux/time.h>
-#include <linux/delay.h>
-#include <linux/etherdevice.h>
-#include <linux/netdevice.h>
-#include <linux/random.h>
-#include <linux/list.h>
-#include <linux/threads.h>
-#include <linux/highmem.h>
-#include <net/arp.h>
-#include <net/ndisc.h>
-#include <net/neighbour.h>
-#include <net/route.h>
-#include <net/addrconf.h>
-#include <net/ip6_route.h>
-#include <net/ip_fib.h>
-#include <net/secure_seq.h>
-#include <net/tcp.h>
-#include <asm/checksum.h>
-
-#include "i40iw.h"
-
-static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *);
-static void i40iw_cm_post_event(struct i40iw_cm_event *event);
-static void i40iw_disconnect_worker(struct work_struct *work);
-
-/**
- * i40iw_free_sqbuf - put back puda buffer if refcount = 0
- * @vsi: pointer to vsi structure
- * @buf: puda buffer to free
- */
-void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp)
-{
- struct i40iw_puda_buf *buf = (struct i40iw_puda_buf *)bufp;
- struct i40iw_puda_rsrc *ilq = vsi->ilq;
-
- if (!atomic_dec_return(&buf->refcount))
- i40iw_puda_ret_bufpool(ilq, buf);
-}
-
-/**
- * i40iw_derive_hw_ird_setting - Calculate IRD
- *
- * @cm_ird: IRD of connection's node
- *
- * The ird from the connection is rounded to a supported HW
- * setting (2,8,32,64) and then encoded for ird_size field of
- * qp_ctx
- */
-static u8 i40iw_derive_hw_ird_setting(u16 cm_ird)
-{
- u8 encoded_ird_size;
-
- /* ird_size field is encoded in qp_ctx */
- switch (cm_ird ? roundup_pow_of_two(cm_ird) : 0) {
- case I40IW_HW_IRD_SETTING_64:
- encoded_ird_size = 3;
- break;
- case I40IW_HW_IRD_SETTING_32:
- case I40IW_HW_IRD_SETTING_16:
- encoded_ird_size = 2;
- break;
- case I40IW_HW_IRD_SETTING_8:
- case I40IW_HW_IRD_SETTING_4:
- encoded_ird_size = 1;
- break;
- case I40IW_HW_IRD_SETTING_2:
- default:
- encoded_ird_size = 0;
- break;
- }
- return encoded_ird_size;
-}
-
-/**
- * i40iw_record_ird_ord - Record IRD/ORD passed in
- * @cm_node: connection's node
- * @conn_ird: connection IRD
- * @conn_ord: connection ORD
- */
-static void i40iw_record_ird_ord(struct i40iw_cm_node *cm_node, u32 conn_ird,
- u32 conn_ord)
-{
- if (conn_ird > I40IW_MAX_IRD_SIZE)
- conn_ird = I40IW_MAX_IRD_SIZE;
-
- if (conn_ord > I40IW_MAX_ORD_SIZE)
- conn_ord = I40IW_MAX_ORD_SIZE;
- else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO)
- conn_ord = 1;
-
- cm_node->ird_size = conn_ird;
- cm_node->ord_size = conn_ord;
-}
-
-/**
- * i40iw_copy_ip_ntohl - change network to host ip
- * @dst: host ip
- * @src: big endian
- */
-void i40iw_copy_ip_ntohl(u32 *dst, __be32 *src)
-{
- *dst++ = ntohl(*src++);
- *dst++ = ntohl(*src++);
- *dst++ = ntohl(*src++);
- *dst = ntohl(*src);
-}
-
-/**
- * i40iw_copy_ip_htonl - change host addr to network ip
- * @dst: host ip
- * @src: little endian
- */
-static inline void i40iw_copy_ip_htonl(__be32 *dst, u32 *src)
-{
- *dst++ = htonl(*src++);
- *dst++ = htonl(*src++);
- *dst++ = htonl(*src++);
- *dst = htonl(*src);
-}
-
-/**
- * i40iw_fill_sockaddr4 - get addr info for passive connection
- * @cm_node: connection's node
- * @event: upper layer's cm event
- */
-static inline void i40iw_fill_sockaddr4(struct i40iw_cm_node *cm_node,
- struct iw_cm_event *event)
-{
- struct sockaddr_in *laddr = (struct sockaddr_in *)&event->local_addr;
- struct sockaddr_in *raddr = (struct sockaddr_in *)&event->remote_addr;
-
- laddr->sin_family = AF_INET;
- raddr->sin_family = AF_INET;
-
- laddr->sin_port = htons(cm_node->loc_port);
- raddr->sin_port = htons(cm_node->rem_port);
-
- laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]);
- raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]);
-}
-
-/**
- * i40iw_fill_sockaddr6 - get ipv6 addr info for passive side
- * @cm_node: connection's node
- * @event: upper layer's cm event
- */
-static inline void i40iw_fill_sockaddr6(struct i40iw_cm_node *cm_node,
- struct iw_cm_event *event)
-{
- struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&event->local_addr;
- struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&event->remote_addr;
-
- laddr6->sin6_family = AF_INET6;
- raddr6->sin6_family = AF_INET6;
-
- laddr6->sin6_port = htons(cm_node->loc_port);
- raddr6->sin6_port = htons(cm_node->rem_port);
-
- i40iw_copy_ip_htonl(laddr6->sin6_addr.in6_u.u6_addr32,
- cm_node->loc_addr);
- i40iw_copy_ip_htonl(raddr6->sin6_addr.in6_u.u6_addr32,
- cm_node->rem_addr);
-}
-
-/**
- * i40iw_get_addr_info
- * @cm_node: contains ip/tcp info
- * @cm_info: to get a copy of the cm_node ip/tcp info
-*/
-static void i40iw_get_addr_info(struct i40iw_cm_node *cm_node,
- struct i40iw_cm_info *cm_info)
-{
- cm_info->ipv4 = cm_node->ipv4;
- cm_info->vlan_id = cm_node->vlan_id;
- memcpy(cm_info->loc_addr, cm_node->loc_addr, sizeof(cm_info->loc_addr));
- memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr));
- cm_info->loc_port = cm_node->loc_port;
- cm_info->rem_port = cm_node->rem_port;
- cm_info->user_pri = cm_node->user_pri;
-}
-
-/**
- * i40iw_get_cmevent_info - for cm event upcall
- * @cm_node: connection's node
- * @cm_id: upper layers cm struct for the event
- * @event: upper layer's cm event
- */
-static inline void i40iw_get_cmevent_info(struct i40iw_cm_node *cm_node,
- struct iw_cm_id *cm_id,
- struct iw_cm_event *event)
-{
- memcpy(&event->local_addr, &cm_id->m_local_addr,
- sizeof(event->local_addr));
- memcpy(&event->remote_addr, &cm_id->m_remote_addr,
- sizeof(event->remote_addr));
- if (cm_node) {
- event->private_data = (void *)cm_node->pdata_buf;
- event->private_data_len = (u8)cm_node->pdata.size;
- event->ird = cm_node->ird_size;
- event->ord = cm_node->ord_size;
- }
-}
-
-/**
- * i40iw_send_cm_event - upcall cm's event handler
- * @cm_node: connection's node
- * @cm_id: upper layer's cm info struct
- * @type: Event type to indicate
- * @status: status for the event type
- */
-static int i40iw_send_cm_event(struct i40iw_cm_node *cm_node,
- struct iw_cm_id *cm_id,
- enum iw_cm_event_type type,
- int status)
-{
- struct iw_cm_event event;
-
- memset(&event, 0, sizeof(event));
- event.event = type;
- event.status = status;
- switch (type) {
- case IW_CM_EVENT_CONNECT_REQUEST:
- if (cm_node->ipv4)
- i40iw_fill_sockaddr4(cm_node, &event);
- else
- i40iw_fill_sockaddr6(cm_node, &event);
- event.provider_data = (void *)cm_node;
- event.private_data = (void *)cm_node->pdata_buf;
- event.private_data_len = (u8)cm_node->pdata.size;
- event.ird = cm_node->ird_size;
- break;
- case IW_CM_EVENT_CONNECT_REPLY:
- i40iw_get_cmevent_info(cm_node, cm_id, &event);
- break;
- case IW_CM_EVENT_ESTABLISHED:
- event.ird = cm_node->ird_size;
- event.ord = cm_node->ord_size;
- break;
- case IW_CM_EVENT_DISCONNECT:
- break;
- case IW_CM_EVENT_CLOSE:
- break;
- default:
- i40iw_pr_err("event type received type = %d\n", type);
- return -1;
- }
- return cm_id->event_handler(cm_id, &event);
-}
-
-/**
- * i40iw_create_event - create cm event
- * @cm_node: connection's node
- * @type: Event type to generate
- */
-static struct i40iw_cm_event *i40iw_create_event(struct i40iw_cm_node *cm_node,
- enum i40iw_cm_event_type type)
-{
- struct i40iw_cm_event *event;
-
- if (!cm_node->cm_id)
- return NULL;
-
- event = kzalloc(sizeof(*event), GFP_ATOMIC);
-
- if (!event)
- return NULL;
-
- event->type = type;
- event->cm_node = cm_node;
- memcpy(event->cm_info.rem_addr, cm_node->rem_addr, sizeof(event->cm_info.rem_addr));
- memcpy(event->cm_info.loc_addr, cm_node->loc_addr, sizeof(event->cm_info.loc_addr));
- event->cm_info.rem_port = cm_node->rem_port;
- event->cm_info.loc_port = cm_node->loc_port;
- event->cm_info.cm_id = cm_node->cm_id;
-
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "node=%p event=%p type=%u dst=%pI4 src=%pI4\n",
- cm_node,
- event,
- type,
- event->cm_info.loc_addr,
- event->cm_info.rem_addr);
-
- i40iw_cm_post_event(event);
- return event;
-}
-
-/**
- * i40iw_free_retrans_entry - free send entry
- * @cm_node: connection's node
- */
-static void i40iw_free_retrans_entry(struct i40iw_cm_node *cm_node)
-{
- struct i40iw_device *iwdev = cm_node->iwdev;
- struct i40iw_timer_entry *send_entry;
-
- send_entry = cm_node->send_entry;
- if (send_entry) {
- cm_node->send_entry = NULL;
- i40iw_free_sqbuf(&iwdev->vsi, (void *)send_entry->sqbuf);
- kfree(send_entry);
- atomic_dec(&cm_node->ref_count);
- }
-}
-
-/**
- * i40iw_cleanup_retrans_entry - free send entry with lock
- * @cm_node: connection's node
- */
-static void i40iw_cleanup_retrans_entry(struct i40iw_cm_node *cm_node)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
- i40iw_free_retrans_entry(cm_node);
- spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
-}
-
-/**
- * i40iw_form_cm_frame - get a free packet and build frame
- * @cm_node: connection's node ionfo to use in frame
- * @options: pointer to options info
- * @hdr: pointer mpa header
- * @pdata: pointer to private data
- * @flags: indicates FIN or ACK
- */
-static struct i40iw_puda_buf *i40iw_form_cm_frame(struct i40iw_cm_node *cm_node,
- struct i40iw_kmem_info *options,
- struct i40iw_kmem_info *hdr,
- struct i40iw_kmem_info *pdata,
- u8 flags)
-{
- struct i40iw_puda_buf *sqbuf;
- struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi;
- u8 *buf;
-
- struct tcphdr *tcph;
- struct iphdr *iph;
- struct ipv6hdr *ip6h;
- struct ethhdr *ethh;
- u16 packetsize;
- u16 eth_hlen = ETH_HLEN;
- u32 opts_len = 0;
- u32 pd_len = 0;
- u32 hdr_len = 0;
- u16 vtag;
-
- sqbuf = i40iw_puda_get_bufpool(vsi->ilq);
- if (!sqbuf)
- return NULL;
- buf = sqbuf->mem.va;
-
- if (options)
- opts_len = (u32)options->size;
-
- if (hdr)
- hdr_len = hdr->size;
-
- if (pdata)
- pd_len = pdata->size;
-
- if (cm_node->vlan_id <= VLAN_VID_MASK)
- eth_hlen += 4;
-
- if (cm_node->ipv4)
- packetsize = sizeof(*iph) + sizeof(*tcph);
- else
- packetsize = sizeof(*ip6h) + sizeof(*tcph);
- packetsize += opts_len + hdr_len + pd_len;
-
- memset(buf, 0x00, eth_hlen + packetsize);
-
- sqbuf->totallen = packetsize + eth_hlen;
- sqbuf->maclen = eth_hlen;
- sqbuf->tcphlen = sizeof(*tcph) + opts_len;
- sqbuf->scratch = (void *)cm_node;
-
- ethh = (struct ethhdr *)buf;
- buf += eth_hlen;
-
- if (cm_node->ipv4) {
- sqbuf->ipv4 = true;
-
- iph = (struct iphdr *)buf;
- buf += sizeof(*iph);
- tcph = (struct tcphdr *)buf;
- buf += sizeof(*tcph);
-
- ether_addr_copy(ethh->h_dest, cm_node->rem_mac);
- ether_addr_copy(ethh->h_source, cm_node->loc_mac);
- if (cm_node->vlan_id <= VLAN_VID_MASK) {
- ((struct vlan_ethhdr *)ethh)->h_vlan_proto = htons(ETH_P_8021Q);
- vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | cm_node->vlan_id;
- ((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
-
- ((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto = htons(ETH_P_IP);
- } else {
- ethh->h_proto = htons(ETH_P_IP);
- }
-
- iph->version = IPVERSION;
- iph->ihl = 5; /* 5 * 4Byte words, IP headr len */
- iph->tos = cm_node->tos;
- iph->tot_len = htons(packetsize);
- iph->id = htons(++cm_node->tcp_cntxt.loc_id);
-
- iph->frag_off = htons(0x4000);
- iph->ttl = 0x40;
- iph->protocol = IPPROTO_TCP;
- iph->saddr = htonl(cm_node->loc_addr[0]);
- iph->daddr = htonl(cm_node->rem_addr[0]);
- } else {
- sqbuf->ipv4 = false;
- ip6h = (struct ipv6hdr *)buf;
- buf += sizeof(*ip6h);
- tcph = (struct tcphdr *)buf;
- buf += sizeof(*tcph);
-
- ether_addr_copy(ethh->h_dest, cm_node->rem_mac);
- ether_addr_copy(ethh->h_source, cm_node->loc_mac);
- if (cm_node->vlan_id <= VLAN_VID_MASK) {
- ((struct vlan_ethhdr *)ethh)->h_vlan_proto = htons(ETH_P_8021Q);
- vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | cm_node->vlan_id;
- ((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
- ((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto = htons(ETH_P_IPV6);
- } else {
- ethh->h_proto = htons(ETH_P_IPV6);
- }
- ip6h->version = 6;
- ip6h->priority = cm_node->tos >> 4;
- ip6h->flow_lbl[0] = cm_node->tos << 4;
- ip6h->flow_lbl[1] = 0;
- ip6h->flow_lbl[2] = 0;
- ip6h->payload_len = htons(packetsize - sizeof(*ip6h));
- ip6h->nexthdr = 6;
- ip6h->hop_limit = 128;
- i40iw_copy_ip_htonl(ip6h->saddr.in6_u.u6_addr32,
- cm_node->loc_addr);
- i40iw_copy_ip_htonl(ip6h->daddr.in6_u.u6_addr32,
- cm_node->rem_addr);
- }
-
- tcph->source = htons(cm_node->loc_port);
- tcph->dest = htons(cm_node->rem_port);
-
- tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
-
- if (flags & SET_ACK) {
- cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
- tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
- tcph->ack = 1;
- } else {
- tcph->ack_seq = 0;
- }
-
- if (flags & SET_SYN) {
- cm_node->tcp_cntxt.loc_seq_num++;
- tcph->syn = 1;
- } else {
- cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;
- }
-
- if (flags & SET_FIN) {
- cm_node->tcp_cntxt.loc_seq_num++;
- tcph->fin = 1;
- }
-
- if (flags & SET_RST)
- tcph->rst = 1;
-
- tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);
- sqbuf->tcphlen = tcph->doff << 2;
- tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);
- tcph->urg_ptr = 0;
-
- if (opts_len) {
- memcpy(buf, options->addr, opts_len);
- buf += opts_len;
- }
-
- if (hdr_len) {
- memcpy(buf, hdr->addr, hdr_len);
- buf += hdr_len;
- }
-
- if (pdata && pdata->addr)
- memcpy(buf, pdata->addr, pdata->size);
-
- atomic_set(&sqbuf->refcount, 1);
-
- return sqbuf;
-}
-
-/**
- * i40iw_send_reset - Send RST packet
- * @cm_node: connection's node
- */
-int i40iw_send_reset(struct i40iw_cm_node *cm_node)
-{
- struct i40iw_puda_buf *sqbuf;
- int flags = SET_RST | SET_ACK;
-
- sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, flags);
- if (!sqbuf) {
- i40iw_pr_err("no sqbuf\n");
- return -1;
- }
-
- return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 0, 1);
-}
-
-/**
- * i40iw_active_open_err - send event for active side cm error
- * @cm_node: connection's node
- * @reset: Flag to send reset or not
- */
-static void i40iw_active_open_err(struct i40iw_cm_node *cm_node, bool reset)
-{
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->cm_core->stats_connect_errs++;
- if (reset) {
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "%s cm_node=%p state=%d\n",
- __func__,
- cm_node,
- cm_node->state);
- atomic_inc(&cm_node->ref_count);
- i40iw_send_reset(cm_node);
- }
-
- cm_node->state = I40IW_CM_STATE_CLOSED;
- i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED);
-}
-
-/**
- * i40iw_passive_open_err - handle passive side cm error
- * @cm_node: connection's node
- * @reset: send reset or just free cm_node
- */
-static void i40iw_passive_open_err(struct i40iw_cm_node *cm_node, bool reset)
-{
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->cm_core->stats_passive_errs++;
- cm_node->state = I40IW_CM_STATE_CLOSED;
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "%s cm_node=%p state =%d\n",
- __func__,
- cm_node,
- cm_node->state);
- if (reset)
- i40iw_send_reset(cm_node);
- else
- i40iw_rem_ref_cm_node(cm_node);
-}
-
-/**
- * i40iw_event_connect_error - to create connect error event
- * @event: cm information for connect event
- */
-static void i40iw_event_connect_error(struct i40iw_cm_event *event)
-{
- struct i40iw_qp *iwqp;
- struct iw_cm_id *cm_id;
-
- cm_id = event->cm_node->cm_id;
- if (!cm_id)
- return;
-
- iwqp = cm_id->provider_data;
-
- if (!iwqp || !iwqp->iwdev)
- return;
-
- iwqp->cm_id = NULL;
- cm_id->provider_data = NULL;
- i40iw_send_cm_event(event->cm_node, cm_id,
- IW_CM_EVENT_CONNECT_REPLY,
- -ECONNRESET);
- cm_id->rem_ref(cm_id);
- i40iw_rem_ref_cm_node(event->cm_node);
-}
-
-/**
- * i40iw_process_options
- * @cm_node: connection's node
- * @optionsloc: point to start of options
- * @optionsize: size of all options
- * @syn_packet: flag if syn packet
- */
-static int i40iw_process_options(struct i40iw_cm_node *cm_node,
- u8 *optionsloc,
- u32 optionsize,
- u32 syn_packet)
-{
- u32 tmp;
- u32 offset = 0;
- union all_known_options *all_options;
- char got_mss_option = 0;
-
- while (offset < optionsize) {
- all_options = (union all_known_options *)(optionsloc + offset);
- switch (all_options->as_base.optionnum) {
- case OPTION_NUMBER_END:
- offset = optionsize;
- break;
- case OPTION_NUMBER_NONE:
- offset += 1;
- continue;
- case OPTION_NUMBER_MSS:
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "%s: MSS Length: %d Offset: %d Size: %d\n",
- __func__,
- all_options->as_mss.length,
- offset,
- optionsize);
- got_mss_option = 1;
- if (all_options->as_mss.length != 4)
- return -1;
- tmp = ntohs(all_options->as_mss.mss);
- if (tmp > 0 && tmp < cm_node->tcp_cntxt.mss)
- cm_node->tcp_cntxt.mss = tmp;
- break;
- case OPTION_NUMBER_WINDOW_SCALE:
- cm_node->tcp_cntxt.snd_wscale =
- all_options->as_windowscale.shiftcount;
- break;
- default:
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "TCP Option not understood: %x\n",
- all_options->as_base.optionnum);
- break;
- }
- offset += all_options->as_base.length;
- }
- if (!got_mss_option && syn_packet)
- cm_node->tcp_cntxt.mss = I40IW_CM_DEFAULT_MSS;
- return 0;
-}
-
-/**
- * i40iw_handle_tcp_options -
- * @cm_node: connection's node
- * @tcph: pointer tcp header
- * @optionsize: size of options rcvd
- * @passive: active or passive flag
- */
-static int i40iw_handle_tcp_options(struct i40iw_cm_node *cm_node,
- struct tcphdr *tcph,
- int optionsize,
- int passive)
-{
- u8 *optionsloc = (u8 *)&tcph[1];
-
- if (optionsize) {
- if (i40iw_process_options(cm_node,
- optionsloc,
- optionsize,
- (u32)tcph->syn)) {
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "%s: Node %p, Sending RESET\n",
- __func__,
- cm_node);
- if (passive)
- i40iw_passive_open_err(cm_node, true);
- else
- i40iw_active_open_err(cm_node, true);
- return -1;
- }
- }
-
- cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->window) <<
- cm_node->tcp_cntxt.snd_wscale;
-
- if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd)
- cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd;
- return 0;
-}
-
-/**
- * i40iw_build_mpa_v1 - build a MPA V1 frame
- * @cm_node: connection's node
- * @mpa_key: to do read0 or write0
- */
-static void i40iw_build_mpa_v1(struct i40iw_cm_node *cm_node,
- void *start_addr,
- u8 mpa_key)
-{
- struct ietf_mpa_v1 *mpa_frame = (struct ietf_mpa_v1 *)start_addr;
-
- switch (mpa_key) {
- case MPA_KEY_REQUEST:
- memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);
- break;
- case MPA_KEY_REPLY:
- memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);
- break;
- default:
- break;
- }
- mpa_frame->flags = IETF_MPA_FLAGS_CRC;
- mpa_frame->rev = cm_node->mpa_frame_rev;
- mpa_frame->priv_data_len = htons(cm_node->pdata.size);
-}
-
-/**
- * i40iw_build_mpa_v2 - build a MPA V2 frame
- * @cm_node: connection's node
- * @start_addr: buffer start address
- * @mpa_key: to do read0 or write0
- */
-static void i40iw_build_mpa_v2(struct i40iw_cm_node *cm_node,
- void *start_addr,
- u8 mpa_key)
-{
- struct ietf_mpa_v2 *mpa_frame = (struct ietf_mpa_v2 *)start_addr;
- struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg;
- u16 ctrl_ird, ctrl_ord;
-
- /* initialize the upper 5 bytes of the frame */
- i40iw_build_mpa_v1(cm_node, start_addr, mpa_key);
- mpa_frame->flags |= IETF_MPA_V2_FLAG;
- mpa_frame->priv_data_len += htons(IETF_RTR_MSG_SIZE);
-
- /* initialize RTR msg */
- if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) {
- ctrl_ird = IETF_NO_IRD_ORD;
- ctrl_ord = IETF_NO_IRD_ORD;
- } else {
- ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ?
- IETF_NO_IRD_ORD : cm_node->ird_size;
- ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ?
- IETF_NO_IRD_ORD : cm_node->ord_size;
- }
-
- ctrl_ird |= IETF_PEER_TO_PEER;
-
- switch (mpa_key) {
- case MPA_KEY_REQUEST:
- ctrl_ord |= IETF_RDMA0_WRITE;
- ctrl_ord |= IETF_RDMA0_READ;
- break;
- case MPA_KEY_REPLY:
- switch (cm_node->send_rdma0_op) {
- case SEND_RDMA_WRITE_ZERO:
- ctrl_ord |= IETF_RDMA0_WRITE;
- break;
- case SEND_RDMA_READ_ZERO:
- ctrl_ord |= IETF_RDMA0_READ;
- break;
- }
- break;
- default:
- break;
- }
- rtr_msg->ctrl_ird = htons(ctrl_ird);
- rtr_msg->ctrl_ord = htons(ctrl_ord);
-}
-
-/**
- * i40iw_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2
- * @cm_node: connection's node
- * @mpa: mpa: data buffer
- * @mpa_key: to do read0 or write0
- */
-static int i40iw_cm_build_mpa_frame(struct i40iw_cm_node *cm_node,
- struct i40iw_kmem_info *mpa,
- u8 mpa_key)
-{
- int hdr_len = 0;
-
- switch (cm_node->mpa_frame_rev) {
- case IETF_MPA_V1:
- hdr_len = sizeof(struct ietf_mpa_v1);
- i40iw_build_mpa_v1(cm_node, mpa->addr, mpa_key);
- break;
- case IETF_MPA_V2:
- hdr_len = sizeof(struct ietf_mpa_v2);
- i40iw_build_mpa_v2(cm_node, mpa->addr, mpa_key);
- break;
- default:
- break;
- }
-
- return hdr_len;
-}
-
-/**
- * i40iw_send_mpa_request - active node send mpa request to passive node
- * @cm_node: connection's node
- */
-static int i40iw_send_mpa_request(struct i40iw_cm_node *cm_node)
-{
- struct i40iw_puda_buf *sqbuf;
-
- if (!cm_node) {
- i40iw_pr_err("cm_node == NULL\n");
- return -1;
- }
-
- cm_node->mpa_hdr.addr = &cm_node->mpa_frame;
- cm_node->mpa_hdr.size = i40iw_cm_build_mpa_frame(cm_node,
- &cm_node->mpa_hdr,
- MPA_KEY_REQUEST);
- if (!cm_node->mpa_hdr.size) {
- i40iw_pr_err("mpa size = %d\n", cm_node->mpa_hdr.size);
- return -1;
- }
-
- sqbuf = i40iw_form_cm_frame(cm_node,
- NULL,
- &cm_node->mpa_hdr,
- &cm_node->pdata,
- SET_ACK);
- if (!sqbuf) {
- i40iw_pr_err("sq_buf == NULL\n");
- return -1;
- }
- return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0);
-}
-
-/**
- * i40iw_send_mpa_reject -
- * @cm_node: connection's node
- * @pdata: reject data for connection
- * @plen: length of reject data
- */
-static int i40iw_send_mpa_reject(struct i40iw_cm_node *cm_node,
- const void *pdata,
- u8 plen)
-{
- struct i40iw_puda_buf *sqbuf;
- struct i40iw_kmem_info priv_info;
-
- cm_node->mpa_hdr.addr = &cm_node->mpa_frame;
- cm_node->mpa_hdr.size = i40iw_cm_build_mpa_frame(cm_node,
- &cm_node->mpa_hdr,
- MPA_KEY_REPLY);
-
- cm_node->mpa_frame.flags |= IETF_MPA_FLAGS_REJECT;
- priv_info.addr = (void *)pdata;
- priv_info.size = plen;
-
- sqbuf = i40iw_form_cm_frame(cm_node,
- NULL,
- &cm_node->mpa_hdr,
- &priv_info,
- SET_ACK | SET_FIN);
- if (!sqbuf) {
- i40iw_pr_err("no sqbuf\n");
- return -ENOMEM;
- }
- cm_node->state = I40IW_CM_STATE_FIN_WAIT1;
- return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0);
-}
-
-/**
- * recv_mpa - process an IETF MPA frame
- * @cm_node: connection's node
- * @buffer: Data pointer
- * @type: to return accept or reject
- * @len: Len of mpa buffer
- */
-static int i40iw_parse_mpa(struct i40iw_cm_node *cm_node, u8 *buffer, u32 *type, u32 len)
-{
- struct ietf_mpa_v1 *mpa_frame;
- struct ietf_mpa_v2 *mpa_v2_frame;
- struct ietf_rtr_msg *rtr_msg;
- int mpa_hdr_len;
- int priv_data_len;
-
- *type = I40IW_MPA_REQUEST_ACCEPT;
-
- if (len < sizeof(struct ietf_mpa_v1)) {
- i40iw_pr_err("ietf buffer small (%x)\n", len);
- return -1;
- }
-
- mpa_frame = (struct ietf_mpa_v1 *)buffer;
- mpa_hdr_len = sizeof(struct ietf_mpa_v1);
- priv_data_len = ntohs(mpa_frame->priv_data_len);
-
- if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
- i40iw_pr_err("large pri_data %d\n", priv_data_len);
- return -1;
- }
- if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) {
- i40iw_pr_err("unsupported mpa rev = %d\n", mpa_frame->rev);
- return -1;
- }
- if (mpa_frame->rev > cm_node->mpa_frame_rev) {
- i40iw_pr_err("rev %d\n", mpa_frame->rev);
- return -1;
- }
- cm_node->mpa_frame_rev = mpa_frame->rev;
-
- if (cm_node->state != I40IW_CM_STATE_MPAREQ_SENT) {
- if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) {
- i40iw_pr_err("Unexpected MPA Key received\n");
- return -1;
- }
- } else {
- if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE)) {
- i40iw_pr_err("Unexpected MPA Key received\n");
- return -1;
- }
- }
-
- if (priv_data_len + mpa_hdr_len > len) {
- i40iw_pr_err("ietf buffer len(%x + %x != %x)\n",
- priv_data_len, mpa_hdr_len, len);
- return -1;
- }
- if (len > MAX_CM_BUFFER) {
- i40iw_pr_err("ietf buffer large len = %d\n", len);
- return -1;
- }
-
- switch (mpa_frame->rev) {
- case IETF_MPA_V2:{
- u16 ird_size;
- u16 ord_size;
- u16 ctrl_ord;
- u16 ctrl_ird;
-
- mpa_v2_frame = (struct ietf_mpa_v2 *)buffer;
- mpa_hdr_len += IETF_RTR_MSG_SIZE;
- rtr_msg = &mpa_v2_frame->rtr_msg;
-
- /* parse rtr message */
- ctrl_ord = ntohs(rtr_msg->ctrl_ord);
- ctrl_ird = ntohs(rtr_msg->ctrl_ird);
- ird_size = ctrl_ird & IETF_NO_IRD_ORD;
- ord_size = ctrl_ord & IETF_NO_IRD_ORD;
-
- if (!(ctrl_ird & IETF_PEER_TO_PEER))
- return -1;
-
- if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) {
- cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD;
- goto negotiate_done;
- }
-
- if (cm_node->state != I40IW_CM_STATE_MPAREQ_SENT) {
- /* responder */
- if (!ord_size && (ctrl_ord & IETF_RDMA0_READ))
- cm_node->ird_size = 1;
- if (cm_node->ord_size > ird_size)
- cm_node->ord_size = ird_size;
- } else {
- /* initiator */
- if (!ird_size && (ctrl_ord & IETF_RDMA0_READ))
- return -1;
- if (cm_node->ord_size > ird_size)
- cm_node->ord_size = ird_size;
-
- if (cm_node->ird_size < ord_size)
- /* no resources available */
- return -1;
- }
-
-negotiate_done:
- if (ctrl_ord & IETF_RDMA0_READ)
- cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
- else if (ctrl_ord & IETF_RDMA0_WRITE)
- cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO;
- else /* Not supported RDMA0 operation */
- return -1;
- i40iw_debug(cm_node->dev, I40IW_DEBUG_CM,
- "MPAV2: Negotiated ORD: %d, IRD: %d\n",
- cm_node->ord_size, cm_node->ird_size);
- break;
- }
- break;
- case IETF_MPA_V1:
- default:
- break;
- }
-
- memcpy(cm_node->pdata_buf, buffer + mpa_hdr_len, priv_data_len);
- cm_node->pdata.size = priv_data_len;
-
- if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT)
- *type = I40IW_MPA_REQUEST_REJECT;
-
- if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS)
- cm_node->snd_mark_en = true;
-
- return 0;
-}
-
-/**
- * i40iw_schedule_cm_timer
- * @@cm_node: connection's node
- * @sqbuf: buffer to send
- * @type: if it is send or close
- * @send_retrans: if rexmits to be done
- * @close_when_complete: is cm_node to be removed
- *
- * note - cm_node needs to be protected before calling this. Encase in:
- * i40iw_rem_ref_cm_node(cm_core, cm_node);
- * i40iw_schedule_cm_timer(...)
- * atomic_inc(&cm_node->ref_count);
- */
-int i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node,
- struct i40iw_puda_buf *sqbuf,
- enum i40iw_timer_type type,
- int send_retrans,
- int close_when_complete)
-{
- struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi;
- struct i40iw_cm_core *cm_core = cm_node->cm_core;
- struct i40iw_timer_entry *new_send;
- int ret = 0;
- u32 was_timer_set;
- unsigned long flags;
-
- new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);
- if (!new_send) {
- if (type != I40IW_TIMER_TYPE_CLOSE)
- i40iw_free_sqbuf(vsi, (void *)sqbuf);
- return -ENOMEM;
- }
- new_send->retrycount = I40IW_DEFAULT_RETRYS;
- new_send->retranscount = I40IW_DEFAULT_RETRANS;
- new_send->sqbuf = sqbuf;
- new_send->timetosend = jiffies;
- new_send->type = type;
- new_send->send_retrans = send_retrans;
- new_send->close_when_complete = close_when_complete;
-
- if (type == I40IW_TIMER_TYPE_CLOSE) {
- new_send->timetosend += (HZ / 10);
- if (cm_node->close_entry) {
- kfree(new_send);
- i40iw_pr_err("already close entry\n");
- return -EINVAL;
- }
- cm_node->close_entry = new_send;
- }
-
- if (type == I40IW_TIMER_TYPE_SEND) {
- spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
- cm_node->send_entry = new_send;
- atomic_inc(&cm_node->ref_count);
- spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
- new_send->timetosend = jiffies + I40IW_RETRY_TIMEOUT;
-
- atomic_inc(&sqbuf->refcount);
- i40iw_puda_send_buf(vsi->ilq, sqbuf);
- if (!send_retrans) {
- i40iw_cleanup_retrans_entry(cm_node);
- if (close_when_complete)
- i40iw_rem_ref_cm_node(cm_node);
- return ret;
- }
- }
-
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- was_timer_set = timer_pending(&cm_core->tcp_timer);
-
- if (!was_timer_set) {
- cm_core->tcp_timer.expires = new_send->timetosend;
- add_timer(&cm_core->tcp_timer);
- }
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-
- return ret;
-}
-
-/**
- * i40iw_retrans_expired - Could not rexmit the packet
- * @cm_node: connection's node
- */
-static void i40iw_retrans_expired(struct i40iw_cm_node *cm_node)
-{
- struct iw_cm_id *cm_id = cm_node->cm_id;
- enum i40iw_cm_node_state state = cm_node->state;
-
- cm_node->state = I40IW_CM_STATE_CLOSED;
- switch (state) {
- case I40IW_CM_STATE_SYN_RCVD:
- case I40IW_CM_STATE_CLOSING:
- i40iw_rem_ref_cm_node(cm_node);
- break;
- case I40IW_CM_STATE_FIN_WAIT1:
- case I40IW_CM_STATE_LAST_ACK:
- if (cm_node->cm_id)
- cm_id->rem_ref(cm_id);
- i40iw_send_reset(cm_node);
- break;
- default:
- atomic_inc(&cm_node->ref_count);
- i40iw_send_reset(cm_node);
- i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED);
- break;
- }
-}
-
-/**
- * i40iw_handle_close_entry - for handling retry/timeouts
- * @cm_node: connection's node
- * @rem_node: flag for remove cm_node
- */
-static void i40iw_handle_close_entry(struct i40iw_cm_node *cm_node, u32 rem_node)
-{
- struct i40iw_timer_entry *close_entry = cm_node->close_entry;
- struct iw_cm_id *cm_id = cm_node->cm_id;
- struct i40iw_qp *iwqp;
- unsigned long flags;
-
- if (!close_entry)
- return;
- iwqp = (struct i40iw_qp *)close_entry->sqbuf;
- if (iwqp) {
- spin_lock_irqsave(&iwqp->lock, flags);
- if (iwqp->cm_id) {
- iwqp->hw_tcp_state = I40IW_TCP_STATE_CLOSED;
- iwqp->hw_iwarp_state = I40IW_QP_STATE_ERROR;
- iwqp->last_aeq = I40IW_AE_RESET_SENT;
- iwqp->ibqp_state = IB_QPS_ERR;
- spin_unlock_irqrestore(&iwqp->lock, flags);
- i40iw_cm_disconn(iwqp);
- } else {
- spin_unlock_irqrestore(&iwqp->lock, flags);
- }
- } else if (rem_node) {
- /* TIME_WAIT state */
- i40iw_rem_ref_cm_node(cm_node);
- }
- if (cm_id)
- cm_id->rem_ref(cm_id);
- kfree(close_entry);
- cm_node->close_entry = NULL;
-}
-
-/**
- * i40iw_build_timer_list - Add cm_nodes to timer list
- * @timer_list: ptr to timer list
- * @hte: ptr to accelerated or non-accelerated list
- */
-static void i40iw_build_timer_list(struct list_head *timer_list,
- struct list_head *hte)
-{
- struct i40iw_cm_node *cm_node;
- struct list_head *list_core_temp, *list_node;
-
- list_for_each_safe(list_node, list_core_temp, hte) {
- cm_node = container_of(list_node, struct i40iw_cm_node, list);
- if (cm_node->close_entry || cm_node->send_entry) {
- atomic_inc(&cm_node->ref_count);
- list_add(&cm_node->timer_entry, timer_list);
- }
- }
-}
-
-/**
- * i40iw_cm_timer_tick - system's timer expired callback
- * @pass: Pointing to cm_core
- */
-static void i40iw_cm_timer_tick(struct timer_list *t)
-{
- unsigned long nexttimeout = jiffies + I40IW_LONG_TIME;
- struct i40iw_cm_node *cm_node;
- struct i40iw_timer_entry *send_entry, *close_entry;
- struct list_head *list_core_temp;
- struct i40iw_sc_vsi *vsi;
- struct list_head *list_node;
- struct i40iw_cm_core *cm_core = from_timer(cm_core, t, tcp_timer);
- u32 settimer = 0;
- unsigned long timetosend;
- unsigned long flags;
-
- struct list_head timer_list;
-
- INIT_LIST_HEAD(&timer_list);
-
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- i40iw_build_timer_list(&timer_list, &cm_core->non_accelerated_list);
- i40iw_build_timer_list(&timer_list, &cm_core->accelerated_list);
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-
- list_for_each_safe(list_node, list_core_temp, &timer_list) {
- cm_node = container_of(list_node,
- struct i40iw_cm_node,
- timer_entry);
- close_entry = cm_node->close_entry;
-
- if (close_entry) {
- if (time_after(close_entry->timetosend, jiffies)) {
- if (nexttimeout > close_entry->timetosend ||
- !settimer) {
- nexttimeout = close_entry->timetosend;
- settimer = 1;
- }
- } else {
- i40iw_handle_close_entry(cm_node, 1);
- }
- }
-
- spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
-
- send_entry = cm_node->send_entry;
- if (!send_entry)
- goto done;
- if (time_after(send_entry->timetosend, jiffies)) {
- if (cm_node->state != I40IW_CM_STATE_OFFLOADED) {
- if ((nexttimeout > send_entry->timetosend) ||
- !settimer) {
- nexttimeout = send_entry->timetosend;
- settimer = 1;
- }
- } else {
- i40iw_free_retrans_entry(cm_node);
- }
- goto done;
- }
-
- if ((cm_node->state == I40IW_CM_STATE_OFFLOADED) ||
- (cm_node->state == I40IW_CM_STATE_CLOSED)) {
- i40iw_free_retrans_entry(cm_node);
- goto done;
- }
-
- if (!send_entry->retranscount || !send_entry->retrycount) {
- i40iw_free_retrans_entry(cm_node);
-
- spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
- i40iw_retrans_expired(cm_node);
- cm_node->state = I40IW_CM_STATE_CLOSED;
- spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
- goto done;
- }
- spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
-
- vsi = &cm_node->iwdev->vsi;
-
- if (!cm_node->ack_rcvd) {
- atomic_inc(&send_entry->sqbuf->refcount);
- i40iw_puda_send_buf(vsi->ilq, send_entry->sqbuf);
- cm_node->cm_core->stats_pkt_retrans++;
- }
- spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
- if (send_entry->send_retrans) {
- send_entry->retranscount--;
- timetosend = (I40IW_RETRY_TIMEOUT <<
- (I40IW_DEFAULT_RETRANS -
- send_entry->retranscount));
-
- send_entry->timetosend = jiffies +
- min(timetosend, I40IW_MAX_TIMEOUT);
- if (nexttimeout > send_entry->timetosend || !settimer) {
- nexttimeout = send_entry->timetosend;
- settimer = 1;
- }
- } else {
- int close_when_complete;
-
- close_when_complete = send_entry->close_when_complete;
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "cm_node=%p state=%d\n",
- cm_node,
- cm_node->state);
- i40iw_free_retrans_entry(cm_node);
- if (close_when_complete)
- i40iw_rem_ref_cm_node(cm_node);
- }
-done:
- spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
- i40iw_rem_ref_cm_node(cm_node);
- }
-
- if (settimer) {
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- if (!timer_pending(&cm_core->tcp_timer)) {
- cm_core->tcp_timer.expires = nexttimeout;
- add_timer(&cm_core->tcp_timer);
- }
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
- }
-}
-
-/**
- * i40iw_send_syn - send SYN packet
- * @cm_node: connection's node
- * @sendack: flag to set ACK bit or not
- */
-int i40iw_send_syn(struct i40iw_cm_node *cm_node, u32 sendack)
-{
- struct i40iw_puda_buf *sqbuf;
- int flags = SET_SYN;
- char optionsbuffer[sizeof(struct option_mss) +
- sizeof(struct option_windowscale) +
- sizeof(struct option_base) + TCP_OPTIONS_PADDING];
- struct i40iw_kmem_info opts;
-
- int optionssize = 0;
- /* Sending MSS option */
- union all_known_options *options;
-
- opts.addr = optionsbuffer;
- if (!cm_node) {
- i40iw_pr_err("no cm_node\n");
- return -EINVAL;
- }
-
- options = (union all_known_options *)&optionsbuffer[optionssize];
- options->as_mss.optionnum = OPTION_NUMBER_MSS;
- options->as_mss.length = sizeof(struct option_mss);
- options->as_mss.mss = htons(cm_node->tcp_cntxt.mss);
- optionssize += sizeof(struct option_mss);
-
- options = (union all_known_options *)&optionsbuffer[optionssize];
- options->as_windowscale.optionnum = OPTION_NUMBER_WINDOW_SCALE;
- options->as_windowscale.length = sizeof(struct option_windowscale);
- options->as_windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale;
- optionssize += sizeof(struct option_windowscale);
- options = (union all_known_options *)&optionsbuffer[optionssize];
- options->as_end = OPTION_NUMBER_END;
- optionssize += 1;
-
- if (sendack)
- flags |= SET_ACK;
-
- opts.size = optionssize;
-
- sqbuf = i40iw_form_cm_frame(cm_node, &opts, NULL, NULL, flags);
- if (!sqbuf) {
- i40iw_pr_err("no sqbuf\n");
- return -1;
- }
- return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0);
-}
-
-/**
- * i40iw_send_ack - Send ACK packet
- * @cm_node: connection's node
- */
-static void i40iw_send_ack(struct i40iw_cm_node *cm_node)
-{
- struct i40iw_puda_buf *sqbuf;
- struct i40iw_sc_vsi *vsi = &cm_node->iwdev->vsi;
-
- sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK);
- if (sqbuf)
- i40iw_puda_send_buf(vsi->ilq, sqbuf);
- else
- i40iw_pr_err("no sqbuf\n");
-}
-
-/**
- * i40iw_send_fin - Send FIN pkt
- * @cm_node: connection's node
- */
-static int i40iw_send_fin(struct i40iw_cm_node *cm_node)
-{
- struct i40iw_puda_buf *sqbuf;
-
- sqbuf = i40iw_form_cm_frame(cm_node, NULL, NULL, NULL, SET_ACK | SET_FIN);
- if (!sqbuf) {
- i40iw_pr_err("no sqbuf\n");
- return -1;
- }
- return i40iw_schedule_cm_timer(cm_node, sqbuf, I40IW_TIMER_TYPE_SEND, 1, 0);
-}
-
-/**
- * i40iw_find_node - find a cm node that matches the reference cm node
- * @cm_core: cm's core
- * @rem_port: remote tcp port num
- * @rem_addr: remote ip addr
- * @loc_port: local tcp port num
- * @loc_addr: loc ip addr
- * @add_refcnt: flag to increment refcount of cm_node
- * @accelerated_list: flag for accelerated vs non-accelerated list to search
- */
-struct i40iw_cm_node *i40iw_find_node(struct i40iw_cm_core *cm_core,
- u16 rem_port,
- u32 *rem_addr,
- u16 loc_port,
- u32 *loc_addr,
- bool add_refcnt,
- bool accelerated_list)
-{
- struct list_head *hte;
- struct i40iw_cm_node *cm_node;
- unsigned long flags;
-
- hte = accelerated_list ?
- &cm_core->accelerated_list : &cm_core->non_accelerated_list;
-
- /* walk list and find cm_node associated with this session ID */
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- list_for_each_entry(cm_node, hte, list) {
- if (!memcmp(cm_node->loc_addr, loc_addr, sizeof(cm_node->loc_addr)) &&
- (cm_node->loc_port == loc_port) &&
- !memcmp(cm_node->rem_addr, rem_addr, sizeof(cm_node->rem_addr)) &&
- (cm_node->rem_port == rem_port)) {
- if (add_refcnt)
- atomic_inc(&cm_node->ref_count);
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
- return cm_node;
- }
- }
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-
- /* no owner node */
- return NULL;
-}
-
-/**
- * i40iw_find_listener - find a cm node listening on this addr-port pair
- * @cm_core: cm's core
- * @dst_port: listener tcp port num
- * @dst_addr: listener ip addr
- * @listener_state: state to match with listen node's
- */
-static struct i40iw_cm_listener *i40iw_find_listener(
- struct i40iw_cm_core *cm_core,
- u32 *dst_addr,
- u16 dst_port,
- u16 vlan_id,
- enum i40iw_cm_listener_state
- listener_state)
-{
- struct i40iw_cm_listener *listen_node;
- static const u32 ip_zero[4] = { 0, 0, 0, 0 };
- u32 listen_addr[4];
- u16 listen_port;
- unsigned long flags;
-
- /* walk list and find cm_node associated with this session ID */
- spin_lock_irqsave(&cm_core->listen_list_lock, flags);
- list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
- memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr));
- listen_port = listen_node->loc_port;
- /* compare node pair, return node handle if a match */
- if ((!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) ||
- !memcmp(listen_addr, ip_zero, sizeof(listen_addr))) &&
- (listen_port == dst_port) &&
- (listener_state & listen_node->listener_state)) {
- atomic_inc(&listen_node->ref_count);
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
- return listen_node;
- }
- }
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
- return NULL;
-}
-
-/**
- * i40iw_add_hte_node - add a cm node to the hash table
- * @cm_core: cm's core
- * @cm_node: connection's node
- */
-static void i40iw_add_hte_node(struct i40iw_cm_core *cm_core,
- struct i40iw_cm_node *cm_node)
-{
- unsigned long flags;
-
- if (!cm_node || !cm_core) {
- i40iw_pr_err("cm_node or cm_core == NULL\n");
- return;
- }
-
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- list_add_tail(&cm_node->list, &cm_core->non_accelerated_list);
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-}
-
-/**
- * i40iw_find_port - find port that matches reference port
- * @hte: ptr to accelerated or non-accelerated list
- * @accelerated_list: flag for accelerated vs non-accelerated list
- */
-static bool i40iw_find_port(struct list_head *hte, u16 port)
-{
- struct i40iw_cm_node *cm_node;
-
- list_for_each_entry(cm_node, hte, list) {
- if (cm_node->loc_port == port)
- return true;
- }
- return false;
-}
-
-/**
- * i40iw_port_in_use - determine if port is in use
- * @cm_core: cm's core
- * @port: port number
- */
-bool i40iw_port_in_use(struct i40iw_cm_core *cm_core, u16 port)
-{
- struct i40iw_cm_listener *listen_node;
- unsigned long flags;
-
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- if (i40iw_find_port(&cm_core->accelerated_list, port) ||
- i40iw_find_port(&cm_core->non_accelerated_list, port)) {
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
- return true;
- }
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-
- spin_lock_irqsave(&cm_core->listen_list_lock, flags);
- list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
- if (listen_node->loc_port == port) {
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
- return true;
- }
- }
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
-
- return false;
-}
-
-/**
- * i40iw_del_multiple_qhash - Remove qhash and child listens
- * @iwdev: iWarp device
- * @cm_info: CM info for parent listen node
- * @cm_parent_listen_node: The parent listen node
- */
-static enum i40iw_status_code i40iw_del_multiple_qhash(
- struct i40iw_device *iwdev,
- struct i40iw_cm_info *cm_info,
- struct i40iw_cm_listener *cm_parent_listen_node)
-{
- struct i40iw_cm_listener *child_listen_node;
- enum i40iw_status_code ret = I40IW_ERR_CONFIG;
- struct list_head *pos, *tpos;
- unsigned long flags;
-
- spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
- list_for_each_safe(pos, tpos, &cm_parent_listen_node->child_listen_list) {
- child_listen_node = list_entry(pos, struct i40iw_cm_listener, child_listen_list);
- if (child_listen_node->ipv4)
- i40iw_debug(&iwdev->sc_dev,
- I40IW_DEBUG_CM,
- "removing child listen for IP=%pI4, port=%d, vlan=%d\n",
- child_listen_node->loc_addr,
- child_listen_node->loc_port,
- child_listen_node->vlan_id);
- else
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
- "removing child listen for IP=%pI6, port=%d, vlan=%d\n",
- child_listen_node->loc_addr,
- child_listen_node->loc_port,
- child_listen_node->vlan_id);
- list_del(pos);
- memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
- sizeof(cm_info->loc_addr));
- cm_info->vlan_id = child_listen_node->vlan_id;
- if (child_listen_node->qhash_set) {
- ret = i40iw_manage_qhash(iwdev, cm_info,
- I40IW_QHASH_TYPE_TCP_SYN,
- I40IW_QHASH_MANAGE_TYPE_DELETE,
- NULL, false);
- child_listen_node->qhash_set = false;
- } else {
- ret = I40IW_SUCCESS;
- }
- i40iw_debug(&iwdev->sc_dev,
- I40IW_DEBUG_CM,
- "freed pointer = %p\n",
- child_listen_node);
- kfree(child_listen_node);
- cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++;
- }
- spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
-
- return ret;
-}
-
-/**
- * i40iw_netdev_vlan_ipv6 - Gets the netdev and vlan
- * @addr: local IPv6 address
- * @vlan_id: vlan id for the given IPv6 address
- *
- * Returns the net_device of the IPv6 address and also sets the
- * vlan id for that address.
- */
-static struct net_device *i40iw_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id)
-{
- struct net_device *ip_dev = NULL;
- struct in6_addr laddr6;
-
- if (!IS_ENABLED(CONFIG_IPV6))
- return NULL;
- i40iw_copy_ip_htonl(laddr6.in6_u.u6_addr32, addr);
- if (vlan_id)
- *vlan_id = I40IW_NO_VLAN;
- rcu_read_lock();
- for_each_netdev_rcu(&init_net, ip_dev) {
- if (ipv6_chk_addr(&init_net, &laddr6, ip_dev, 1)) {
- if (vlan_id)
- *vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
- break;
- }
- }
- rcu_read_unlock();
- return ip_dev;
-}
-
-/**
- * i40iw_get_vlan_ipv4 - Returns the vlan_id for IPv4 address
- * @addr: local IPv4 address
- */
-static u16 i40iw_get_vlan_ipv4(u32 *addr)
-{
- struct net_device *netdev;
- u16 vlan_id = I40IW_NO_VLAN;
-
- netdev = ip_dev_find(&init_net, htonl(addr[0]));
- if (netdev) {
- vlan_id = rdma_vlan_dev_vlan_id(netdev);
- dev_put(netdev);
- }
- return vlan_id;
-}
-
-/**
- * i40iw_add_mqh_6 - Adds multiple qhashes for IPv6
- * @iwdev: iWarp device
- * @cm_info: CM info for parent listen node
- * @cm_parent_listen_node: The parent listen node
- *
- * Adds a qhash and a child listen node for every IPv6 address
- * on the adapter and adds the associated qhash filter
- */
-static enum i40iw_status_code i40iw_add_mqh_6(struct i40iw_device *iwdev,
- struct i40iw_cm_info *cm_info,
- struct i40iw_cm_listener *cm_parent_listen_node)
-{
- struct net_device *ip_dev;
- struct inet6_dev *idev;
- struct inet6_ifaddr *ifp, *tmp;
- enum i40iw_status_code ret = 0;
- struct i40iw_cm_listener *child_listen_node;
- unsigned long flags;
-
- rtnl_lock();
- for_each_netdev(&init_net, ip_dev) {
- if ((((rdma_vlan_dev_vlan_id(ip_dev) < I40IW_NO_VLAN) &&
- (rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev)) ||
- (ip_dev == iwdev->netdev)) && (ip_dev->flags & IFF_UP)) {
- idev = __in6_dev_get(ip_dev);
- if (!idev) {
- i40iw_pr_err("idev == NULL\n");
- break;
- }
- list_for_each_entry_safe(ifp, tmp, &idev->addr_list, if_list) {
- i40iw_debug(&iwdev->sc_dev,
- I40IW_DEBUG_CM,
- "IP=%pI6, vlan_id=%d, MAC=%pM\n",
- &ifp->addr,
- rdma_vlan_dev_vlan_id(ip_dev),
- ip_dev->dev_addr);
- child_listen_node =
- kzalloc(sizeof(*child_listen_node), GFP_ATOMIC);
- i40iw_debug(&iwdev->sc_dev,
- I40IW_DEBUG_CM,
- "Allocating child listener %p\n",
- child_listen_node);
- if (!child_listen_node) {
- ret = I40IW_ERR_NO_MEMORY;
- goto exit;
- }
- cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
- cm_parent_listen_node->vlan_id = cm_info->vlan_id;
-
- memcpy(child_listen_node, cm_parent_listen_node,
- sizeof(*child_listen_node));
-
- i40iw_copy_ip_ntohl(child_listen_node->loc_addr,
- ifp->addr.in6_u.u6_addr32);
- memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
- sizeof(cm_info->loc_addr));
-
- ret = i40iw_manage_qhash(iwdev, cm_info,
- I40IW_QHASH_TYPE_TCP_SYN,
- I40IW_QHASH_MANAGE_TYPE_ADD,
- NULL, true);
- if (!ret) {
- child_listen_node->qhash_set = true;
- spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
- list_add(&child_listen_node->child_listen_list,
- &cm_parent_listen_node->child_listen_list);
- spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
- cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
- } else {
- kfree(child_listen_node);
- }
- }
- }
- }
-exit:
- rtnl_unlock();
- return ret;
-}
-
-/**
- * i40iw_add_mqh_4 - Adds multiple qhashes for IPv4
- * @iwdev: iWarp device
- * @cm_info: CM info for parent listen node
- * @cm_parent_listen_node: The parent listen node
- *
- * Adds a qhash and a child listen node for every IPv4 address
- * on the adapter and adds the associated qhash filter
- */
-static enum i40iw_status_code i40iw_add_mqh_4(
- struct i40iw_device *iwdev,
- struct i40iw_cm_info *cm_info,
- struct i40iw_cm_listener *cm_parent_listen_node)
-{
- struct net_device *dev;
- struct in_device *idev;
- struct i40iw_cm_listener *child_listen_node;
- enum i40iw_status_code ret = 0;
- unsigned long flags;
-
- rtnl_lock();
- for_each_netdev(&init_net, dev) {
- if ((((rdma_vlan_dev_vlan_id(dev) < I40IW_NO_VLAN) &&
- (rdma_vlan_dev_real_dev(dev) == iwdev->netdev)) ||
- (dev == iwdev->netdev)) && (dev->flags & IFF_UP)) {
- const struct in_ifaddr *ifa;
-
- idev = in_dev_get(dev);
-
- in_dev_for_each_ifa_rtnl(ifa, idev) {
- i40iw_debug(&iwdev->sc_dev,
- I40IW_DEBUG_CM,
- "Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n",
- &ifa->ifa_address,
- rdma_vlan_dev_vlan_id(dev),
- dev->dev_addr);
- child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
- cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
- i40iw_debug(&iwdev->sc_dev,
- I40IW_DEBUG_CM,
- "Allocating child listener %p\n",
- child_listen_node);
- if (!child_listen_node) {
- in_dev_put(idev);
- ret = I40IW_ERR_NO_MEMORY;
- goto exit;
- }
- cm_info->vlan_id = rdma_vlan_dev_vlan_id(dev);
- cm_parent_listen_node->vlan_id = cm_info->vlan_id;
- memcpy(child_listen_node,
- cm_parent_listen_node,
- sizeof(*child_listen_node));
-
- child_listen_node->loc_addr[0] = ntohl(ifa->ifa_address);
- memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
- sizeof(cm_info->loc_addr));
-
- ret = i40iw_manage_qhash(iwdev,
- cm_info,
- I40IW_QHASH_TYPE_TCP_SYN,
- I40IW_QHASH_MANAGE_TYPE_ADD,
- NULL,
- true);
- if (!ret) {
- child_listen_node->qhash_set = true;
- spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
- list_add(&child_listen_node->child_listen_list,
- &cm_parent_listen_node->child_listen_list);
- spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
- } else {
- kfree(child_listen_node);
- cm_parent_listen_node->cm_core->stats_listen_nodes_created--;
- }
- }
-
- in_dev_put(idev);
- }
- }
-exit:
- rtnl_unlock();
- return ret;
-}
-
-/**
- * i40iw_dec_refcnt_listen - delete listener and associated cm nodes
- * @cm_core: cm's core
- * @free_hanging_nodes: to free associated cm_nodes
- * @apbvt_del: flag to delete the apbvt
- */
-static int i40iw_dec_refcnt_listen(struct i40iw_cm_core *cm_core,
- struct i40iw_cm_listener *listener,
- int free_hanging_nodes, bool apbvt_del)
-{
- int ret = -EINVAL;
- int err = 0;
- struct list_head *list_pos;
- struct list_head *list_temp;
- struct i40iw_cm_node *cm_node;
- struct list_head reset_list;
- struct i40iw_cm_info nfo;
- struct i40iw_cm_node *loopback;
- enum i40iw_cm_node_state old_state;
- unsigned long flags;
-
- /* free non-accelerated child nodes for this listener */
- INIT_LIST_HEAD(&reset_list);
- if (free_hanging_nodes) {
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- list_for_each_safe(list_pos,
- list_temp, &cm_core->non_accelerated_list) {
- cm_node = container_of(list_pos, struct i40iw_cm_node, list);
- if ((cm_node->listener == listener) &&
- !cm_node->accelerated) {
- atomic_inc(&cm_node->ref_count);
- list_add(&cm_node->reset_entry, &reset_list);
- }
- }
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
- }
-
- list_for_each_safe(list_pos, list_temp, &reset_list) {
- cm_node = container_of(list_pos, struct i40iw_cm_node, reset_entry);
- loopback = cm_node->loopbackpartner;
- if (cm_node->state >= I40IW_CM_STATE_FIN_WAIT1) {
- i40iw_rem_ref_cm_node(cm_node);
- } else {
- if (!loopback) {
- i40iw_cleanup_retrans_entry(cm_node);
- err = i40iw_send_reset(cm_node);
- if (err) {
- cm_node->state = I40IW_CM_STATE_CLOSED;
- i40iw_pr_err("send reset\n");
- } else {
- old_state = cm_node->state;
- cm_node->state = I40IW_CM_STATE_LISTENER_DESTROYED;
- if (old_state != I40IW_CM_STATE_MPAREQ_RCVD)
- i40iw_rem_ref_cm_node(cm_node);
- }
- } else {
- struct i40iw_cm_event event;
-
- event.cm_node = loopback;
- memcpy(event.cm_info.rem_addr,
- loopback->rem_addr, sizeof(event.cm_info.rem_addr));
- memcpy(event.cm_info.loc_addr,
- loopback->loc_addr, sizeof(event.cm_info.loc_addr));
- event.cm_info.rem_port = loopback->rem_port;
- event.cm_info.loc_port = loopback->loc_port;
- event.cm_info.cm_id = loopback->cm_id;
- event.cm_info.ipv4 = loopback->ipv4;
- atomic_inc(&loopback->ref_count);
- loopback->state = I40IW_CM_STATE_CLOSED;
- i40iw_event_connect_error(&event);
- cm_node->state = I40IW_CM_STATE_LISTENER_DESTROYED;
- i40iw_rem_ref_cm_node(cm_node);
- }
- }
- }
-
- if (!atomic_dec_return(&listener->ref_count)) {
- spin_lock_irqsave(&cm_core->listen_list_lock, flags);
- list_del(&listener->list);
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
-
- if (listener->iwdev) {
- if (apbvt_del)
- i40iw_manage_apbvt(listener->iwdev,
- listener->loc_port,
- I40IW_MANAGE_APBVT_DEL);
-
- memcpy(nfo.loc_addr, listener->loc_addr, sizeof(nfo.loc_addr));
- nfo.loc_port = listener->loc_port;
- nfo.ipv4 = listener->ipv4;
- nfo.vlan_id = listener->vlan_id;
- nfo.user_pri = listener->user_pri;
-
- if (!list_empty(&listener->child_listen_list)) {
- i40iw_del_multiple_qhash(listener->iwdev, &nfo, listener);
- } else {
- if (listener->qhash_set)
- i40iw_manage_qhash(listener->iwdev,
- &nfo,
- I40IW_QHASH_TYPE_TCP_SYN,
- I40IW_QHASH_MANAGE_TYPE_DELETE,
- NULL,
- false);
- }
- }
-
- cm_core->stats_listen_destroyed++;
- kfree(listener);
- cm_core->stats_listen_nodes_destroyed++;
- listener = NULL;
- ret = 0;
- }
-
- if (listener) {
- if (atomic_read(&listener->pend_accepts_cnt) > 0)
- i40iw_debug(cm_core->dev,
- I40IW_DEBUG_CM,
- "%s: listener (%p) pending accepts=%u\n",
- __func__,
- listener,
- atomic_read(&listener->pend_accepts_cnt));
- }
-
- return ret;
-}
-
-/**
- * i40iw_cm_del_listen - delete a linstener
- * @cm_core: cm's core
- * @listener: passive connection's listener
- * @apbvt_del: flag to delete apbvt
- */
-static int i40iw_cm_del_listen(struct i40iw_cm_core *cm_core,
- struct i40iw_cm_listener *listener,
- bool apbvt_del)
-{
- listener->listener_state = I40IW_CM_LISTENER_PASSIVE_STATE;
- listener->cm_id = NULL; /* going to be destroyed pretty soon */
- return i40iw_dec_refcnt_listen(cm_core, listener, 1, apbvt_del);
-}
-
-/**
- * i40iw_addr_resolve_neigh - resolve neighbor address
- * @iwdev: iwarp device structure
- * @src_ip: local ip address
- * @dst_ip: remote ip address
- * @arpindex: if there is an arp entry
- */
-static int i40iw_addr_resolve_neigh(struct i40iw_device *iwdev,
- u32 src_ip,
- u32 dst_ip,
- int arpindex)
-{
- struct rtable *rt;
- struct neighbour *neigh;
- int rc = arpindex;
- __be32 dst_ipaddr = htonl(dst_ip);
- __be32 src_ipaddr = htonl(src_ip);
-
- rt = ip_route_output(&init_net, dst_ipaddr, src_ipaddr, 0, 0);
- if (IS_ERR(rt)) {
- i40iw_pr_err("ip_route_output\n");
- return rc;
- }
-
- neigh = dst_neigh_lookup(&rt->dst, &dst_ipaddr);
-
- rcu_read_lock();
- if (neigh) {
- if (neigh->nud_state & NUD_VALID) {
- if (arpindex >= 0) {
- if (ether_addr_equal(iwdev->arp_table[arpindex].mac_addr,
- neigh->ha))
- /* Mac address same as arp table */
- goto resolve_neigh_exit;
- i40iw_manage_arp_cache(iwdev,
- iwdev->arp_table[arpindex].mac_addr,
- &dst_ip,
- true,
- I40IW_ARP_DELETE);
- }
-
- i40iw_manage_arp_cache(iwdev, neigh->ha, &dst_ip, true, I40IW_ARP_ADD);
- rc = i40iw_arp_table(iwdev, &dst_ip, true, NULL, I40IW_ARP_RESOLVE);
- } else {
- neigh_event_send(neigh, NULL);
- }
- }
- resolve_neigh_exit:
-
- rcu_read_unlock();
- if (neigh)
- neigh_release(neigh);
-
- ip_rt_put(rt);
- return rc;
-}
-
-/**
- * i40iw_get_dst_ipv6
- */
-static struct dst_entry *i40iw_get_dst_ipv6(struct sockaddr_in6 *src_addr,
- struct sockaddr_in6 *dst_addr)
-{
- struct dst_entry *dst;
- struct flowi6 fl6;
-
- memset(&fl6, 0, sizeof(fl6));
- fl6.daddr = dst_addr->sin6_addr;
- fl6.saddr = src_addr->sin6_addr;
- if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)
- fl6.flowi6_oif = dst_addr->sin6_scope_id;
-
- dst = ip6_route_output(&init_net, NULL, &fl6);
- return dst;
-}
-
-/**
- * i40iw_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address
- * @iwdev: iwarp device structure
- * @dst_ip: remote ip address
- * @arpindex: if there is an arp entry
- */
-static int i40iw_addr_resolve_neigh_ipv6(struct i40iw_device *iwdev,
- u32 *src,
- u32 *dest,
- int arpindex)
-{
- struct neighbour *neigh;
- int rc = arpindex;
- struct dst_entry *dst;
- struct sockaddr_in6 dst_addr;
- struct sockaddr_in6 src_addr;
-
- memset(&dst_addr, 0, sizeof(dst_addr));
- dst_addr.sin6_family = AF_INET6;
- i40iw_copy_ip_htonl(dst_addr.sin6_addr.in6_u.u6_addr32, dest);
- memset(&src_addr, 0, sizeof(src_addr));
- src_addr.sin6_family = AF_INET6;
- i40iw_copy_ip_htonl(src_addr.sin6_addr.in6_u.u6_addr32, src);
- dst = i40iw_get_dst_ipv6(&src_addr, &dst_addr);
- if (!dst || dst->error) {
- if (dst) {
- i40iw_pr_err("ip6_route_output returned dst->error = %d\n",
- dst->error);
- dst_release(dst);
- }
- return rc;
- }
-
- neigh = dst_neigh_lookup(dst, dst_addr.sin6_addr.in6_u.u6_addr32);
-
- rcu_read_lock();
- if (neigh) {
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, "dst_neigh_lookup MAC=%pM\n", neigh->ha);
- if (neigh->nud_state & NUD_VALID) {
- if (arpindex >= 0) {
- if (ether_addr_equal
- (iwdev->arp_table[arpindex].mac_addr,
- neigh->ha)) {
- /* Mac address same as in arp table */
- goto resolve_neigh_exit6;
- }
- i40iw_manage_arp_cache(iwdev,
- iwdev->arp_table[arpindex].mac_addr,
- dest,
- false,
- I40IW_ARP_DELETE);
- }
- i40iw_manage_arp_cache(iwdev,
- neigh->ha,
- dest,
- false,
- I40IW_ARP_ADD);
- rc = i40iw_arp_table(iwdev,
- dest,
- false,
- NULL,
- I40IW_ARP_RESOLVE);
- } else {
- neigh_event_send(neigh, NULL);
- }
- }
-
- resolve_neigh_exit6:
- rcu_read_unlock();
- if (neigh)
- neigh_release(neigh);
- dst_release(dst);
- return rc;
-}
-
-/**
- * i40iw_ipv4_is_loopback - check if loopback
- * @loc_addr: local addr to compare
- * @rem_addr: remote address
- */
-static bool i40iw_ipv4_is_loopback(u32 loc_addr, u32 rem_addr)
-{
- return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr);
-}
-
-/**
- * i40iw_ipv6_is_loopback - check if loopback
- * @loc_addr: local addr to compare
- * @rem_addr: remote address
- */
-static bool i40iw_ipv6_is_loopback(u32 *loc_addr, u32 *rem_addr)
-{
- struct in6_addr raddr6;
-
- i40iw_copy_ip_htonl(raddr6.in6_u.u6_addr32, rem_addr);
- return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6);
-}
-
-/**
- * i40iw_make_cm_node - create a new instance of a cm node
- * @cm_core: cm's core
- * @iwdev: iwarp device structure
- * @cm_info: quad info for connection
- * @listener: passive connection's listener
- */
-static struct i40iw_cm_node *i40iw_make_cm_node(
- struct i40iw_cm_core *cm_core,
- struct i40iw_device *iwdev,
- struct i40iw_cm_info *cm_info,
- struct i40iw_cm_listener *listener)
-{
- struct i40iw_cm_node *cm_node;
- int oldarpindex;
- int arpindex;
- struct net_device *netdev = iwdev->netdev;
-
- /* create an hte and cm_node for this instance */
- cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC);
- if (!cm_node)
- return NULL;
-
- /* set our node specific transport info */
- cm_node->ipv4 = cm_info->ipv4;
- cm_node->vlan_id = cm_info->vlan_id;
- if ((cm_node->vlan_id == I40IW_NO_VLAN) && iwdev->dcb)
- cm_node->vlan_id = 0;
- cm_node->tos = cm_info->tos;
- cm_node->user_pri = cm_info->user_pri;
- if (listener) {
- if (listener->tos != cm_info->tos)
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_DCB,
- "application TOS[%d] and remote client TOS[%d] mismatch\n",
- listener->tos, cm_info->tos);
- cm_node->tos = max(listener->tos, cm_info->tos);
- cm_node->user_pri = rt_tos2priority(cm_node->tos);
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_DCB, "listener: TOS:[%d] UP:[%d]\n",
- cm_node->tos, cm_node->user_pri);
- }
- memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr));
- memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr));
- cm_node->loc_port = cm_info->loc_port;
- cm_node->rem_port = cm_info->rem_port;
-
- cm_node->mpa_frame_rev = iwdev->mpa_version;
- cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
- cm_node->ird_size = I40IW_MAX_IRD_SIZE;
- cm_node->ord_size = I40IW_MAX_ORD_SIZE;
-
- cm_node->listener = listener;
- cm_node->cm_id = cm_info->cm_id;
- ether_addr_copy(cm_node->loc_mac, netdev->dev_addr);
- spin_lock_init(&cm_node->retrans_list_lock);
- cm_node->ack_rcvd = false;
-
- atomic_set(&cm_node->ref_count, 1);
- /* associate our parent CM core */
- cm_node->cm_core = cm_core;
- cm_node->tcp_cntxt.loc_id = I40IW_CM_DEF_LOCAL_ID;
- cm_node->tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE;
- cm_node->tcp_cntxt.rcv_wnd =
- I40IW_CM_DEFAULT_RCV_WND_SCALED >> I40IW_CM_DEFAULT_RCV_WND_SCALE;
- if (cm_node->ipv4) {
- cm_node->tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node->loc_addr[0]),
- htonl(cm_node->rem_addr[0]),
- htons(cm_node->loc_port),
- htons(cm_node->rem_port));
- cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - I40IW_MTU_TO_MSS_IPV4;
- } else if (IS_ENABLED(CONFIG_IPV6)) {
- __be32 loc[4] = {
- htonl(cm_node->loc_addr[0]), htonl(cm_node->loc_addr[1]),
- htonl(cm_node->loc_addr[2]), htonl(cm_node->loc_addr[3])
- };
- __be32 rem[4] = {
- htonl(cm_node->rem_addr[0]), htonl(cm_node->rem_addr[1]),
- htonl(cm_node->rem_addr[2]), htonl(cm_node->rem_addr[3])
- };
- cm_node->tcp_cntxt.loc_seq_num = secure_tcpv6_seq(loc, rem,
- htons(cm_node->loc_port),
- htons(cm_node->rem_port));
- cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - I40IW_MTU_TO_MSS_IPV6;
- }
-
- cm_node->iwdev = iwdev;
- cm_node->dev = &iwdev->sc_dev;
-
- if ((cm_node->ipv4 &&
- i40iw_ipv4_is_loopback(cm_node->loc_addr[0], cm_node->rem_addr[0])) ||
- (!cm_node->ipv4 && i40iw_ipv6_is_loopback(cm_node->loc_addr,
- cm_node->rem_addr))) {
- arpindex = i40iw_arp_table(iwdev,
- cm_node->rem_addr,
- false,
- NULL,
- I40IW_ARP_RESOLVE);
- } else {
- oldarpindex = i40iw_arp_table(iwdev,
- cm_node->rem_addr,
- false,
- NULL,
- I40IW_ARP_RESOLVE);
- if (cm_node->ipv4)
- arpindex = i40iw_addr_resolve_neigh(iwdev,
- cm_info->loc_addr[0],
- cm_info->rem_addr[0],
- oldarpindex);
- else if (IS_ENABLED(CONFIG_IPV6))
- arpindex = i40iw_addr_resolve_neigh_ipv6(iwdev,
- cm_info->loc_addr,
- cm_info->rem_addr,
- oldarpindex);
- else
- arpindex = -EINVAL;
- }
- if (arpindex < 0) {
- i40iw_pr_err("cm_node arpindex\n");
- kfree(cm_node);
- return NULL;
- }
- ether_addr_copy(cm_node->rem_mac, iwdev->arp_table[arpindex].mac_addr);
- i40iw_add_hte_node(cm_core, cm_node);
- cm_core->stats_nodes_created++;
- return cm_node;
-}
-
-/**
- * i40iw_rem_ref_cm_node - destroy an instance of a cm node
- * @cm_node: connection's node
- */
-static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *cm_node)
-{
- struct i40iw_cm_core *cm_core = cm_node->cm_core;
- struct i40iw_qp *iwqp;
- struct i40iw_cm_info nfo;
- unsigned long flags;
-
- spin_lock_irqsave(&cm_node->cm_core->ht_lock, flags);
- if (atomic_dec_return(&cm_node->ref_count)) {
- spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags);
- return;
- }
- list_del(&cm_node->list);
- spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags);
-
- /* if the node is destroyed before connection was accelerated */
- if (!cm_node->accelerated && cm_node->accept_pend) {
- pr_err("node destroyed before established\n");
- atomic_dec(&cm_node->listener->pend_accepts_cnt);
- }
- if (cm_node->close_entry)
- i40iw_handle_close_entry(cm_node, 0);
- if (cm_node->listener) {
- i40iw_dec_refcnt_listen(cm_core, cm_node->listener, 0, true);
- } else {
- if (cm_node->apbvt_set) {
- i40iw_manage_apbvt(cm_node->iwdev,
- cm_node->loc_port,
- I40IW_MANAGE_APBVT_DEL);
- cm_node->apbvt_set = 0;
- }
- i40iw_get_addr_info(cm_node, &nfo);
- if (cm_node->qhash_set) {
- i40iw_manage_qhash(cm_node->iwdev,
- &nfo,
- I40IW_QHASH_TYPE_TCP_ESTABLISHED,
- I40IW_QHASH_MANAGE_TYPE_DELETE,
- NULL,
- false);
- cm_node->qhash_set = 0;
- }
- }
-
- iwqp = cm_node->iwqp;
- if (iwqp) {
- iwqp->cm_node = NULL;
- i40iw_rem_ref(&iwqp->ibqp);
- cm_node->iwqp = NULL;
- } else if (cm_node->qhash_set) {
- i40iw_get_addr_info(cm_node, &nfo);
- i40iw_manage_qhash(cm_node->iwdev,
- &nfo,
- I40IW_QHASH_TYPE_TCP_ESTABLISHED,
- I40IW_QHASH_MANAGE_TYPE_DELETE,
- NULL,
- false);
- cm_node->qhash_set = 0;
- }
-
- cm_node->cm_core->stats_nodes_destroyed++;
- kfree(cm_node);
-}
-
-/**
- * i40iw_handle_fin_pkt - FIN packet received
- * @cm_node: connection's node
- */
-static void i40iw_handle_fin_pkt(struct i40iw_cm_node *cm_node)
-{
- u32 ret;
-
- switch (cm_node->state) {
- case I40IW_CM_STATE_SYN_RCVD:
- case I40IW_CM_STATE_SYN_SENT:
- case I40IW_CM_STATE_ESTABLISHED:
- case I40IW_CM_STATE_MPAREJ_RCVD:
- cm_node->tcp_cntxt.rcv_nxt++;
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->state = I40IW_CM_STATE_LAST_ACK;
- i40iw_send_fin(cm_node);
- break;
- case I40IW_CM_STATE_MPAREQ_SENT:
- i40iw_create_event(cm_node, I40IW_CM_EVENT_ABORTED);
- cm_node->tcp_cntxt.rcv_nxt++;
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->state = I40IW_CM_STATE_CLOSED;
- atomic_inc(&cm_node->ref_count);
- i40iw_send_reset(cm_node);
- break;
- case I40IW_CM_STATE_FIN_WAIT1:
- cm_node->tcp_cntxt.rcv_nxt++;
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->state = I40IW_CM_STATE_CLOSING;
- i40iw_send_ack(cm_node);
- /*
- * Wait for ACK as this is simultaneous close.
- * After we receive ACK, do not send anything.
- * Just rm the node.
- */
- break;
- case I40IW_CM_STATE_FIN_WAIT2:
- cm_node->tcp_cntxt.rcv_nxt++;
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->state = I40IW_CM_STATE_TIME_WAIT;
- i40iw_send_ack(cm_node);
- ret =
- i40iw_schedule_cm_timer(cm_node, NULL, I40IW_TIMER_TYPE_CLOSE, 1, 0);
- if (ret)
- i40iw_pr_err("node %p state = %d\n", cm_node, cm_node->state);
- break;
- case I40IW_CM_STATE_TIME_WAIT:
- cm_node->tcp_cntxt.rcv_nxt++;
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->state = I40IW_CM_STATE_CLOSED;
- i40iw_rem_ref_cm_node(cm_node);
- break;
- case I40IW_CM_STATE_OFFLOADED:
- default:
- i40iw_pr_err("bad state node %p state = %d\n", cm_node, cm_node->state);
- break;
- }
-}
-
-/**
- * i40iw_handle_rst_pkt - process received RST packet
- * @cm_node: connection's node
- * @rbuf: receive buffer
- */
-static void i40iw_handle_rst_pkt(struct i40iw_cm_node *cm_node,
- struct i40iw_puda_buf *rbuf)
-{
- i40iw_cleanup_retrans_entry(cm_node);
- switch (cm_node->state) {
- case I40IW_CM_STATE_SYN_SENT:
- case I40IW_CM_STATE_MPAREQ_SENT:
- switch (cm_node->mpa_frame_rev) {
- case IETF_MPA_V2:
- cm_node->mpa_frame_rev = IETF_MPA_V1;
- /* send a syn and goto syn sent state */
- cm_node->state = I40IW_CM_STATE_SYN_SENT;
- if (i40iw_send_syn(cm_node, 0))
- i40iw_active_open_err(cm_node, false);
- break;
- case IETF_MPA_V1:
- default:
- i40iw_active_open_err(cm_node, false);
- break;
- }
- break;
- case I40IW_CM_STATE_MPAREQ_RCVD:
- atomic_add_return(1, &cm_node->passive_state);
- break;
- case I40IW_CM_STATE_ESTABLISHED:
- case I40IW_CM_STATE_SYN_RCVD:
- case I40IW_CM_STATE_LISTENING:
- i40iw_pr_err("Bad state state = %d\n", cm_node->state);
- i40iw_passive_open_err(cm_node, false);
- break;
- case I40IW_CM_STATE_OFFLOADED:
- i40iw_active_open_err(cm_node, false);
- break;
- case I40IW_CM_STATE_CLOSED:
- break;
- case I40IW_CM_STATE_FIN_WAIT2:
- case I40IW_CM_STATE_FIN_WAIT1:
- case I40IW_CM_STATE_LAST_ACK:
- cm_node->cm_id->rem_ref(cm_node->cm_id);
- /* fall through */
- case I40IW_CM_STATE_TIME_WAIT:
- cm_node->state = I40IW_CM_STATE_CLOSED;
- i40iw_rem_ref_cm_node(cm_node);
- break;
- default:
- break;
- }
-}
-
-/**
- * i40iw_handle_rcv_mpa - Process a recv'd mpa buffer
- * @cm_node: connection's node
- * @rbuf: receive buffer
- */
-static void i40iw_handle_rcv_mpa(struct i40iw_cm_node *cm_node,
- struct i40iw_puda_buf *rbuf)
-{
- int ret;
- int datasize = rbuf->datalen;
- u8 *dataloc = rbuf->data;
-
- enum i40iw_cm_event_type type = I40IW_CM_EVENT_UNKNOWN;
- u32 res_type;
-
- ret = i40iw_parse_mpa(cm_node, dataloc, &res_type, datasize);
- if (ret) {
- if (cm_node->state == I40IW_CM_STATE_MPAREQ_SENT)
- i40iw_active_open_err(cm_node, true);
- else
- i40iw_passive_open_err(cm_node, true);
- return;
- }
-
- switch (cm_node->state) {
- case I40IW_CM_STATE_ESTABLISHED:
- if (res_type == I40IW_MPA_REQUEST_REJECT)
- i40iw_pr_err("state for reject\n");
- cm_node->state = I40IW_CM_STATE_MPAREQ_RCVD;
- type = I40IW_CM_EVENT_MPA_REQ;
- i40iw_send_ack(cm_node); /* ACK received MPA request */
- atomic_set(&cm_node->passive_state,
- I40IW_PASSIVE_STATE_INDICATED);
- break;
- case I40IW_CM_STATE_MPAREQ_SENT:
- i40iw_cleanup_retrans_entry(cm_node);
- if (res_type == I40IW_MPA_REQUEST_REJECT) {
- type = I40IW_CM_EVENT_MPA_REJECT;
- cm_node->state = I40IW_CM_STATE_MPAREJ_RCVD;
- } else {
- type = I40IW_CM_EVENT_CONNECTED;
- cm_node->state = I40IW_CM_STATE_OFFLOADED;
- }
- i40iw_send_ack(cm_node);
- break;
- default:
- pr_err("%s wrong cm_node state =%d\n", __func__, cm_node->state);
- break;
- }
- i40iw_create_event(cm_node, type);
-}
-
-/**
- * i40iw_indicate_pkt_err - Send up err event to cm
- * @cm_node: connection's node
- */
-static void i40iw_indicate_pkt_err(struct i40iw_cm_node *cm_node)
-{
- switch (cm_node->state) {
- case I40IW_CM_STATE_SYN_SENT:
- case I40IW_CM_STATE_MPAREQ_SENT:
- i40iw_active_open_err(cm_node, true);
- break;
- case I40IW_CM_STATE_ESTABLISHED:
- case I40IW_CM_STATE_SYN_RCVD:
- i40iw_passive_open_err(cm_node, true);
- break;
- case I40IW_CM_STATE_OFFLOADED:
- default:
- break;
- }
-}
-
-/**
- * i40iw_check_syn - Check for error on received syn ack
- * @cm_node: connection's node
- * @tcph: pointer tcp header
- */
-static int i40iw_check_syn(struct i40iw_cm_node *cm_node, struct tcphdr *tcph)
-{
- int err = 0;
-
- if (ntohl(tcph->ack_seq) != cm_node->tcp_cntxt.loc_seq_num) {
- err = 1;
- i40iw_active_open_err(cm_node, true);
- }
- return err;
-}
-
-/**
- * i40iw_check_seq - check seq numbers if OK
- * @cm_node: connection's node
- * @tcph: pointer tcp header
- */
-static int i40iw_check_seq(struct i40iw_cm_node *cm_node, struct tcphdr *tcph)
-{
- int err = 0;
- u32 seq;
- u32 ack_seq;
- u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num;
- u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
- u32 rcv_wnd;
-
- seq = ntohl(tcph->seq);
- ack_seq = ntohl(tcph->ack_seq);
- rcv_wnd = cm_node->tcp_cntxt.rcv_wnd;
- if (ack_seq != loc_seq_num)
- err = -1;
- else if (!between(seq, rcv_nxt, (rcv_nxt + rcv_wnd)))
- err = -1;
- if (err) {
- i40iw_pr_err("seq number\n");
- i40iw_indicate_pkt_err(cm_node);
- }
- return err;
-}
-
-/**
- * i40iw_handle_syn_pkt - is for Passive node
- * @cm_node: connection's node
- * @rbuf: receive buffer
- */
-static void i40iw_handle_syn_pkt(struct i40iw_cm_node *cm_node,
- struct i40iw_puda_buf *rbuf)
-{
- struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
- int ret;
- u32 inc_sequence;
- int optionsize;
- struct i40iw_cm_info nfo;
-
- optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
- inc_sequence = ntohl(tcph->seq);
-
- switch (cm_node->state) {
- case I40IW_CM_STATE_SYN_SENT:
- case I40IW_CM_STATE_MPAREQ_SENT:
- /* Rcvd syn on active open connection */
- i40iw_active_open_err(cm_node, 1);
- break;
- case I40IW_CM_STATE_LISTENING:
- /* Passive OPEN */
- if (atomic_read(&cm_node->listener->pend_accepts_cnt) >
- cm_node->listener->backlog) {
- cm_node->cm_core->stats_backlog_drops++;
- i40iw_passive_open_err(cm_node, false);
- break;
- }
- ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 1);
- if (ret) {
- i40iw_passive_open_err(cm_node, false);
- /* drop pkt */
- break;
- }
- cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
- cm_node->accept_pend = 1;
- atomic_inc(&cm_node->listener->pend_accepts_cnt);
-
- cm_node->state = I40IW_CM_STATE_SYN_RCVD;
- i40iw_get_addr_info(cm_node, &nfo);
- ret = i40iw_manage_qhash(cm_node->iwdev,
- &nfo,
- I40IW_QHASH_TYPE_TCP_ESTABLISHED,
- I40IW_QHASH_MANAGE_TYPE_ADD,
- (void *)cm_node,
- false);
- cm_node->qhash_set = true;
- break;
- case I40IW_CM_STATE_CLOSED:
- i40iw_cleanup_retrans_entry(cm_node);
- atomic_inc(&cm_node->ref_count);
- i40iw_send_reset(cm_node);
- break;
- case I40IW_CM_STATE_OFFLOADED:
- case I40IW_CM_STATE_ESTABLISHED:
- case I40IW_CM_STATE_FIN_WAIT1:
- case I40IW_CM_STATE_FIN_WAIT2:
- case I40IW_CM_STATE_MPAREQ_RCVD:
- case I40IW_CM_STATE_LAST_ACK:
- case I40IW_CM_STATE_CLOSING:
- case I40IW_CM_STATE_UNKNOWN:
- default:
- break;
- }
-}
-
-/**
- * i40iw_handle_synack_pkt - Process SYN+ACK packet (active side)
- * @cm_node: connection's node
- * @rbuf: receive buffer
- */
-static void i40iw_handle_synack_pkt(struct i40iw_cm_node *cm_node,
- struct i40iw_puda_buf *rbuf)
-{
- struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
- int ret;
- u32 inc_sequence;
- int optionsize;
-
- optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
- inc_sequence = ntohl(tcph->seq);
- switch (cm_node->state) {
- case I40IW_CM_STATE_SYN_SENT:
- i40iw_cleanup_retrans_entry(cm_node);
- /* active open */
- if (i40iw_check_syn(cm_node, tcph)) {
- i40iw_pr_err("check syn fail\n");
- return;
- }
- cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
- /* setup options */
- ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 0);
- if (ret) {
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "cm_node=%p tcp_options failed\n",
- cm_node);
- break;
- }
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
- i40iw_send_ack(cm_node); /* ACK for the syn_ack */
- ret = i40iw_send_mpa_request(cm_node);
- if (ret) {
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "cm_node=%p i40iw_send_mpa_request failed\n",
- cm_node);
- break;
- }
- cm_node->state = I40IW_CM_STATE_MPAREQ_SENT;
- break;
- case I40IW_CM_STATE_MPAREQ_RCVD:
- i40iw_passive_open_err(cm_node, true);
- break;
- case I40IW_CM_STATE_LISTENING:
- cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->state = I40IW_CM_STATE_CLOSED;
- i40iw_send_reset(cm_node);
- break;
- case I40IW_CM_STATE_CLOSED:
- cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
- i40iw_cleanup_retrans_entry(cm_node);
- atomic_inc(&cm_node->ref_count);
- i40iw_send_reset(cm_node);
- break;
- case I40IW_CM_STATE_ESTABLISHED:
- case I40IW_CM_STATE_FIN_WAIT1:
- case I40IW_CM_STATE_FIN_WAIT2:
- case I40IW_CM_STATE_LAST_ACK:
- case I40IW_CM_STATE_OFFLOADED:
- case I40IW_CM_STATE_CLOSING:
- case I40IW_CM_STATE_UNKNOWN:
- case I40IW_CM_STATE_MPAREQ_SENT:
- default:
- break;
- }
-}
-
-/**
- * i40iw_handle_ack_pkt - process packet with ACK
- * @cm_node: connection's node
- * @rbuf: receive buffer
- */
-static int i40iw_handle_ack_pkt(struct i40iw_cm_node *cm_node,
- struct i40iw_puda_buf *rbuf)
-{
- struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
- u32 inc_sequence;
- int ret = 0;
- int optionsize;
- u32 datasize = rbuf->datalen;
-
- optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
-
- if (i40iw_check_seq(cm_node, tcph))
- return -EINVAL;
-
- inc_sequence = ntohl(tcph->seq);
- switch (cm_node->state) {
- case I40IW_CM_STATE_SYN_RCVD:
- i40iw_cleanup_retrans_entry(cm_node);
- ret = i40iw_handle_tcp_options(cm_node, tcph, optionsize, 1);
- if (ret)
- break;
- cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
- cm_node->state = I40IW_CM_STATE_ESTABLISHED;
- if (datasize) {
- cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
- i40iw_handle_rcv_mpa(cm_node, rbuf);
- }
- break;
- case I40IW_CM_STATE_ESTABLISHED:
- i40iw_cleanup_retrans_entry(cm_node);
- if (datasize) {
- cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
- i40iw_handle_rcv_mpa(cm_node, rbuf);
- }
- break;
- case I40IW_CM_STATE_MPAREQ_SENT:
- cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
- if (datasize) {
- cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
- cm_node->ack_rcvd = false;
- i40iw_handle_rcv_mpa(cm_node, rbuf);
- } else {
- cm_node->ack_rcvd = true;
- }
- break;
- case I40IW_CM_STATE_LISTENING:
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->state = I40IW_CM_STATE_CLOSED;
- i40iw_send_reset(cm_node);
- break;
- case I40IW_CM_STATE_CLOSED:
- i40iw_cleanup_retrans_entry(cm_node);
- atomic_inc(&cm_node->ref_count);
- i40iw_send_reset(cm_node);
- break;
- case I40IW_CM_STATE_LAST_ACK:
- case I40IW_CM_STATE_CLOSING:
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->state = I40IW_CM_STATE_CLOSED;
- if (!cm_node->accept_pend)
- cm_node->cm_id->rem_ref(cm_node->cm_id);
- i40iw_rem_ref_cm_node(cm_node);
- break;
- case I40IW_CM_STATE_FIN_WAIT1:
- i40iw_cleanup_retrans_entry(cm_node);
- cm_node->state = I40IW_CM_STATE_FIN_WAIT2;
- break;
- case I40IW_CM_STATE_SYN_SENT:
- case I40IW_CM_STATE_FIN_WAIT2:
- case I40IW_CM_STATE_OFFLOADED:
- case I40IW_CM_STATE_MPAREQ_RCVD:
- case I40IW_CM_STATE_UNKNOWN:
- default:
- i40iw_cleanup_retrans_entry(cm_node);
- break;
- }
- return ret;
-}
-
-/**
- * i40iw_process_packet - process cm packet
- * @cm_node: connection's node
- * @rbuf: receive buffer
- */
-static void i40iw_process_packet(struct i40iw_cm_node *cm_node,
- struct i40iw_puda_buf *rbuf)
-{
- enum i40iw_tcpip_pkt_type pkt_type = I40IW_PKT_TYPE_UNKNOWN;
- struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
- u32 fin_set = 0;
- int ret;
-
- if (tcph->rst) {
- pkt_type = I40IW_PKT_TYPE_RST;
- } else if (tcph->syn) {
- pkt_type = I40IW_PKT_TYPE_SYN;
- if (tcph->ack)
- pkt_type = I40IW_PKT_TYPE_SYNACK;
- } else if (tcph->ack) {
- pkt_type = I40IW_PKT_TYPE_ACK;
- }
- if (tcph->fin)
- fin_set = 1;
-
- switch (pkt_type) {
- case I40IW_PKT_TYPE_SYN:
- i40iw_handle_syn_pkt(cm_node, rbuf);
- break;
- case I40IW_PKT_TYPE_SYNACK:
- i40iw_handle_synack_pkt(cm_node, rbuf);
- break;
- case I40IW_PKT_TYPE_ACK:
- ret = i40iw_handle_ack_pkt(cm_node, rbuf);
- if (fin_set && !ret)
- i40iw_handle_fin_pkt(cm_node);
- break;
- case I40IW_PKT_TYPE_RST:
- i40iw_handle_rst_pkt(cm_node, rbuf);
- break;
- default:
- if (fin_set &&
- (!i40iw_check_seq(cm_node, (struct tcphdr *)rbuf->tcph)))
- i40iw_handle_fin_pkt(cm_node);
- break;
- }
-}
-
-/**
- * i40iw_make_listen_node - create a listen node with params
- * @cm_core: cm's core
- * @iwdev: iwarp device structure
- * @cm_info: quad info for connection
- */
-static struct i40iw_cm_listener *i40iw_make_listen_node(
- struct i40iw_cm_core *cm_core,
- struct i40iw_device *iwdev,
- struct i40iw_cm_info *cm_info)
-{
- struct i40iw_cm_listener *listener;
- unsigned long flags;
-
- /* cannot have multiple matching listeners */
- listener = i40iw_find_listener(cm_core, cm_info->loc_addr,
- cm_info->loc_port,
- cm_info->vlan_id,
- I40IW_CM_LISTENER_EITHER_STATE);
- if (listener &&
- (listener->listener_state == I40IW_CM_LISTENER_ACTIVE_STATE)) {
- atomic_dec(&listener->ref_count);
- i40iw_debug(cm_core->dev,
- I40IW_DEBUG_CM,
- "Not creating listener since it already exists\n");
- return NULL;
- }
-
- if (!listener) {
- /* create a CM listen node (1/2 node to compare incoming traffic to) */
- listener = kzalloc(sizeof(*listener), GFP_KERNEL);
- if (!listener)
- return NULL;
- cm_core->stats_listen_nodes_created++;
- memcpy(listener->loc_addr, cm_info->loc_addr, sizeof(listener->loc_addr));
- listener->loc_port = cm_info->loc_port;
-
- INIT_LIST_HEAD(&listener->child_listen_list);
-
- atomic_set(&listener->ref_count, 1);
- } else {
- listener->reused_node = 1;
- }
-
- listener->cm_id = cm_info->cm_id;
- listener->ipv4 = cm_info->ipv4;
- listener->vlan_id = cm_info->vlan_id;
- atomic_set(&listener->pend_accepts_cnt, 0);
- listener->cm_core = cm_core;
- listener->iwdev = iwdev;
-
- listener->backlog = cm_info->backlog;
- listener->listener_state = I40IW_CM_LISTENER_ACTIVE_STATE;
-
- if (!listener->reused_node) {
- spin_lock_irqsave(&cm_core->listen_list_lock, flags);
- list_add(&listener->list, &cm_core->listen_nodes);
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
- }
-
- return listener;
-}
-
-/**
- * i40iw_create_cm_node - make a connection node with params
- * @cm_core: cm's core
- * @iwdev: iwarp device structure
- * @conn_param: upper layer connection parameters
- * @cm_info: quad info for connection
- */
-static struct i40iw_cm_node *i40iw_create_cm_node(
- struct i40iw_cm_core *cm_core,
- struct i40iw_device *iwdev,
- struct iw_cm_conn_param *conn_param,
- struct i40iw_cm_info *cm_info)
-{
- struct i40iw_cm_node *cm_node;
- struct i40iw_cm_listener *loopback_remotelistener;
- struct i40iw_cm_node *loopback_remotenode;
- struct i40iw_cm_info loopback_cm_info;
-
- u16 private_data_len = conn_param->private_data_len;
- const void *private_data = conn_param->private_data;
-
- /* create a CM connection node */
- cm_node = i40iw_make_cm_node(cm_core, iwdev, cm_info, NULL);
- if (!cm_node)
- return ERR_PTR(-ENOMEM);
- /* set our node side to client (active) side */
- cm_node->tcp_cntxt.client = 1;
- cm_node->tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE;
-
- i40iw_record_ird_ord(cm_node, conn_param->ird, conn_param->ord);
-
- if (!memcmp(cm_info->loc_addr, cm_info->rem_addr, sizeof(cm_info->loc_addr))) {
- loopback_remotelistener = i40iw_find_listener(
- cm_core,
- cm_info->rem_addr,
- cm_node->rem_port,
- cm_node->vlan_id,
- I40IW_CM_LISTENER_ACTIVE_STATE);
- if (!loopback_remotelistener) {
- i40iw_rem_ref_cm_node(cm_node);
- return ERR_PTR(-ECONNREFUSED);
- } else {
- loopback_cm_info = *cm_info;
- loopback_cm_info.loc_port = cm_info->rem_port;
- loopback_cm_info.rem_port = cm_info->loc_port;
- loopback_cm_info.cm_id = loopback_remotelistener->cm_id;
- loopback_cm_info.ipv4 = cm_info->ipv4;
- loopback_remotenode = i40iw_make_cm_node(cm_core,
- iwdev,
- &loopback_cm_info,
- loopback_remotelistener);
- if (!loopback_remotenode) {
- i40iw_rem_ref_cm_node(cm_node);
- return ERR_PTR(-ENOMEM);
- }
- cm_core->stats_loopbacks++;
- loopback_remotenode->loopbackpartner = cm_node;
- loopback_remotenode->tcp_cntxt.rcv_wscale =
- I40IW_CM_DEFAULT_RCV_WND_SCALE;
- cm_node->loopbackpartner = loopback_remotenode;
- memcpy(loopback_remotenode->pdata_buf, private_data,
- private_data_len);
- loopback_remotenode->pdata.size = private_data_len;
-
- if (loopback_remotenode->ord_size > cm_node->ird_size)
- loopback_remotenode->ord_size =
- cm_node->ird_size;
-
- cm_node->state = I40IW_CM_STATE_OFFLOADED;
- cm_node->tcp_cntxt.rcv_nxt =
- loopback_remotenode->tcp_cntxt.loc_seq_num;
- loopback_remotenode->tcp_cntxt.rcv_nxt =
- cm_node->tcp_cntxt.loc_seq_num;
- cm_node->tcp_cntxt.max_snd_wnd =
- loopback_remotenode->tcp_cntxt.rcv_wnd;
- loopback_remotenode->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.rcv_wnd;
- cm_node->tcp_cntxt.snd_wnd = loopback_remotenode->tcp_cntxt.rcv_wnd;
- loopback_remotenode->tcp_cntxt.snd_wnd = cm_node->tcp_cntxt.rcv_wnd;
- cm_node->tcp_cntxt.snd_wscale = loopback_remotenode->tcp_cntxt.rcv_wscale;
- loopback_remotenode->tcp_cntxt.snd_wscale = cm_node->tcp_cntxt.rcv_wscale;
- }
- return cm_node;
- }
-
- cm_node->pdata.size = private_data_len;
- cm_node->pdata.addr = cm_node->pdata_buf;
-
- memcpy(cm_node->pdata_buf, private_data, private_data_len);
-
- cm_node->state = I40IW_CM_STATE_SYN_SENT;
- return cm_node;
-}
-
-/**
- * i40iw_cm_reject - reject and teardown a connection
- * @cm_node: connection's node
- * @pdate: ptr to private data for reject
- * @plen: size of private data
- */
-static int i40iw_cm_reject(struct i40iw_cm_node *cm_node, const void *pdata, u8 plen)
-{
- int ret = 0;
- int err;
- int passive_state;
- struct iw_cm_id *cm_id = cm_node->cm_id;
- struct i40iw_cm_node *loopback = cm_node->loopbackpartner;
-
- if (cm_node->tcp_cntxt.client)
- return ret;
- i40iw_cleanup_retrans_entry(cm_node);
-
- if (!loopback) {
- passive_state = atomic_add_return(1, &cm_node->passive_state);
- if (passive_state == I40IW_SEND_RESET_EVENT) {
- cm_node->state = I40IW_CM_STATE_CLOSED;
- i40iw_rem_ref_cm_node(cm_node);
- } else {
- if (cm_node->state == I40IW_CM_STATE_LISTENER_DESTROYED) {
- i40iw_rem_ref_cm_node(cm_node);
- } else {
- ret = i40iw_send_mpa_reject(cm_node, pdata, plen);
- if (ret) {
- cm_node->state = I40IW_CM_STATE_CLOSED;
- err = i40iw_send_reset(cm_node);
- if (err)
- i40iw_pr_err("send reset failed\n");
- } else {
- cm_id->add_ref(cm_id);
- }
- }
- }
- } else {
- cm_node->cm_id = NULL;
- if (cm_node->state == I40IW_CM_STATE_LISTENER_DESTROYED) {
- i40iw_rem_ref_cm_node(cm_node);
- i40iw_rem_ref_cm_node(loopback);
- } else {
- ret = i40iw_send_cm_event(loopback,
- loopback->cm_id,
- IW_CM_EVENT_CONNECT_REPLY,
- -ECONNREFUSED);
- i40iw_rem_ref_cm_node(cm_node);
- loopback->state = I40IW_CM_STATE_CLOSING;
-
- cm_id = loopback->cm_id;
- i40iw_rem_ref_cm_node(loopback);
- cm_id->rem_ref(cm_id);
- }
- }
-
- return ret;
-}
-
-/**
- * i40iw_cm_close - close of cm connection
- * @cm_node: connection's node
- */
-static int i40iw_cm_close(struct i40iw_cm_node *cm_node)
-{
- int ret = 0;
-
- if (!cm_node)
- return -EINVAL;
-
- switch (cm_node->state) {
- case I40IW_CM_STATE_SYN_RCVD:
- case I40IW_CM_STATE_SYN_SENT:
- case I40IW_CM_STATE_ONE_SIDE_ESTABLISHED:
- case I40IW_CM_STATE_ESTABLISHED:
- case I40IW_CM_STATE_ACCEPTING:
- case I40IW_CM_STATE_MPAREQ_SENT:
- case I40IW_CM_STATE_MPAREQ_RCVD:
- i40iw_cleanup_retrans_entry(cm_node);
- i40iw_send_reset(cm_node);
- break;
- case I40IW_CM_STATE_CLOSE_WAIT:
- cm_node->state = I40IW_CM_STATE_LAST_ACK;
- i40iw_send_fin(cm_node);
- break;
- case I40IW_CM_STATE_FIN_WAIT1:
- case I40IW_CM_STATE_FIN_WAIT2:
- case I40IW_CM_STATE_LAST_ACK:
- case I40IW_CM_STATE_TIME_WAIT:
- case I40IW_CM_STATE_CLOSING:
- ret = -1;
- break;
- case I40IW_CM_STATE_LISTENING:
- i40iw_cleanup_retrans_entry(cm_node);
- i40iw_send_reset(cm_node);
- break;
- case I40IW_CM_STATE_MPAREJ_RCVD:
- case I40IW_CM_STATE_UNKNOWN:
- case I40IW_CM_STATE_INITED:
- case I40IW_CM_STATE_CLOSED:
- case I40IW_CM_STATE_LISTENER_DESTROYED:
- i40iw_rem_ref_cm_node(cm_node);
- break;
- case I40IW_CM_STATE_OFFLOADED:
- if (cm_node->send_entry)
- i40iw_pr_err("send_entry\n");
- i40iw_rem_ref_cm_node(cm_node);
- break;
- }
- return ret;
-}
-
-/**
- * i40iw_receive_ilq - recv an ETHERNET packet, and process it
- * through CM
- * @vsi: pointer to the vsi structure
- * @rbuf: receive buffer
- */
-void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf)
-{
- struct i40iw_cm_node *cm_node;
- struct i40iw_cm_listener *listener;
- struct iphdr *iph;
- struct ipv6hdr *ip6h;
- struct tcphdr *tcph;
- struct i40iw_cm_info cm_info;
- struct i40iw_sc_dev *dev = vsi->dev;
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
- struct i40iw_cm_core *cm_core = &iwdev->cm_core;
- struct vlan_ethhdr *ethh;
- u16 vtag;
-
- /* if vlan, then maclen = 18 else 14 */
- iph = (struct iphdr *)rbuf->iph;
- memset(&cm_info, 0, sizeof(cm_info));
-
- i40iw_debug_buf(dev,
- I40IW_DEBUG_ILQ,
- "RECEIVE ILQ BUFFER",
- rbuf->mem.va,
- rbuf->totallen);
- ethh = (struct vlan_ethhdr *)rbuf->mem.va;
-
- if (ethh->h_vlan_proto == htons(ETH_P_8021Q)) {
- vtag = ntohs(ethh->h_vlan_TCI);
- cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
- cm_info.vlan_id = vtag & VLAN_VID_MASK;
- i40iw_debug(cm_core->dev,
- I40IW_DEBUG_CM,
- "%s vlan_id=%d\n",
- __func__,
- cm_info.vlan_id);
- } else {
- cm_info.vlan_id = I40IW_NO_VLAN;
- }
- tcph = (struct tcphdr *)rbuf->tcph;
-
- if (rbuf->ipv4) {
- cm_info.loc_addr[0] = ntohl(iph->daddr);
- cm_info.rem_addr[0] = ntohl(iph->saddr);
- cm_info.ipv4 = true;
- cm_info.tos = iph->tos;
- } else {
- ip6h = (struct ipv6hdr *)rbuf->iph;
- i40iw_copy_ip_ntohl(cm_info.loc_addr,
- ip6h->daddr.in6_u.u6_addr32);
- i40iw_copy_ip_ntohl(cm_info.rem_addr,
- ip6h->saddr.in6_u.u6_addr32);
- cm_info.ipv4 = false;
- cm_info.tos = (ip6h->priority << 4) | (ip6h->flow_lbl[0] >> 4);
- }
- cm_info.loc_port = ntohs(tcph->dest);
- cm_info.rem_port = ntohs(tcph->source);
- cm_node = i40iw_find_node(cm_core,
- cm_info.rem_port,
- cm_info.rem_addr,
- cm_info.loc_port,
- cm_info.loc_addr,
- true,
- false);
-
- if (!cm_node) {
- /* Only type of packet accepted are for */
- /* the PASSIVE open (syn only) */
- if (!tcph->syn || tcph->ack)
- return;
- listener =
- i40iw_find_listener(cm_core,
- cm_info.loc_addr,
- cm_info.loc_port,
- cm_info.vlan_id,
- I40IW_CM_LISTENER_ACTIVE_STATE);
- if (!listener) {
- cm_info.cm_id = NULL;
- i40iw_debug(cm_core->dev,
- I40IW_DEBUG_CM,
- "%s no listener found\n",
- __func__);
- return;
- }
- cm_info.cm_id = listener->cm_id;
- cm_node = i40iw_make_cm_node(cm_core, iwdev, &cm_info, listener);
- if (!cm_node) {
- i40iw_debug(cm_core->dev,
- I40IW_DEBUG_CM,
- "%s allocate node failed\n",
- __func__);
- atomic_dec(&listener->ref_count);
- return;
- }
- if (!tcph->rst && !tcph->fin) {
- cm_node->state = I40IW_CM_STATE_LISTENING;
- } else {
- i40iw_rem_ref_cm_node(cm_node);
- return;
- }
- atomic_inc(&cm_node->ref_count);
- } else if (cm_node->state == I40IW_CM_STATE_OFFLOADED) {
- i40iw_rem_ref_cm_node(cm_node);
- return;
- }
- i40iw_process_packet(cm_node, rbuf);
- i40iw_rem_ref_cm_node(cm_node);
-}
-
-/**
- * i40iw_setup_cm_core - allocate a top level instance of a cm
- * core
- * @iwdev: iwarp device structure
- */
-int i40iw_setup_cm_core(struct i40iw_device *iwdev)
-{
- struct i40iw_cm_core *cm_core = &iwdev->cm_core;
-
- cm_core->iwdev = iwdev;
- cm_core->dev = &iwdev->sc_dev;
-
- INIT_LIST_HEAD(&cm_core->accelerated_list);
- INIT_LIST_HEAD(&cm_core->non_accelerated_list);
- INIT_LIST_HEAD(&cm_core->listen_nodes);
-
- timer_setup(&cm_core->tcp_timer, i40iw_cm_timer_tick, 0);
-
- spin_lock_init(&cm_core->ht_lock);
- spin_lock_init(&cm_core->listen_list_lock);
- spin_lock_init(&cm_core->apbvt_lock);
-
- cm_core->event_wq = alloc_ordered_workqueue("iwewq",
- WQ_MEM_RECLAIM);
- if (!cm_core->event_wq)
- goto error;
-
- cm_core->disconn_wq = alloc_ordered_workqueue("iwdwq",
- WQ_MEM_RECLAIM);
- if (!cm_core->disconn_wq)
- goto error;
-
- return 0;
-error:
- i40iw_cleanup_cm_core(&iwdev->cm_core);
-
- return -ENOMEM;
-}
-
-/**
- * i40iw_cleanup_cm_core - deallocate a top level instance of a
- * cm core
- * @cm_core: cm's core
- */
-void i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core)
-{
- unsigned long flags;
-
- if (!cm_core)
- return;
-
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- if (timer_pending(&cm_core->tcp_timer))
- del_timer_sync(&cm_core->tcp_timer);
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-
- if (cm_core->event_wq)
- destroy_workqueue(cm_core->event_wq);
- if (cm_core->disconn_wq)
- destroy_workqueue(cm_core->disconn_wq);
-}
-
-/**
- * i40iw_init_tcp_ctx - setup qp context
- * @cm_node: connection's node
- * @tcp_info: offload info for tcp
- * @iwqp: associate qp for the connection
- */
-static void i40iw_init_tcp_ctx(struct i40iw_cm_node *cm_node,
- struct i40iw_tcp_offload_info *tcp_info,
- struct i40iw_qp *iwqp)
-{
- tcp_info->ipv4 = cm_node->ipv4;
- tcp_info->drop_ooo_seg = true;
- tcp_info->wscale = true;
- tcp_info->ignore_tcp_opt = true;
- tcp_info->ignore_tcp_uns_opt = true;
- tcp_info->no_nagle = false;
-
- tcp_info->ttl = I40IW_DEFAULT_TTL;
- tcp_info->rtt_var = cpu_to_le32(I40IW_DEFAULT_RTT_VAR);
- tcp_info->ss_thresh = cpu_to_le32(I40IW_DEFAULT_SS_THRESH);
- tcp_info->rexmit_thresh = I40IW_DEFAULT_REXMIT_THRESH;
-
- tcp_info->tcp_state = I40IW_TCP_STATE_ESTABLISHED;
- tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale;
- tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale;
-
- tcp_info->snd_nxt = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
- tcp_info->snd_wnd = cpu_to_le32(cm_node->tcp_cntxt.snd_wnd);
- tcp_info->rcv_nxt = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt);
- tcp_info->snd_max = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
-
- tcp_info->snd_una = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
- tcp_info->cwnd = cpu_to_le32(2 * cm_node->tcp_cntxt.mss);
- tcp_info->snd_wl1 = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt);
- tcp_info->snd_wl2 = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num);
- tcp_info->max_snd_window = cpu_to_le32(cm_node->tcp_cntxt.max_snd_wnd);
- tcp_info->rcv_wnd = cpu_to_le32(cm_node->tcp_cntxt.rcv_wnd <<
- cm_node->tcp_cntxt.rcv_wscale);
-
- tcp_info->flow_label = 0;
- tcp_info->snd_mss = cpu_to_le32(((u32)cm_node->tcp_cntxt.mss));
- if (cm_node->vlan_id <= VLAN_VID_MASK) {
- tcp_info->insert_vlan_tag = true;
- tcp_info->vlan_tag = cpu_to_le16(((u16)cm_node->user_pri << I40IW_VLAN_PRIO_SHIFT) |
- cm_node->vlan_id);
- }
- if (cm_node->ipv4) {
- tcp_info->src_port = cpu_to_le16(cm_node->loc_port);
- tcp_info->dst_port = cpu_to_le16(cm_node->rem_port);
-
- tcp_info->dest_ip_addr3 = cpu_to_le32(cm_node->rem_addr[0]);
- tcp_info->local_ipaddr3 = cpu_to_le32(cm_node->loc_addr[0]);
- tcp_info->arp_idx =
- cpu_to_le16((u16)i40iw_arp_table(
- iwqp->iwdev,
- &tcp_info->dest_ip_addr3,
- true,
- NULL,
- I40IW_ARP_RESOLVE));
- } else {
- tcp_info->src_port = cpu_to_le16(cm_node->loc_port);
- tcp_info->dst_port = cpu_to_le16(cm_node->rem_port);
- tcp_info->dest_ip_addr0 = cpu_to_le32(cm_node->rem_addr[0]);
- tcp_info->dest_ip_addr1 = cpu_to_le32(cm_node->rem_addr[1]);
- tcp_info->dest_ip_addr2 = cpu_to_le32(cm_node->rem_addr[2]);
- tcp_info->dest_ip_addr3 = cpu_to_le32(cm_node->rem_addr[3]);
- tcp_info->local_ipaddr0 = cpu_to_le32(cm_node->loc_addr[0]);
- tcp_info->local_ipaddr1 = cpu_to_le32(cm_node->loc_addr[1]);
- tcp_info->local_ipaddr2 = cpu_to_le32(cm_node->loc_addr[2]);
- tcp_info->local_ipaddr3 = cpu_to_le32(cm_node->loc_addr[3]);
- tcp_info->arp_idx =
- cpu_to_le16((u16)i40iw_arp_table(
- iwqp->iwdev,
- &tcp_info->dest_ip_addr0,
- false,
- NULL,
- I40IW_ARP_RESOLVE));
- }
-}
-
-/**
- * i40iw_cm_init_tsa_conn - setup qp for RTS
- * @iwqp: associate qp for the connection
- * @cm_node: connection's node
- */
-static void i40iw_cm_init_tsa_conn(struct i40iw_qp *iwqp,
- struct i40iw_cm_node *cm_node)
-{
- struct i40iw_tcp_offload_info tcp_info;
- struct i40iwarp_offload_info *iwarp_info;
- struct i40iw_qp_host_ctx_info *ctx_info;
- struct i40iw_device *iwdev = iwqp->iwdev;
- struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev;
-
- memset(&tcp_info, 0x00, sizeof(struct i40iw_tcp_offload_info));
- iwarp_info = &iwqp->iwarp_info;
- ctx_info = &iwqp->ctx_info;
-
- ctx_info->tcp_info = &tcp_info;
- ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
- ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
-
- iwarp_info->ord_size = cm_node->ord_size;
- iwarp_info->ird_size = i40iw_derive_hw_ird_setting(cm_node->ird_size);
-
- if (iwarp_info->ord_size == 1)
- iwarp_info->ord_size = 2;
-
- iwarp_info->rd_enable = true;
- iwarp_info->rdmap_ver = 1;
- iwarp_info->ddp_ver = 1;
-
- iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id;
-
- ctx_info->tcp_info_valid = true;
- ctx_info->iwarp_info_valid = true;
- ctx_info->add_to_qoslist = true;
- ctx_info->user_pri = cm_node->user_pri;
-
- i40iw_init_tcp_ctx(cm_node, &tcp_info, iwqp);
- if (cm_node->snd_mark_en) {
- iwarp_info->snd_mark_en = true;
- iwarp_info->snd_mark_offset = (tcp_info.snd_nxt &
- SNDMARKER_SEQNMASK) + cm_node->lsmm_size;
- }
-
- cm_node->state = I40IW_CM_STATE_OFFLOADED;
- tcp_info.tcp_state = I40IW_TCP_STATE_ESTABLISHED;
- tcp_info.src_mac_addr_idx = iwdev->mac_ip_table_idx;
- tcp_info.tos = cm_node->tos;
-
- dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp, (u64 *)(iwqp->host_ctx.va), ctx_info);
-
- /* once tcp_info is set, no need to do it again */
- ctx_info->tcp_info_valid = false;
- ctx_info->iwarp_info_valid = false;
- ctx_info->add_to_qoslist = false;
-}
-
-/**
- * i40iw_cm_disconn - when a connection is being closed
- * @iwqp: associate qp for the connection
- */
-void i40iw_cm_disconn(struct i40iw_qp *iwqp)
-{
- struct disconn_work *work;
- struct i40iw_device *iwdev = iwqp->iwdev;
- struct i40iw_cm_core *cm_core = &iwdev->cm_core;
- unsigned long flags;
-
- work = kzalloc(sizeof(*work), GFP_ATOMIC);
- if (!work)
- return; /* Timer will clean up */
-
- spin_lock_irqsave(&iwdev->qptable_lock, flags);
- if (!iwdev->qp_table[iwqp->ibqp.qp_num]) {
- spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
- "%s qp_id %d is already freed\n",
- __func__, iwqp->ibqp.qp_num);
- kfree(work);
- return;
- }
- i40iw_add_ref(&iwqp->ibqp);
- spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
-
- work->iwqp = iwqp;
- INIT_WORK(&work->work, i40iw_disconnect_worker);
- queue_work(cm_core->disconn_wq, &work->work);
- return;
-}
-
-/**
- * i40iw_qp_disconnect - free qp and close cm
- * @iwqp: associate qp for the connection
- */
-static void i40iw_qp_disconnect(struct i40iw_qp *iwqp)
-{
- struct i40iw_device *iwdev;
- struct i40iw_ib_device *iwibdev;
-
- iwdev = to_iwdev(iwqp->ibqp.device);
- if (!iwdev) {
- i40iw_pr_err("iwdev == NULL\n");
- return;
- }
-
- iwibdev = iwdev->iwibdev;
-
- if (iwqp->active_conn) {
- /* indicate this connection is NOT active */
- iwqp->active_conn = 0;
- } else {
- /* Need to free the Last Streaming Mode Message */
- if (iwqp->ietf_mem.va) {
- if (iwqp->lsmm_mr)
- iwibdev->ibdev.ops.dereg_mr(iwqp->lsmm_mr,
- NULL);
- i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->ietf_mem);
- }
- }
-
- /* close the CM node down if it is still active */
- if (iwqp->cm_node) {
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM, "%s Call close API\n", __func__);
- i40iw_cm_close(iwqp->cm_node);
- }
-}
-
-/**
- * i40iw_cm_disconn_true - called by worker thread to disconnect qp
- * @iwqp: associate qp for the connection
- */
-static void i40iw_cm_disconn_true(struct i40iw_qp *iwqp)
-{
- struct iw_cm_id *cm_id;
- struct i40iw_device *iwdev;
- struct i40iw_sc_qp *qp = &iwqp->sc_qp;
- u16 last_ae;
- u8 original_hw_tcp_state;
- u8 original_ibqp_state;
- int disconn_status = 0;
- int issue_disconn = 0;
- int issue_close = 0;
- int issue_flush = 0;
- struct ib_event ibevent;
- unsigned long flags;
- int ret;
-
- if (!iwqp) {
- i40iw_pr_err("iwqp == NULL\n");
- return;
- }
-
- spin_lock_irqsave(&iwqp->lock, flags);
- cm_id = iwqp->cm_id;
- /* make sure we havent already closed this connection */
- if (!cm_id) {
- spin_unlock_irqrestore(&iwqp->lock, flags);
- return;
- }
-
- iwdev = to_iwdev(iwqp->ibqp.device);
-
- original_hw_tcp_state = iwqp->hw_tcp_state;
- original_ibqp_state = iwqp->ibqp_state;
- last_ae = iwqp->last_aeq;
-
- if (qp->term_flags) {
- issue_disconn = 1;
- issue_close = 1;
- iwqp->cm_id = NULL;
- /*When term timer expires after cm_timer, don't want
- *terminate-handler to issue cm_disconn which can re-free
- *a QP even after its refcnt=0.
- */
- i40iw_terminate_del_timer(qp);
- if (!iwqp->flush_issued) {
- iwqp->flush_issued = 1;
- issue_flush = 1;
- }
- } else if ((original_hw_tcp_state == I40IW_TCP_STATE_CLOSE_WAIT) ||
- ((original_ibqp_state == IB_QPS_RTS) &&
- (last_ae == I40IW_AE_LLP_CONNECTION_RESET))) {
- issue_disconn = 1;
- if (last_ae == I40IW_AE_LLP_CONNECTION_RESET)
- disconn_status = -ECONNRESET;
- }
-
- if (((original_hw_tcp_state == I40IW_TCP_STATE_CLOSED) ||
- (original_hw_tcp_state == I40IW_TCP_STATE_TIME_WAIT) ||
- (last_ae == I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE) ||
- (last_ae == I40IW_AE_LLP_CONNECTION_RESET) ||
- iwdev->reset)) {
- issue_close = 1;
- iwqp->cm_id = NULL;
- if (!iwqp->flush_issued) {
- iwqp->flush_issued = 1;
- issue_flush = 1;
- }
- }
-
- spin_unlock_irqrestore(&iwqp->lock, flags);
- if (issue_flush && !iwqp->destroyed) {
- /* Flush the queues */
- i40iw_flush_wqes(iwdev, iwqp);
-
- if (qp->term_flags && iwqp->ibqp.event_handler) {
- ibevent.device = iwqp->ibqp.device;
- ibevent.event = (qp->eventtype == TERM_EVENT_QP_FATAL) ?
- IB_EVENT_QP_FATAL : IB_EVENT_QP_ACCESS_ERR;
- ibevent.element.qp = &iwqp->ibqp;
- iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
- }
- }
-
- if (cm_id && cm_id->event_handler) {
- if (issue_disconn) {
- ret = i40iw_send_cm_event(NULL,
- cm_id,
- IW_CM_EVENT_DISCONNECT,
- disconn_status);
-
- if (ret)
- i40iw_debug(&iwdev->sc_dev,
- I40IW_DEBUG_CM,
- "disconnect event failed %s: - cm_id = %p\n",
- __func__, cm_id);
- }
- if (issue_close) {
- i40iw_qp_disconnect(iwqp);
- cm_id->provider_data = iwqp;
- ret = i40iw_send_cm_event(NULL, cm_id, IW_CM_EVENT_CLOSE, 0);
- if (ret)
- i40iw_debug(&iwdev->sc_dev,
- I40IW_DEBUG_CM,
- "close event failed %s: - cm_id = %p\n",
- __func__, cm_id);
- cm_id->rem_ref(cm_id);
- }
- }
-}
-
-/**
- * i40iw_disconnect_worker - worker for connection close
- * @work: points or disconn structure
- */
-static void i40iw_disconnect_worker(struct work_struct *work)
-{
- struct disconn_work *dwork = container_of(work, struct disconn_work, work);
- struct i40iw_qp *iwqp = dwork->iwqp;
-
- kfree(dwork);
- i40iw_cm_disconn_true(iwqp);
- i40iw_rem_ref(&iwqp->ibqp);
-}
-
-/**
- * i40iw_accept - registered call for connection to be accepted
- * @cm_id: cm information for passive connection
- * @conn_param: accpet parameters
- */
-int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
-{
- struct ib_qp *ibqp;
- struct i40iw_qp *iwqp;
- struct i40iw_device *iwdev;
- struct i40iw_sc_dev *dev;
- struct i40iw_cm_core *cm_core;
- struct i40iw_cm_node *cm_node;
- struct ib_qp_attr attr;
- int passive_state;
- struct ib_mr *ibmr;
- struct i40iw_pd *iwpd;
- u16 buf_len = 0;
- struct i40iw_kmem_info accept;
- enum i40iw_status_code status;
- u64 tagged_offset;
- unsigned long flags;
-
- memset(&attr, 0, sizeof(attr));
- ibqp = i40iw_get_qp(cm_id->device, conn_param->qpn);
- if (!ibqp)
- return -EINVAL;
-
- iwqp = to_iwqp(ibqp);
- iwdev = iwqp->iwdev;
- dev = &iwdev->sc_dev;
- cm_core = &iwdev->cm_core;
- cm_node = (struct i40iw_cm_node *)cm_id->provider_data;
-
- if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) {
- cm_node->ipv4 = true;
- cm_node->vlan_id = i40iw_get_vlan_ipv4(cm_node->loc_addr);
- } else {
- cm_node->ipv4 = false;
- i40iw_netdev_vlan_ipv6(cm_node->loc_addr, &cm_node->vlan_id);
- }
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "Accept vlan_id=%d\n",
- cm_node->vlan_id);
- if (cm_node->state == I40IW_CM_STATE_LISTENER_DESTROYED) {
- if (cm_node->loopbackpartner)
- i40iw_rem_ref_cm_node(cm_node->loopbackpartner);
- i40iw_rem_ref_cm_node(cm_node);
- return -EINVAL;
- }
-
- passive_state = atomic_add_return(1, &cm_node->passive_state);
- if (passive_state == I40IW_SEND_RESET_EVENT) {
- i40iw_rem_ref_cm_node(cm_node);
- return -ECONNRESET;
- }
-
- cm_node->cm_core->stats_accepts++;
- iwqp->cm_node = (void *)cm_node;
- cm_node->iwqp = iwqp;
-
- buf_len = conn_param->private_data_len + I40IW_MAX_IETF_SIZE;
-
- status = i40iw_allocate_dma_mem(dev->hw, &iwqp->ietf_mem, buf_len, 1);
-
- if (status)
- return -ENOMEM;
- cm_node->pdata.size = conn_param->private_data_len;
- accept.addr = iwqp->ietf_mem.va;
- accept.size = i40iw_cm_build_mpa_frame(cm_node, &accept, MPA_KEY_REPLY);
- memcpy(accept.addr + accept.size, conn_param->private_data,
- conn_param->private_data_len);
-
- /* setup our first outgoing iWarp send WQE (the IETF frame response) */
- if ((cm_node->ipv4 &&
- !i40iw_ipv4_is_loopback(cm_node->loc_addr[0], cm_node->rem_addr[0])) ||
- (!cm_node->ipv4 &&
- !i40iw_ipv6_is_loopback(cm_node->loc_addr, cm_node->rem_addr))) {
- iwpd = iwqp->iwpd;
- tagged_offset = (uintptr_t)iwqp->ietf_mem.va;
- ibmr = i40iw_reg_phys_mr(&iwpd->ibpd,
- iwqp->ietf_mem.pa,
- buf_len,
- IB_ACCESS_LOCAL_WRITE,
- &tagged_offset);
- if (IS_ERR(ibmr)) {
- i40iw_free_dma_mem(dev->hw, &iwqp->ietf_mem);
- return -ENOMEM;
- }
-
- ibmr->pd = &iwpd->ibpd;
- ibmr->device = iwpd->ibpd.device;
- iwqp->lsmm_mr = ibmr;
- if (iwqp->page)
- iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page);
- dev->iw_priv_qp_ops->qp_send_lsmm(&iwqp->sc_qp,
- iwqp->ietf_mem.va,
- (accept.size + conn_param->private_data_len),
- ibmr->lkey);
-
- } else {
- if (iwqp->page)
- iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page);
- dev->iw_priv_qp_ops->qp_send_lsmm(&iwqp->sc_qp, NULL, 0, 0);
- }
-
- if (iwqp->page)
- kunmap(iwqp->page);
-
- iwqp->cm_id = cm_id;
- cm_node->cm_id = cm_id;
-
- cm_id->provider_data = (void *)iwqp;
- iwqp->active_conn = 0;
-
- cm_node->lsmm_size = accept.size + conn_param->private_data_len;
- i40iw_cm_init_tsa_conn(iwqp, cm_node);
- cm_id->add_ref(cm_id);
- i40iw_add_ref(&iwqp->ibqp);
-
- attr.qp_state = IB_QPS_RTS;
- cm_node->qhash_set = false;
- i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
-
- cm_node->accelerated = true;
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- list_move_tail(&cm_node->list, &cm_core->accelerated_list);
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-
- status =
- i40iw_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0);
- if (status)
- i40iw_debug(dev, I40IW_DEBUG_CM, "error sending cm event - ESTABLISHED\n");
-
- if (cm_node->loopbackpartner) {
- cm_node->loopbackpartner->pdata.size = conn_param->private_data_len;
-
- /* copy entire MPA frame to our cm_node's frame */
- memcpy(cm_node->loopbackpartner->pdata_buf,
- conn_param->private_data,
- conn_param->private_data_len);
- i40iw_create_event(cm_node->loopbackpartner, I40IW_CM_EVENT_CONNECTED);
- }
-
- if (cm_node->accept_pend) {
- atomic_dec(&cm_node->listener->pend_accepts_cnt);
- cm_node->accept_pend = 0;
- }
- return 0;
-}
-
-/**
- * i40iw_reject - registered call for connection to be rejected
- * @cm_id: cm information for passive connection
- * @pdata: private data to be sent
- * @pdata_len: private data length
- */
-int i40iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
-{
- struct i40iw_device *iwdev;
- struct i40iw_cm_node *cm_node;
- struct i40iw_cm_node *loopback;
-
- cm_node = (struct i40iw_cm_node *)cm_id->provider_data;
- loopback = cm_node->loopbackpartner;
- cm_node->cm_id = cm_id;
- cm_node->pdata.size = pdata_len;
-
- iwdev = to_iwdev(cm_id->device);
- if (!iwdev)
- return -EINVAL;
- cm_node->cm_core->stats_rejects++;
-
- if (pdata_len + sizeof(struct ietf_mpa_v2) > MAX_CM_BUFFER)
- return -EINVAL;
-
- if (loopback) {
- memcpy(&loopback->pdata_buf, pdata, pdata_len);
- loopback->pdata.size = pdata_len;
- }
-
- return i40iw_cm_reject(cm_node, pdata, pdata_len);
-}
-
-/**
- * i40iw_connect - registered call for connection to be established
- * @cm_id: cm information for passive connection
- * @conn_param: Information about the connection
- */
-int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
-{
- struct ib_qp *ibqp;
- struct i40iw_qp *iwqp;
- struct i40iw_device *iwdev;
- struct i40iw_cm_node *cm_node;
- struct i40iw_cm_info cm_info;
- struct sockaddr_in *laddr;
- struct sockaddr_in *raddr;
- struct sockaddr_in6 *laddr6;
- struct sockaddr_in6 *raddr6;
- int ret = 0;
-
- ibqp = i40iw_get_qp(cm_id->device, conn_param->qpn);
- if (!ibqp)
- return -EINVAL;
- iwqp = to_iwqp(ibqp);
- if (!iwqp)
- return -EINVAL;
- iwdev = to_iwdev(iwqp->ibqp.device);
- if (!iwdev)
- return -EINVAL;
-
- laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
- raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
- laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
- raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr;
-
- if (!(laddr->sin_port) || !(raddr->sin_port))
- return -EINVAL;
-
- iwqp->active_conn = 1;
- iwqp->cm_id = NULL;
- cm_id->provider_data = iwqp;
-
- /* set up the connection params for the node */
- if (cm_id->remote_addr.ss_family == AF_INET) {
- cm_info.ipv4 = true;
- memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr));
- memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr));
- cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
- cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr);
- cm_info.loc_port = ntohs(laddr->sin_port);
- cm_info.rem_port = ntohs(raddr->sin_port);
- cm_info.vlan_id = i40iw_get_vlan_ipv4(cm_info.loc_addr);
- } else {
- cm_info.ipv4 = false;
- i40iw_copy_ip_ntohl(cm_info.loc_addr,
- laddr6->sin6_addr.in6_u.u6_addr32);
- i40iw_copy_ip_ntohl(cm_info.rem_addr,
- raddr6->sin6_addr.in6_u.u6_addr32);
- cm_info.loc_port = ntohs(laddr6->sin6_port);
- cm_info.rem_port = ntohs(raddr6->sin6_port);
- i40iw_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id);
- }
- cm_info.cm_id = cm_id;
- cm_info.tos = cm_id->tos;
- cm_info.user_pri = rt_tos2priority(cm_id->tos);
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_DCB, "%s TOS:[%d] UP:[%d]\n",
- __func__, cm_id->tos, cm_info.user_pri);
- cm_id->add_ref(cm_id);
- cm_node = i40iw_create_cm_node(&iwdev->cm_core, iwdev,
- conn_param, &cm_info);
-
- if (IS_ERR(cm_node)) {
- ret = PTR_ERR(cm_node);
- cm_id->rem_ref(cm_id);
- return ret;
- }
-
- if ((cm_info.ipv4 && (laddr->sin_addr.s_addr != raddr->sin_addr.s_addr)) ||
- (!cm_info.ipv4 && memcmp(laddr6->sin6_addr.in6_u.u6_addr32,
- raddr6->sin6_addr.in6_u.u6_addr32,
- sizeof(laddr6->sin6_addr.in6_u.u6_addr32)))) {
- if (i40iw_manage_qhash(iwdev, &cm_info, I40IW_QHASH_TYPE_TCP_ESTABLISHED,
- I40IW_QHASH_MANAGE_TYPE_ADD, NULL, true)) {
- ret = -EINVAL;
- goto err;
- }
- cm_node->qhash_set = true;
- }
-
- if (i40iw_manage_apbvt(iwdev, cm_info.loc_port,
- I40IW_MANAGE_APBVT_ADD)) {
- ret = -EINVAL;
- goto err;
- }
-
- cm_node->apbvt_set = true;
- iwqp->cm_node = cm_node;
- cm_node->iwqp = iwqp;
- iwqp->cm_id = cm_id;
- i40iw_add_ref(&iwqp->ibqp);
-
- if (cm_node->state != I40IW_CM_STATE_OFFLOADED) {
- cm_node->state = I40IW_CM_STATE_SYN_SENT;
- ret = i40iw_send_syn(cm_node, 0);
- if (ret)
- goto err;
- }
-
- if (cm_node->loopbackpartner) {
- cm_node->loopbackpartner->state = I40IW_CM_STATE_MPAREQ_RCVD;
- i40iw_create_event(cm_node->loopbackpartner,
- I40IW_CM_EVENT_MPA_REQ);
- }
-
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "Api - connect(): port=0x%04x, cm_node=%p, cm_id = %p.\n",
- cm_node->rem_port,
- cm_node,
- cm_node->cm_id);
-
- return 0;
-
-err:
- if (cm_info.ipv4)
- i40iw_debug(&iwdev->sc_dev,
- I40IW_DEBUG_CM,
- "Api - connect() FAILED: dest addr=%pI4",
- cm_info.rem_addr);
- else
- i40iw_debug(&iwdev->sc_dev,
- I40IW_DEBUG_CM,
- "Api - connect() FAILED: dest addr=%pI6",
- cm_info.rem_addr);
-
- i40iw_rem_ref_cm_node(cm_node);
- cm_id->rem_ref(cm_id);
- iwdev->cm_core.stats_connect_errs++;
- return ret;
-}
-
-/**
- * i40iw_create_listen - registered call creating listener
- * @cm_id: cm information for passive connection
- * @backlog: to max accept pending count
- */
-int i40iw_create_listen(struct iw_cm_id *cm_id, int backlog)
-{
- struct i40iw_device *iwdev;
- struct i40iw_cm_listener *cm_listen_node;
- struct i40iw_cm_info cm_info;
- enum i40iw_status_code ret;
- struct sockaddr_in *laddr;
- struct sockaddr_in6 *laddr6;
- bool wildcard = false;
-
- iwdev = to_iwdev(cm_id->device);
- if (!iwdev)
- return -EINVAL;
-
- laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
- laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
- memset(&cm_info, 0, sizeof(cm_info));
- if (laddr->sin_family == AF_INET) {
- cm_info.ipv4 = true;
- cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
- cm_info.loc_port = ntohs(laddr->sin_port);
-
- if (laddr->sin_addr.s_addr != INADDR_ANY)
- cm_info.vlan_id = i40iw_get_vlan_ipv4(cm_info.loc_addr);
- else
- wildcard = true;
-
- } else {
- cm_info.ipv4 = false;
- i40iw_copy_ip_ntohl(cm_info.loc_addr,
- laddr6->sin6_addr.in6_u.u6_addr32);
- cm_info.loc_port = ntohs(laddr6->sin6_port);
- if (ipv6_addr_type(&laddr6->sin6_addr) != IPV6_ADDR_ANY)
- i40iw_netdev_vlan_ipv6(cm_info.loc_addr,
- &cm_info.vlan_id);
- else
- wildcard = true;
- }
- cm_info.backlog = backlog;
- cm_info.cm_id = cm_id;
-
- cm_listen_node = i40iw_make_listen_node(&iwdev->cm_core, iwdev, &cm_info);
- if (!cm_listen_node) {
- i40iw_pr_err("cm_listen_node == NULL\n");
- return -ENOMEM;
- }
-
- cm_id->provider_data = cm_listen_node;
-
- cm_listen_node->tos = cm_id->tos;
- cm_listen_node->user_pri = rt_tos2priority(cm_id->tos);
- cm_info.user_pri = cm_listen_node->user_pri;
-
- if (!cm_listen_node->reused_node) {
- if (wildcard) {
- if (cm_info.ipv4)
- ret = i40iw_add_mqh_4(iwdev,
- &cm_info,
- cm_listen_node);
- else
- ret = i40iw_add_mqh_6(iwdev,
- &cm_info,
- cm_listen_node);
- if (ret)
- goto error;
-
- ret = i40iw_manage_apbvt(iwdev,
- cm_info.loc_port,
- I40IW_MANAGE_APBVT_ADD);
-
- if (ret)
- goto error;
- } else {
- ret = i40iw_manage_qhash(iwdev,
- &cm_info,
- I40IW_QHASH_TYPE_TCP_SYN,
- I40IW_QHASH_MANAGE_TYPE_ADD,
- NULL,
- true);
- if (ret)
- goto error;
- cm_listen_node->qhash_set = true;
- ret = i40iw_manage_apbvt(iwdev,
- cm_info.loc_port,
- I40IW_MANAGE_APBVT_ADD);
- if (ret)
- goto error;
- }
- }
- cm_id->add_ref(cm_id);
- cm_listen_node->cm_core->stats_listen_created++;
- return 0;
- error:
- i40iw_cm_del_listen(&iwdev->cm_core, (void *)cm_listen_node, false);
- return -EINVAL;
-}
-
-/**
- * i40iw_destroy_listen - registered call to destroy listener
- * @cm_id: cm information for passive connection
- */
-int i40iw_destroy_listen(struct iw_cm_id *cm_id)
-{
- struct i40iw_device *iwdev;
-
- iwdev = to_iwdev(cm_id->device);
- if (cm_id->provider_data)
- i40iw_cm_del_listen(&iwdev->cm_core, cm_id->provider_data, true);
- else
- i40iw_pr_err("cm_id->provider_data was NULL\n");
-
- cm_id->rem_ref(cm_id);
-
- return 0;
-}
-
-/**
- * i40iw_cm_event_connected - handle connected active node
- * @event: the info for cm_node of connection
- */
-static void i40iw_cm_event_connected(struct i40iw_cm_event *event)
-{
- struct i40iw_qp *iwqp;
- struct i40iw_device *iwdev;
- struct i40iw_cm_core *cm_core;
- struct i40iw_cm_node *cm_node;
- struct i40iw_sc_dev *dev;
- struct ib_qp_attr attr;
- struct iw_cm_id *cm_id;
- unsigned long flags;
- int status;
- bool read0;
-
- cm_node = event->cm_node;
- cm_id = cm_node->cm_id;
- iwqp = (struct i40iw_qp *)cm_id->provider_data;
- iwdev = to_iwdev(iwqp->ibqp.device);
- dev = &iwdev->sc_dev;
- cm_core = &iwdev->cm_core;
-
- if (iwqp->destroyed) {
- status = -ETIMEDOUT;
- goto error;
- }
- i40iw_cm_init_tsa_conn(iwqp, cm_node);
- read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO);
- if (iwqp->page)
- iwqp->sc_qp.qp_uk.sq_base = kmap(iwqp->page);
- dev->iw_priv_qp_ops->qp_send_rtt(&iwqp->sc_qp, read0);
- if (iwqp->page)
- kunmap(iwqp->page);
-
- memset(&attr, 0, sizeof(attr));
- attr.qp_state = IB_QPS_RTS;
- cm_node->qhash_set = false;
- i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
-
- cm_node->accelerated = true;
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- list_move_tail(&cm_node->list, &cm_core->accelerated_list);
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
- status = i40iw_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,
- 0);
- if (status)
- i40iw_debug(dev, I40IW_DEBUG_CM, "error sending cm event - CONNECT_REPLY\n");
-
- return;
-
-error:
- iwqp->cm_id = NULL;
- cm_id->provider_data = NULL;
- i40iw_send_cm_event(event->cm_node,
- cm_id,
- IW_CM_EVENT_CONNECT_REPLY,
- status);
- cm_id->rem_ref(cm_id);
- i40iw_rem_ref_cm_node(event->cm_node);
-}
-
-/**
- * i40iw_cm_event_reset - handle reset
- * @event: the info for cm_node of connection
- */
-static void i40iw_cm_event_reset(struct i40iw_cm_event *event)
-{
- struct i40iw_cm_node *cm_node = event->cm_node;
- struct iw_cm_id *cm_id = cm_node->cm_id;
- struct i40iw_qp *iwqp;
-
- if (!cm_id)
- return;
-
- iwqp = cm_id->provider_data;
- if (!iwqp)
- return;
-
- i40iw_debug(cm_node->dev,
- I40IW_DEBUG_CM,
- "reset event %p - cm_id = %p\n",
- event->cm_node, cm_id);
- iwqp->cm_id = NULL;
-
- i40iw_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT, -ECONNRESET);
- i40iw_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CLOSE, 0);
-}
-
-/**
- * i40iw_cm_event_handler - worker thread callback to send event to cm upper layer
- * @work: pointer of cm event info.
- */
-static void i40iw_cm_event_handler(struct work_struct *work)
-{
- struct i40iw_cm_event *event = container_of(work,
- struct i40iw_cm_event,
- event_work);
- struct i40iw_cm_node *cm_node;
-
- if (!event || !event->cm_node || !event->cm_node->cm_core)
- return;
-
- cm_node = event->cm_node;
-
- switch (event->type) {
- case I40IW_CM_EVENT_MPA_REQ:
- i40iw_send_cm_event(cm_node,
- cm_node->cm_id,
- IW_CM_EVENT_CONNECT_REQUEST,
- 0);
- break;
- case I40IW_CM_EVENT_RESET:
- i40iw_cm_event_reset(event);
- break;
- case I40IW_CM_EVENT_CONNECTED:
- if (!event->cm_node->cm_id ||
- (event->cm_node->state != I40IW_CM_STATE_OFFLOADED))
- break;
- i40iw_cm_event_connected(event);
- break;
- case I40IW_CM_EVENT_MPA_REJECT:
- if (!event->cm_node->cm_id ||
- (cm_node->state == I40IW_CM_STATE_OFFLOADED))
- break;
- i40iw_send_cm_event(cm_node,
- cm_node->cm_id,
- IW_CM_EVENT_CONNECT_REPLY,
- -ECONNREFUSED);
- break;
- case I40IW_CM_EVENT_ABORTED:
- if (!event->cm_node->cm_id ||
- (event->cm_node->state == I40IW_CM_STATE_OFFLOADED))
- break;
- i40iw_event_connect_error(event);
- break;
- default:
- i40iw_pr_err("event type = %d\n", event->type);
- break;
- }
-
- event->cm_info.cm_id->rem_ref(event->cm_info.cm_id);
- i40iw_rem_ref_cm_node(event->cm_node);
- kfree(event);
-}
-
-/**
- * i40iw_cm_post_event - queue event request for worker thread
- * @event: cm node's info for up event call
- */
-static void i40iw_cm_post_event(struct i40iw_cm_event *event)
-{
- atomic_inc(&event->cm_node->ref_count);
- event->cm_info.cm_id->add_ref(event->cm_info.cm_id);
- INIT_WORK(&event->event_work, i40iw_cm_event_handler);
-
- queue_work(event->cm_node->cm_core->event_wq, &event->event_work);
-}
-
-/**
- * i40iw_qhash_ctrl - enable/disable qhash for list
- * @iwdev: device pointer
- * @parent_listen_node: parent listen node
- * @nfo: cm info node
- * @ipaddr: Pointer to IPv4 or IPv6 address
- * @ipv4: flag indicating IPv4 when true
- * @ifup: flag indicating interface up when true
- *
- * Enables or disables the qhash for the node in the child
- * listen list that matches ipaddr. If no matching IP was found
- * it will allocate and add a new child listen node to the
- * parent listen node. The listen_list_lock is assumed to be
- * held when called.
- */
-static void i40iw_qhash_ctrl(struct i40iw_device *iwdev,
- struct i40iw_cm_listener *parent_listen_node,
- struct i40iw_cm_info *nfo,
- u32 *ipaddr, bool ipv4, bool ifup)
-{
- struct list_head *child_listen_list = &parent_listen_node->child_listen_list;
- struct i40iw_cm_listener *child_listen_node;
- struct list_head *pos, *tpos;
- enum i40iw_status_code ret;
- bool node_allocated = false;
- enum i40iw_quad_hash_manage_type op =
- ifup ? I40IW_QHASH_MANAGE_TYPE_ADD : I40IW_QHASH_MANAGE_TYPE_DELETE;
-
- list_for_each_safe(pos, tpos, child_listen_list) {
- child_listen_node =
- list_entry(pos,
- struct i40iw_cm_listener,
- child_listen_list);
- if (!memcmp(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16))
- goto set_qhash;
- }
-
- /* if not found then add a child listener if interface is going up */
- if (!ifup)
- return;
- child_listen_node = kmemdup(parent_listen_node,
- sizeof(*child_listen_node), GFP_ATOMIC);
- if (!child_listen_node)
- return;
- node_allocated = true;
-
- memcpy(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16);
-
-set_qhash:
- memcpy(nfo->loc_addr,
- child_listen_node->loc_addr,
- sizeof(nfo->loc_addr));
- nfo->vlan_id = child_listen_node->vlan_id;
- ret = i40iw_manage_qhash(iwdev, nfo,
- I40IW_QHASH_TYPE_TCP_SYN,
- op,
- NULL, false);
- if (!ret) {
- child_listen_node->qhash_set = ifup;
- if (node_allocated)
- list_add(&child_listen_node->child_listen_list,
- &parent_listen_node->child_listen_list);
- } else if (node_allocated) {
- kfree(child_listen_node);
- }
-}
-
-/**
- * i40iw_cm_teardown_connections - teardown QPs
- * @iwdev: device pointer
- * @ipaddr: Pointer to IPv4 or IPv6 address
- * @ipv4: flag indicating IPv4 when true
- * @disconnect_all: flag indicating disconnect all QPs
- * teardown QPs where source or destination addr matches ip addr
- */
-void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr,
- struct i40iw_cm_info *nfo,
- bool disconnect_all)
-{
- struct i40iw_cm_core *cm_core = &iwdev->cm_core;
- struct list_head *list_core_temp;
- struct list_head *list_node;
- struct i40iw_cm_node *cm_node;
- unsigned long flags;
- struct list_head teardown_list;
- struct ib_qp_attr attr;
-
- INIT_LIST_HEAD(&teardown_list);
- spin_lock_irqsave(&cm_core->ht_lock, flags);
- list_for_each_safe(list_node, list_core_temp,
- &cm_core->accelerated_list) {
- cm_node = container_of(list_node, struct i40iw_cm_node, list);
- if (disconnect_all ||
- (nfo->vlan_id == cm_node->vlan_id &&
- (!memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16) ||
- !memcmp(cm_node->rem_addr, ipaddr, nfo->ipv4 ? 4 : 16)))) {
- atomic_inc(&cm_node->ref_count);
- list_add(&cm_node->teardown_entry, &teardown_list);
- }
- }
- list_for_each_safe(list_node, list_core_temp,
- &cm_core->non_accelerated_list) {
- cm_node = container_of(list_node, struct i40iw_cm_node, list);
- if (disconnect_all ||
- (nfo->vlan_id == cm_node->vlan_id &&
- (!memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16) ||
- !memcmp(cm_node->rem_addr, ipaddr, nfo->ipv4 ? 4 : 16)))) {
- atomic_inc(&cm_node->ref_count);
- list_add(&cm_node->teardown_entry, &teardown_list);
- }
- }
- spin_unlock_irqrestore(&cm_core->ht_lock, flags);
-
- list_for_each_safe(list_node, list_core_temp, &teardown_list) {
- cm_node = container_of(list_node, struct i40iw_cm_node,
- teardown_entry);
- attr.qp_state = IB_QPS_ERR;
- i40iw_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL);
- if (iwdev->reset)
- i40iw_cm_disconn(cm_node->iwqp);
- i40iw_rem_ref_cm_node(cm_node);
- }
-}
-
-/**
- * i40iw_ifdown_notify - process an ifdown on an interface
- * @iwdev: device pointer
- * @ipaddr: Pointer to IPv4 or IPv6 address
- * @ipv4: flag indicating IPv4 when true
- * @ifup: flag indicating interface up when true
- */
-void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,
- u32 *ipaddr, bool ipv4, bool ifup)
-{
- struct i40iw_cm_core *cm_core = &iwdev->cm_core;
- unsigned long flags;
- struct i40iw_cm_listener *listen_node;
- static const u32 ip_zero[4] = { 0, 0, 0, 0 };
- struct i40iw_cm_info nfo;
- u16 vlan_id = rdma_vlan_dev_vlan_id(netdev);
- enum i40iw_status_code ret;
- enum i40iw_quad_hash_manage_type op =
- ifup ? I40IW_QHASH_MANAGE_TYPE_ADD : I40IW_QHASH_MANAGE_TYPE_DELETE;
-
- nfo.vlan_id = vlan_id;
- nfo.ipv4 = ipv4;
-
- /* Disable or enable qhash for listeners */
- spin_lock_irqsave(&cm_core->listen_list_lock, flags);
- list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
- if (vlan_id == listen_node->vlan_id &&
- (!memcmp(listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16) ||
- !memcmp(listen_node->loc_addr, ip_zero, ipv4 ? 4 : 16))) {
- memcpy(nfo.loc_addr, listen_node->loc_addr,
- sizeof(nfo.loc_addr));
- nfo.loc_port = listen_node->loc_port;
- nfo.user_pri = listen_node->user_pri;
- if (!list_empty(&listen_node->child_listen_list)) {
- i40iw_qhash_ctrl(iwdev,
- listen_node,
- &nfo,
- ipaddr, ipv4, ifup);
- } else if (memcmp(listen_node->loc_addr, ip_zero,
- ipv4 ? 4 : 16)) {
- ret = i40iw_manage_qhash(iwdev,
- &nfo,
- I40IW_QHASH_TYPE_TCP_SYN,
- op,
- NULL,
- false);
- if (!ret)
- listen_node->qhash_set = ifup;
- }
- }
- }
- spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
-
- /* teardown connected qp's on ifdown */
- if (!ifup)
- i40iw_cm_teardown_connections(iwdev, ipaddr, &nfo, false);
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.h b/drivers/infiniband/hw/i40iw/i40iw_cm.h
deleted file mode 100644
index 6e43e4d730f4..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.h
+++ /dev/null
@@ -1,462 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_CM_H
-#define I40IW_CM_H
-
-#define QUEUE_EVENTS
-
-#define I40IW_MANAGE_APBVT_DEL 0
-#define I40IW_MANAGE_APBVT_ADD 1
-
-#define I40IW_MPA_REQUEST_ACCEPT 1
-#define I40IW_MPA_REQUEST_REJECT 2
-
-/* IETF MPA -- defines, enums, structs */
-#define IEFT_MPA_KEY_REQ "MPA ID Req Frame"
-#define IEFT_MPA_KEY_REP "MPA ID Rep Frame"
-#define IETF_MPA_KEY_SIZE 16
-#define IETF_MPA_VERSION 1
-#define IETF_MAX_PRIV_DATA_LEN 512
-#define IETF_MPA_FRAME_SIZE 20
-#define IETF_RTR_MSG_SIZE 4
-#define IETF_MPA_V2_FLAG 0x10
-#define SNDMARKER_SEQNMASK 0x000001FF
-
-#define I40IW_MAX_IETF_SIZE 32
-
-/* IETF RTR MSG Fields */
-#define IETF_PEER_TO_PEER 0x8000
-#define IETF_FLPDU_ZERO_LEN 0x4000
-#define IETF_RDMA0_WRITE 0x8000
-#define IETF_RDMA0_READ 0x4000
-#define IETF_NO_IRD_ORD 0x3FFF
-
-/* HW-supported IRD sizes*/
-#define I40IW_HW_IRD_SETTING_2 2
-#define I40IW_HW_IRD_SETTING_4 4
-#define I40IW_HW_IRD_SETTING_8 8
-#define I40IW_HW_IRD_SETTING_16 16
-#define I40IW_HW_IRD_SETTING_32 32
-#define I40IW_HW_IRD_SETTING_64 64
-
-#define MAX_PORTS 65536
-#define I40IW_VLAN_PRIO_SHIFT 13
-
-enum ietf_mpa_flags {
- IETF_MPA_FLAGS_MARKERS = 0x80, /* receive Markers */
- IETF_MPA_FLAGS_CRC = 0x40, /* receive Markers */
- IETF_MPA_FLAGS_REJECT = 0x20, /* Reject */
-};
-
-struct ietf_mpa_v1 {
- u8 key[IETF_MPA_KEY_SIZE];
- u8 flags;
- u8 rev;
- __be16 priv_data_len;
- u8 priv_data[];
-};
-
-#define ietf_mpa_req_resp_frame ietf_mpa_frame
-
-struct ietf_rtr_msg {
- __be16 ctrl_ird;
- __be16 ctrl_ord;
-};
-
-struct ietf_mpa_v2 {
- u8 key[IETF_MPA_KEY_SIZE];
- u8 flags;
- u8 rev;
- __be16 priv_data_len;
- struct ietf_rtr_msg rtr_msg;
- u8 priv_data[];
-};
-
-struct i40iw_cm_node;
-enum i40iw_timer_type {
- I40IW_TIMER_TYPE_SEND,
- I40IW_TIMER_TYPE_RECV,
- I40IW_TIMER_NODE_CLEANUP,
- I40IW_TIMER_TYPE_CLOSE,
-};
-
-#define I40IW_PASSIVE_STATE_INDICATED 0
-#define I40IW_DO_NOT_SEND_RESET_EVENT 1
-#define I40IW_SEND_RESET_EVENT 2
-
-#define MAX_I40IW_IFS 4
-
-#define SET_ACK 0x1
-#define SET_SYN 0x2
-#define SET_FIN 0x4
-#define SET_RST 0x8
-
-#define TCP_OPTIONS_PADDING 3
-
-struct option_base {
- u8 optionnum;
- u8 length;
-};
-
-enum option_numbers {
- OPTION_NUMBER_END,
- OPTION_NUMBER_NONE,
- OPTION_NUMBER_MSS,
- OPTION_NUMBER_WINDOW_SCALE,
- OPTION_NUMBER_SACK_PERM,
- OPTION_NUMBER_SACK,
- OPTION_NUMBER_WRITE0 = 0xbc
-};
-
-struct option_mss {
- u8 optionnum;
- u8 length;
- __be16 mss;
-};
-
-struct option_windowscale {
- u8 optionnum;
- u8 length;
- u8 shiftcount;
-};
-
-union all_known_options {
- char as_end;
- struct option_base as_base;
- struct option_mss as_mss;
- struct option_windowscale as_windowscale;
-};
-
-struct i40iw_timer_entry {
- struct list_head list;
- unsigned long timetosend; /* jiffies */
- struct i40iw_puda_buf *sqbuf;
- u32 type;
- u32 retrycount;
- u32 retranscount;
- u32 context;
- u32 send_retrans;
- int close_when_complete;
-};
-
-#define I40IW_DEFAULT_RETRYS 64
-#define I40IW_DEFAULT_RETRANS 8
-#define I40IW_DEFAULT_TTL 0x40
-#define I40IW_DEFAULT_RTT_VAR 0x6
-#define I40IW_DEFAULT_SS_THRESH 0x3FFFFFFF
-#define I40IW_DEFAULT_REXMIT_THRESH 8
-
-#define I40IW_RETRY_TIMEOUT HZ
-#define I40IW_SHORT_TIME 10
-#define I40IW_LONG_TIME (2 * HZ)
-#define I40IW_MAX_TIMEOUT ((unsigned long)(12 * HZ))
-
-#define I40IW_CM_HASHTABLE_SIZE 1024
-#define I40IW_CM_TCP_TIMER_INTERVAL 3000
-#define I40IW_CM_DEFAULT_MTU 1540
-#define I40IW_CM_DEFAULT_FRAME_CNT 10
-#define I40IW_CM_THREAD_STACK_SIZE 256
-#define I40IW_CM_DEFAULT_RCV_WND 64240
-#define I40IW_CM_DEFAULT_RCV_WND_SCALED 0x3fffc
-#define I40IW_CM_DEFAULT_RCV_WND_SCALE 2
-#define I40IW_CM_DEFAULT_FREE_PKTS 0x000A
-#define I40IW_CM_FREE_PKT_LO_WATERMARK 2
-
-#define I40IW_CM_DEFAULT_MSS 536
-
-#define I40IW_CM_DEF_SEQ 0x159bf75f
-#define I40IW_CM_DEF_LOCAL_ID 0x3b47
-
-#define I40IW_CM_DEF_SEQ2 0x18ed5740
-#define I40IW_CM_DEF_LOCAL_ID2 0xb807
-#define MAX_CM_BUFFER (I40IW_MAX_IETF_SIZE + IETF_MAX_PRIV_DATA_LEN)
-
-typedef u32 i40iw_addr_t;
-
-#define i40iw_cm_tsa_context i40iw_qp_context
-
-struct i40iw_qp;
-
-/* cm node transition states */
-enum i40iw_cm_node_state {
- I40IW_CM_STATE_UNKNOWN,
- I40IW_CM_STATE_INITED,
- I40IW_CM_STATE_LISTENING,
- I40IW_CM_STATE_SYN_RCVD,
- I40IW_CM_STATE_SYN_SENT,
- I40IW_CM_STATE_ONE_SIDE_ESTABLISHED,
- I40IW_CM_STATE_ESTABLISHED,
- I40IW_CM_STATE_ACCEPTING,
- I40IW_CM_STATE_MPAREQ_SENT,
- I40IW_CM_STATE_MPAREQ_RCVD,
- I40IW_CM_STATE_MPAREJ_RCVD,
- I40IW_CM_STATE_OFFLOADED,
- I40IW_CM_STATE_FIN_WAIT1,
- I40IW_CM_STATE_FIN_WAIT2,
- I40IW_CM_STATE_CLOSE_WAIT,
- I40IW_CM_STATE_TIME_WAIT,
- I40IW_CM_STATE_LAST_ACK,
- I40IW_CM_STATE_CLOSING,
- I40IW_CM_STATE_LISTENER_DESTROYED,
- I40IW_CM_STATE_CLOSED
-};
-
-enum mpa_frame_version {
- IETF_MPA_V1 = 1,
- IETF_MPA_V2 = 2
-};
-
-enum mpa_frame_key {
- MPA_KEY_REQUEST,
- MPA_KEY_REPLY
-};
-
-enum send_rdma0 {
- SEND_RDMA_READ_ZERO = 1,
- SEND_RDMA_WRITE_ZERO = 2
-};
-
-enum i40iw_tcpip_pkt_type {
- I40IW_PKT_TYPE_UNKNOWN,
- I40IW_PKT_TYPE_SYN,
- I40IW_PKT_TYPE_SYNACK,
- I40IW_PKT_TYPE_ACK,
- I40IW_PKT_TYPE_FIN,
- I40IW_PKT_TYPE_RST
-};
-
-/* CM context params */
-struct i40iw_cm_tcp_context {
- u8 client;
-
- u32 loc_seq_num;
- u32 loc_ack_num;
- u32 rem_ack_num;
- u32 rcv_nxt;
-
- u32 loc_id;
- u32 rem_id;
-
- u32 snd_wnd;
- u32 max_snd_wnd;
-
- u32 rcv_wnd;
- u32 mss;
- u8 snd_wscale;
- u8 rcv_wscale;
-};
-
-enum i40iw_cm_listener_state {
- I40IW_CM_LISTENER_PASSIVE_STATE = 1,
- I40IW_CM_LISTENER_ACTIVE_STATE = 2,
- I40IW_CM_LISTENER_EITHER_STATE = 3
-};
-
-struct i40iw_cm_listener {
- struct list_head list;
- struct i40iw_cm_core *cm_core;
- u8 loc_mac[ETH_ALEN];
- u32 loc_addr[4];
- u16 loc_port;
- struct iw_cm_id *cm_id;
- atomic_t ref_count;
- struct i40iw_device *iwdev;
- atomic_t pend_accepts_cnt;
- int backlog;
- enum i40iw_cm_listener_state listener_state;
- u32 reused_node;
- u8 user_pri;
- u8 tos;
- u16 vlan_id;
- bool qhash_set;
- bool ipv4;
- struct list_head child_listen_list;
-
-};
-
-struct i40iw_kmem_info {
- void *addr;
- u32 size;
-};
-
-/* per connection node and node state information */
-struct i40iw_cm_node {
- u32 loc_addr[4], rem_addr[4];
- u16 loc_port, rem_port;
- u16 vlan_id;
- enum i40iw_cm_node_state state;
- u8 loc_mac[ETH_ALEN];
- u8 rem_mac[ETH_ALEN];
- atomic_t ref_count;
- struct i40iw_qp *iwqp;
- struct i40iw_device *iwdev;
- struct i40iw_sc_dev *dev;
- struct i40iw_cm_tcp_context tcp_cntxt;
- struct i40iw_cm_core *cm_core;
- struct i40iw_cm_node *loopbackpartner;
- struct i40iw_timer_entry *send_entry;
- struct i40iw_timer_entry *close_entry;
- spinlock_t retrans_list_lock; /* cm transmit packet */
- enum send_rdma0 send_rdma0_op;
- u16 ird_size;
- u16 ord_size;
- u16 mpav2_ird_ord;
- struct iw_cm_id *cm_id;
- struct list_head list;
- bool accelerated;
- struct i40iw_cm_listener *listener;
- int apbvt_set;
- int accept_pend;
- struct list_head timer_entry;
- struct list_head reset_entry;
- struct list_head teardown_entry;
- atomic_t passive_state;
- bool qhash_set;
- u8 user_pri;
- u8 tos;
- bool ipv4;
- bool snd_mark_en;
- u16 lsmm_size;
- enum mpa_frame_version mpa_frame_rev;
- struct i40iw_kmem_info pdata;
- union {
- struct ietf_mpa_v1 mpa_frame;
- struct ietf_mpa_v2 mpa_v2_frame;
- };
-
- u8 pdata_buf[IETF_MAX_PRIV_DATA_LEN];
- struct i40iw_kmem_info mpa_hdr;
- bool ack_rcvd;
-};
-
-/* structure for client or CM to fill when making CM api calls. */
-/* - only need to set relevant data, based on op. */
-struct i40iw_cm_info {
- struct iw_cm_id *cm_id;
- u16 loc_port;
- u16 rem_port;
- u32 loc_addr[4];
- u32 rem_addr[4];
- u16 vlan_id;
- int backlog;
- u8 user_pri;
- u8 tos;
- bool ipv4;
-};
-
-/* CM event codes */
-enum i40iw_cm_event_type {
- I40IW_CM_EVENT_UNKNOWN,
- I40IW_CM_EVENT_ESTABLISHED,
- I40IW_CM_EVENT_MPA_REQ,
- I40IW_CM_EVENT_MPA_CONNECT,
- I40IW_CM_EVENT_MPA_ACCEPT,
- I40IW_CM_EVENT_MPA_REJECT,
- I40IW_CM_EVENT_MPA_ESTABLISHED,
- I40IW_CM_EVENT_CONNECTED,
- I40IW_CM_EVENT_RESET,
- I40IW_CM_EVENT_ABORTED
-};
-
-/* event to post to CM event handler */
-struct i40iw_cm_event {
- enum i40iw_cm_event_type type;
- struct i40iw_cm_info cm_info;
- struct work_struct event_work;
- struct i40iw_cm_node *cm_node;
-};
-
-struct i40iw_cm_core {
- struct i40iw_device *iwdev;
- struct i40iw_sc_dev *dev;
-
- struct list_head listen_nodes;
- struct list_head accelerated_list;
- struct list_head non_accelerated_list;
-
- struct timer_list tcp_timer;
-
- struct workqueue_struct *event_wq;
- struct workqueue_struct *disconn_wq;
-
- spinlock_t ht_lock; /* manage hash table */
- spinlock_t listen_list_lock; /* listen list */
- spinlock_t apbvt_lock; /*manage apbvt entries*/
-
- unsigned long ports_in_use[BITS_TO_LONGS(MAX_PORTS)];
-
- u64 stats_nodes_created;
- u64 stats_nodes_destroyed;
- u64 stats_listen_created;
- u64 stats_listen_destroyed;
- u64 stats_listen_nodes_created;
- u64 stats_listen_nodes_destroyed;
- u64 stats_loopbacks;
- u64 stats_accepts;
- u64 stats_rejects;
- u64 stats_connect_errs;
- u64 stats_passive_errs;
- u64 stats_pkt_retrans;
- u64 stats_backlog_drops;
-};
-
-int i40iw_schedule_cm_timer(struct i40iw_cm_node *cm_node,
- struct i40iw_puda_buf *sqbuf,
- enum i40iw_timer_type type,
- int send_retrans,
- int close_when_complete);
-
-int i40iw_accept(struct iw_cm_id *, struct iw_cm_conn_param *);
-int i40iw_reject(struct iw_cm_id *, const void *, u8);
-int i40iw_connect(struct iw_cm_id *, struct iw_cm_conn_param *);
-int i40iw_create_listen(struct iw_cm_id *, int);
-int i40iw_destroy_listen(struct iw_cm_id *);
-
-int i40iw_cm_start(struct i40iw_device *);
-int i40iw_cm_stop(struct i40iw_device *);
-
-int i40iw_arp_table(struct i40iw_device *iwdev,
- u32 *ip_addr,
- bool ipv4,
- u8 *mac_addr,
- u32 action);
-
-void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,
- u32 *ipaddr, bool ipv4, bool ifup);
-void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr,
- struct i40iw_cm_info *nfo,
- bool disconnect_all);
-bool i40iw_port_in_use(struct i40iw_cm_core *cm_core, u16 port);
-#endif /* I40IW_CM_H */
diff --git a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
deleted file mode 100644
index 688f19667221..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
+++ /dev/null
@@ -1,5294 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include "i40iw_osdep.h"
-#include "i40iw_register.h"
-#include "i40iw_status.h"
-#include "i40iw_hmc.h"
-
-#include "i40iw_d.h"
-#include "i40iw_type.h"
-#include "i40iw_p.h"
-#include "i40iw_vf.h"
-#include "i40iw_virtchnl.h"
-
-/**
- * i40iw_insert_wqe_hdr - write wqe header
- * @wqe: cqp wqe for header
- * @header: header for the cqp wqe
- */
-void i40iw_insert_wqe_hdr(u64 *wqe, u64 header)
-{
- wmb(); /* make sure WQE is populated before polarity is set */
- set_64bit_val(wqe, 24, header);
-}
-
-void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev)
-{
- if (cqp_timeout->compl_cqp_cmds != dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]) {
- cqp_timeout->compl_cqp_cmds = dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS];
- cqp_timeout->count = 0;
- } else {
- if (dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] != cqp_timeout->compl_cqp_cmds)
- cqp_timeout->count++;
- }
-}
-
-/**
- * i40iw_get_cqp_reg_info - get head and tail for cqp using registers
- * @cqp: struct for cqp hw
- * @val: cqp tail register value
- * @tail:wqtail register value
- * @error: cqp processing err
- */
-static inline void i40iw_get_cqp_reg_info(struct i40iw_sc_cqp *cqp,
- u32 *val,
- u32 *tail,
- u32 *error)
-{
- if (cqp->dev->is_pf) {
- *val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPTAIL);
- *tail = RS_32(*val, I40E_PFPE_CQPTAIL_WQTAIL);
- *error = RS_32(*val, I40E_PFPE_CQPTAIL_CQP_OP_ERR);
- } else {
- *val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPTAIL1);
- *tail = RS_32(*val, I40E_VFPE_CQPTAIL_WQTAIL);
- *error = RS_32(*val, I40E_VFPE_CQPTAIL_CQP_OP_ERR);
- }
-}
-
-/**
- * i40iw_cqp_poll_registers - poll cqp registers
- * @cqp: struct for cqp hw
- * @tail:wqtail register value
- * @count: how many times to try for completion
- */
-static enum i40iw_status_code i40iw_cqp_poll_registers(
- struct i40iw_sc_cqp *cqp,
- u32 tail,
- u32 count)
-{
- u32 i = 0;
- u32 newtail, error, val;
-
- while (i < count) {
- i++;
- i40iw_get_cqp_reg_info(cqp, &val, &newtail, &error);
- if (error) {
- error = (cqp->dev->is_pf) ?
- i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES) :
- i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
- return I40IW_ERR_CQP_COMPL_ERROR;
- }
- if (newtail != tail) {
- /* SUCCESS */
- I40IW_RING_MOVE_TAIL(cqp->sq_ring);
- cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
- return 0;
- }
- udelay(I40IW_SLEEP_COUNT);
- }
- return I40IW_ERR_TIMEOUT;
-}
-
-/**
- * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer
- * @buf: ptr to fpm commit buffer
- * @info: ptr to i40iw_hmc_obj_info struct
- * @sd: number of SDs for HMC objects
- *
- * parses fpm commit info and copy base value
- * of hmc objects in hmc_info
- */
-static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf(
- u64 *buf,
- struct i40iw_hmc_obj_info *info,
- u32 *sd)
-{
- u64 temp;
- u64 size;
- u64 base = 0;
- u32 i, j;
- u32 k = 0;
-
- /* copy base values in obj_info */
- for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE; i++, j += 8) {
- if ((i == I40IW_HMC_IW_SRQ) ||
- (i == I40IW_HMC_IW_FSIMC) ||
- (i == I40IW_HMC_IW_FSIAV)) {
- info[i].base = 0;
- info[i].cnt = 0;
- continue;
- }
- get_64bit_val(buf, j, &temp);
- info[i].base = RS_64_1(temp, 32) * 512;
- if (info[i].base > base) {
- base = info[i].base;
- k = i;
- }
- if (i == I40IW_HMC_IW_APBVT_ENTRY) {
- info[i].cnt = 1;
- continue;
- }
- if (i == I40IW_HMC_IW_QP)
- info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
- else if (i == I40IW_HMC_IW_CQ)
- info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
- else
- info[i].cnt = (u32)(temp);
- }
- size = info[k].cnt * info[k].size + info[k].base;
- if (size & 0x1FFFFF)
- *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */
- else
- *sd = (u32)(size >> 21);
-
- return 0;
-}
-
-/**
- * i40iw_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
- * @buf: ptr to fpm query buffer
- * @buf_idx: index into buf
- * @info: ptr to i40iw_hmc_obj_info struct
- * @rsrc_idx: resource index into info
- *
- * Decode a 64 bit value from fpm query buffer into max count and size
- */
-static u64 i40iw_sc_decode_fpm_query(u64 *buf,
- u32 buf_idx,
- struct i40iw_hmc_obj_info *obj_info,
- u32 rsrc_idx)
-{
- u64 temp;
- u32 size;
-
- get_64bit_val(buf, buf_idx, &temp);
- obj_info[rsrc_idx].max_cnt = (u32)temp;
- size = (u32)RS_64_1(temp, 32);
- obj_info[rsrc_idx].size = LS_64_1(1, size);
-
- return temp;
-}
-
-/**
- * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer
- * @buf: ptr to fpm query buffer
- * @info: ptr to i40iw_hmc_obj_info struct
- * @hmc_fpm_misc: ptr to fpm data
- *
- * parses fpm query buffer and copy max_cnt and
- * size value of hmc objects in hmc_info
- */
-static enum i40iw_status_code i40iw_sc_parse_fpm_query_buf(
- u64 *buf,
- struct i40iw_hmc_info *hmc_info,
- struct i40iw_hmc_fpm_misc *hmc_fpm_misc)
-{
- struct i40iw_hmc_obj_info *obj_info;
- u64 temp;
- u32 size;
- u16 max_pe_sds;
-
- obj_info = hmc_info->hmc_obj;
-
- get_64bit_val(buf, 0, &temp);
- hmc_info->first_sd_index = (u16)RS_64(temp, I40IW_QUERY_FPM_FIRST_PE_SD_INDEX);
- max_pe_sds = (u16)RS_64(temp, I40IW_QUERY_FPM_MAX_PE_SDS);
-
- /* Reduce SD count for VFs by 1 to account for PBLE backing page rounding */
- if (hmc_info->hmc_fn_id >= I40IW_FIRST_VF_FPM_ID)
- max_pe_sds--;
- hmc_fpm_misc->max_sds = max_pe_sds;
- hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
-
- get_64bit_val(buf, 8, &temp);
- obj_info[I40IW_HMC_IW_QP].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
- size = (u32)RS_64_1(temp, 32);
- obj_info[I40IW_HMC_IW_QP].size = LS_64_1(1, size);
-
- get_64bit_val(buf, 16, &temp);
- obj_info[I40IW_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
- size = (u32)RS_64_1(temp, 32);
- obj_info[I40IW_HMC_IW_CQ].size = LS_64_1(1, size);
-
- i40iw_sc_decode_fpm_query(buf, 32, obj_info, I40IW_HMC_IW_HTE);
- i40iw_sc_decode_fpm_query(buf, 40, obj_info, I40IW_HMC_IW_ARP);
-
- obj_info[I40IW_HMC_IW_APBVT_ENTRY].size = 8192;
- obj_info[I40IW_HMC_IW_APBVT_ENTRY].max_cnt = 1;
-
- i40iw_sc_decode_fpm_query(buf, 48, obj_info, I40IW_HMC_IW_MR);
- i40iw_sc_decode_fpm_query(buf, 56, obj_info, I40IW_HMC_IW_XF);
-
- get_64bit_val(buf, 64, &temp);
- obj_info[I40IW_HMC_IW_XFFL].max_cnt = (u32)temp;
- obj_info[I40IW_HMC_IW_XFFL].size = 4;
- hmc_fpm_misc->xf_block_size = RS_64(temp, I40IW_QUERY_FPM_XFBLOCKSIZE);
- if (!hmc_fpm_misc->xf_block_size)
- return I40IW_ERR_INVALID_SIZE;
-
- i40iw_sc_decode_fpm_query(buf, 72, obj_info, I40IW_HMC_IW_Q1);
-
- get_64bit_val(buf, 80, &temp);
- obj_info[I40IW_HMC_IW_Q1FL].max_cnt = (u32)temp;
- obj_info[I40IW_HMC_IW_Q1FL].size = 4;
- hmc_fpm_misc->q1_block_size = RS_64(temp, I40IW_QUERY_FPM_Q1BLOCKSIZE);
- if (!hmc_fpm_misc->q1_block_size)
- return I40IW_ERR_INVALID_SIZE;
-
- i40iw_sc_decode_fpm_query(buf, 88, obj_info, I40IW_HMC_IW_TIMER);
-
- get_64bit_val(buf, 112, &temp);
- obj_info[I40IW_HMC_IW_PBLE].max_cnt = (u32)temp;
- obj_info[I40IW_HMC_IW_PBLE].size = 8;
-
- get_64bit_val(buf, 120, &temp);
- hmc_fpm_misc->max_ceqs = (u8)RS_64(temp, I40IW_QUERY_FPM_MAX_CEQS);
- hmc_fpm_misc->ht_multiplier = RS_64(temp, I40IW_QUERY_FPM_HTMULTIPLIER);
- hmc_fpm_misc->timer_bucket = RS_64(temp, I40IW_QUERY_FPM_TIMERBUCKET);
-
- return 0;
-}
-
-/**
- * i40iw_fill_qos_list - Change all unknown qs handles to available ones
- * @qs_list: list of qs_handles to be fixed with valid qs_handles
- */
-static void i40iw_fill_qos_list(u16 *qs_list)
-{
- u16 qshandle = qs_list[0];
- int i;
-
- for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
- if (qs_list[i] == QS_HANDLE_UNKNOWN)
- qs_list[i] = qshandle;
- else
- qshandle = qs_list[i];
- }
-}
-
-/**
- * i40iw_qp_from_entry - Given entry, get to the qp structure
- * @entry: Points to list of qp structure
- */
-static struct i40iw_sc_qp *i40iw_qp_from_entry(struct list_head *entry)
-{
- if (!entry)
- return NULL;
-
- return (struct i40iw_sc_qp *)((char *)entry - offsetof(struct i40iw_sc_qp, list));
-}
-
-/**
- * i40iw_get_qp - get the next qp from the list given current qp
- * @head: Listhead of qp's
- * @qp: current qp
- */
-static struct i40iw_sc_qp *i40iw_get_qp(struct list_head *head, struct i40iw_sc_qp *qp)
-{
- struct list_head *entry = NULL;
- struct list_head *lastentry;
-
- if (list_empty(head))
- return NULL;
-
- if (!qp) {
- entry = head->next;
- } else {
- lastentry = &qp->list;
- entry = (lastentry != head) ? lastentry->next : NULL;
- }
-
- return i40iw_qp_from_entry(entry);
-}
-
-/**
- * i40iw_change_l2params - given the new l2 parameters, change all qp
- * @vsi: pointer to the vsi structure
- * @l2params: New paramaters from l2
- */
-void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params)
-{
- struct i40iw_sc_dev *dev = vsi->dev;
- struct i40iw_sc_qp *qp = NULL;
- bool qs_handle_change = false;
- unsigned long flags;
- u16 qs_handle;
- int i;
-
- if (vsi->mtu != l2params->mtu) {
- vsi->mtu = l2params->mtu;
- i40iw_reinitialize_ieq(dev);
- }
-
- i40iw_fill_qos_list(l2params->qs_handle_list);
- for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
- qs_handle = l2params->qs_handle_list[i];
- if (vsi->qos[i].qs_handle != qs_handle)
- qs_handle_change = true;
- spin_lock_irqsave(&vsi->qos[i].lock, flags);
- qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
- while (qp) {
- if (qs_handle_change) {
- qp->qs_handle = qs_handle;
- /* issue cqp suspend command */
- i40iw_qp_suspend_resume(dev, qp, true);
- }
- qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
- }
- spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
- vsi->qos[i].qs_handle = qs_handle;
- }
-}
-
-/**
- * i40iw_qp_rem_qos - remove qp from qos lists during destroy qp
- * @qp: qp to be removed from qos
- */
-void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp)
-{
- struct i40iw_sc_vsi *vsi = qp->vsi;
- unsigned long flags;
-
- if (!qp->on_qoslist)
- return;
- spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
- list_del(&qp->list);
- spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
-}
-
-/**
- * i40iw_qp_add_qos - called during setctx fot qp to be added to qos
- * @qp: qp to be added to qos
- */
-void i40iw_qp_add_qos(struct i40iw_sc_qp *qp)
-{
- struct i40iw_sc_vsi *vsi = qp->vsi;
- unsigned long flags;
-
- if (qp->on_qoslist)
- return;
- spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
- qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
- list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
- qp->on_qoslist = true;
- spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
-}
-
-/**
- * i40iw_sc_pd_init - initialize sc pd struct
- * @dev: sc device struct
- * @pd: sc pd ptr
- * @pd_id: pd_id for allocated pd
- * @abi_ver: ABI version from user context, -1 if not valid
- */
-static void i40iw_sc_pd_init(struct i40iw_sc_dev *dev,
- struct i40iw_sc_pd *pd,
- u16 pd_id,
- int abi_ver)
-{
- pd->size = sizeof(*pd);
- pd->pd_id = pd_id;
- pd->abi_ver = abi_ver;
- pd->dev = dev;
-}
-
-/**
- * i40iw_get_encoded_wqe_size - given wq size, returns hardware encoded size
- * @wqsize: size of the wq (sq, rq, srq) to encoded_size
- * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's
- */
-u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq)
-{
- u8 encoded_size = 0;
-
- /* cqp sq's hw coded value starts from 1 for size of 4
- * while it starts from 0 for qp' wq's.
- */
- if (cqpsq)
- encoded_size = 1;
- wqsize >>= 2;
- while (wqsize >>= 1)
- encoded_size++;
- return encoded_size;
-}
-
-/**
- * i40iw_sc_cqp_init - Initialize buffers for a control Queue Pair
- * @cqp: IWARP control queue pair pointer
- * @info: IWARP control queue pair init info pointer
- *
- * Initializes the object and context buffers for a control Queue Pair.
- */
-static enum i40iw_status_code i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp,
- struct i40iw_cqp_init_info *info)
-{
- u8 hw_sq_size;
-
- if ((info->sq_size > I40IW_CQP_SW_SQSIZE_2048) ||
- (info->sq_size < I40IW_CQP_SW_SQSIZE_4) ||
- ((info->sq_size & (info->sq_size - 1))))
- return I40IW_ERR_INVALID_SIZE;
-
- hw_sq_size = i40iw_get_encoded_wqe_size(info->sq_size, true);
- cqp->size = sizeof(*cqp);
- cqp->sq_size = info->sq_size;
- cqp->hw_sq_size = hw_sq_size;
- cqp->sq_base = info->sq;
- cqp->host_ctx = info->host_ctx;
- cqp->sq_pa = info->sq_pa;
- cqp->host_ctx_pa = info->host_ctx_pa;
- cqp->dev = info->dev;
- cqp->struct_ver = info->struct_ver;
- cqp->scratch_array = info->scratch_array;
- cqp->polarity = 0;
- cqp->en_datacenter_tcp = info->en_datacenter_tcp;
- cqp->enabled_vf_count = info->enabled_vf_count;
- cqp->hmc_profile = info->hmc_profile;
- info->dev->cqp = cqp;
-
- I40IW_RING_INIT(cqp->sq_ring, cqp->sq_size);
- cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] = 0;
- cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS] = 0;
- INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head); /* for the cqp commands backlog. */
-
- i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPTAIL, 0);
- i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, 0);
-
- i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
- "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04X]\n",
- __func__, cqp->sq_size, cqp->hw_sq_size,
- cqp->sq_base, cqp->sq_pa, cqp, cqp->polarity);
- return 0;
-}
-
-/**
- * i40iw_sc_cqp_create - create cqp during bringup
- * @cqp: struct for cqp hw
- * @maj_err: If error, major err number
- * @min_err: If error, minor err number
- */
-static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,
- u16 *maj_err,
- u16 *min_err)
-{
- u64 temp;
- u32 cnt = 0, p1, p2, val = 0, err_code;
- enum i40iw_status_code ret_code;
-
- *maj_err = 0;
- *min_err = 0;
-
- ret_code = i40iw_allocate_dma_mem(cqp->dev->hw,
- &cqp->sdbuf,
- I40IW_UPDATE_SD_BUF_SIZE * cqp->sq_size,
- I40IW_SD_BUF_ALIGNMENT);
-
- if (ret_code)
- goto exit;
-
- temp = LS_64(cqp->hw_sq_size, I40IW_CQPHC_SQSIZE) |
- LS_64(cqp->struct_ver, I40IW_CQPHC_SVER);
-
- set_64bit_val(cqp->host_ctx, 0, temp);
- set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
- temp = LS_64(cqp->enabled_vf_count, I40IW_CQPHC_ENABLED_VFS) |
- LS_64(cqp->hmc_profile, I40IW_CQPHC_HMC_PROFILE);
- set_64bit_val(cqp->host_ctx, 16, temp);
- set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);
- set_64bit_val(cqp->host_ctx, 32, 0);
- set_64bit_val(cqp->host_ctx, 40, 0);
- set_64bit_val(cqp->host_ctx, 48, 0);
- set_64bit_val(cqp->host_ctx, 56, 0);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQP_HOST_CTX",
- cqp->host_ctx, I40IW_CQP_CTX_SIZE * 8);
-
- p1 = RS_32_1(cqp->host_ctx_pa, 32);
- p2 = (u32)cqp->host_ctx_pa;
-
- if (cqp->dev->is_pf) {
- i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, p1);
- i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, p2);
- } else {
- i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, p1);
- i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, p2);
- }
- do {
- if (cnt++ > I40IW_DONE_COUNT) {
- i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
- ret_code = I40IW_ERR_TIMEOUT;
- /*
- * read PFPE_CQPERRORCODES register to get the minor
- * and major error code
- */
- if (cqp->dev->is_pf)
- err_code = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES);
- else
- err_code = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
- *min_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE);
- *maj_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE);
- goto exit;
- }
- udelay(I40IW_SLEEP_COUNT);
- if (cqp->dev->is_pf)
- val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CCQPSTATUS);
- else
- val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CCQPSTATUS1);
- } while (!val);
-
-exit:
- if (!ret_code)
- cqp->process_cqp_sds = i40iw_update_sds_noccq;
- return ret_code;
-}
-
-/**
- * i40iw_sc_cqp_post_sq - post of cqp's sq
- * @cqp: struct for cqp hw
- */
-void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp)
-{
- if (cqp->dev->is_pf)
- i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
- else
- i40iw_wr32(cqp->dev->hw, I40E_VFPE_CQPDB1, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
-
- i40iw_debug(cqp->dev,
- I40IW_DEBUG_WQE,
- "%s: HEAD_TAIL[%04d,%04d,%04d]\n",
- __func__,
- cqp->sq_ring.head,
- cqp->sq_ring.tail,
- cqp->sq_ring.size);
-}
-
-/**
- * i40iw_sc_cqp_get_next_send_wqe_idx - get next WQE on CQP SQ and pass back the index
- * @cqp: pointer to CQP structure
- * @scratch: private data for CQP WQE
- * @wqe_idx: WQE index for next WQE on CQP SQ
- */
-static u64 *i40iw_sc_cqp_get_next_send_wqe_idx(struct i40iw_sc_cqp *cqp,
- u64 scratch, u32 *wqe_idx)
-{
- u64 *wqe = NULL;
- enum i40iw_status_code ret_code;
-
- if (I40IW_RING_FULL_ERR(cqp->sq_ring)) {
- i40iw_debug(cqp->dev,
- I40IW_DEBUG_WQE,
- "%s: ring is full head %x tail %x size %x\n",
- __func__,
- cqp->sq_ring.head,
- cqp->sq_ring.tail,
- cqp->sq_ring.size);
- return NULL;
- }
- I40IW_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);
- cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS]++;
- if (ret_code)
- return NULL;
- if (!*wqe_idx)
- cqp->polarity = !cqp->polarity;
-
- wqe = cqp->sq_base[*wqe_idx].elem;
- cqp->scratch_array[*wqe_idx] = scratch;
- I40IW_CQP_INIT_WQE(wqe);
-
- return wqe;
-}
-
-/**
- * i40iw_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
- * @cqp: struct for cqp hw
- * @scratch: private data for CQP WQE
- */
-u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch)
-{
- u32 wqe_idx;
-
- return i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
-}
-
-/**
- * i40iw_sc_cqp_destroy - destroy cqp during close
- * @cqp: struct for cqp hw
- */
-static enum i40iw_status_code i40iw_sc_cqp_destroy(struct i40iw_sc_cqp *cqp)
-{
- u32 cnt = 0, val = 1;
- enum i40iw_status_code ret_code = 0;
- u32 cqpstat_addr;
-
- if (cqp->dev->is_pf) {
- i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, 0);
- i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, 0);
- cqpstat_addr = I40E_PFPE_CCQPSTATUS;
- } else {
- i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, 0);
- i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, 0);
- cqpstat_addr = I40E_VFPE_CCQPSTATUS1;
- }
- do {
- if (cnt++ > I40IW_DONE_COUNT) {
- ret_code = I40IW_ERR_TIMEOUT;
- break;
- }
- udelay(I40IW_SLEEP_COUNT);
- val = i40iw_rd32(cqp->dev->hw, cqpstat_addr);
- } while (val);
-
- i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
- return ret_code;
-}
-
-/**
- * i40iw_sc_ccq_arm - enable intr for control cq
- * @ccq: ccq sc struct
- */
-static void i40iw_sc_ccq_arm(struct i40iw_sc_cq *ccq)
-{
- u64 temp_val;
- u16 sw_cq_sel;
- u8 arm_next_se;
- u8 arm_seq_num;
-
- /* write to cq doorbell shadow area */
- /* arm next se should always be zero */
- get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);
-
- sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);
- arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);
-
- arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);
- arm_seq_num++;
-
- temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) |
- LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) |
- LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) |
- LS_64(1, I40IW_CQ_DBSA_ARM_NEXT);
-
- set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);
-
- wmb(); /* make sure shadow area is updated before arming */
-
- if (ccq->dev->is_pf)
- i40iw_wr32(ccq->dev->hw, I40E_PFPE_CQARM, ccq->cq_uk.cq_id);
- else
- i40iw_wr32(ccq->dev->hw, I40E_VFPE_CQARM1, ccq->cq_uk.cq_id);
-}
-
-/**
- * i40iw_sc_ccq_get_cqe_info - get ccq's cq entry
- * @ccq: ccq sc struct
- * @info: completion q entry to return
- */
-static enum i40iw_status_code i40iw_sc_ccq_get_cqe_info(
- struct i40iw_sc_cq *ccq,
- struct i40iw_ccq_cqe_info *info)
-{
- u64 qp_ctx, temp, temp1;
- u64 *cqe;
- struct i40iw_sc_cqp *cqp;
- u32 wqe_idx;
- u8 polarity;
- enum i40iw_status_code ret_code = 0;
-
- if (ccq->cq_uk.avoid_mem_cflct)
- cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(&ccq->cq_uk);
- else
- cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&ccq->cq_uk);
-
- get_64bit_val(cqe, 24, &temp);
- polarity = (u8)RS_64(temp, I40IW_CQ_VALID);
- if (polarity != ccq->cq_uk.polarity)
- return I40IW_ERR_QUEUE_EMPTY;
-
- get_64bit_val(cqe, 8, &qp_ctx);
- cqp = (struct i40iw_sc_cqp *)(unsigned long)qp_ctx;
- info->error = (bool)RS_64(temp, I40IW_CQ_ERROR);
- info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
- if (info->error) {
- info->maj_err_code = (u16)RS_64(temp, I40IW_CQ_MAJERR);
- info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
- }
- wqe_idx = (u32)RS_64(temp, I40IW_CQ_WQEIDX);
- info->scratch = cqp->scratch_array[wqe_idx];
-
- get_64bit_val(cqe, 16, &temp1);
- info->op_ret_val = (u32)RS_64(temp1, I40IW_CCQ_OPRETVAL);
- get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);
- info->op_code = (u8)RS_64(temp1, I40IW_CQPSQ_OPCODE);
- info->cqp = cqp;
-
- /* move the head for cq */
- I40IW_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);
- if (I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring) == 0)
- ccq->cq_uk.polarity ^= 1;
-
- /* update cq tail in cq shadow memory also */
- I40IW_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);
- set_64bit_val(ccq->cq_uk.shadow_area,
- 0,
- I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring));
- wmb(); /* write shadow area before tail */
- I40IW_RING_MOVE_TAIL(cqp->sq_ring);
- ccq->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
-
- return ret_code;
-}
-
-/**
- * i40iw_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
- * @cqp: struct for cqp hw
- * @op_code: cqp opcode for completion
- * @info: completion q entry to return
- */
-static enum i40iw_status_code i40iw_sc_poll_for_cqp_op_done(
- struct i40iw_sc_cqp *cqp,
- u8 op_code,
- struct i40iw_ccq_cqe_info *compl_info)
-{
- struct i40iw_ccq_cqe_info info;
- struct i40iw_sc_cq *ccq;
- enum i40iw_status_code ret_code = 0;
- u32 cnt = 0;
-
- memset(&info, 0, sizeof(info));
- ccq = cqp->dev->ccq;
- while (1) {
- if (cnt++ > I40IW_DONE_COUNT)
- return I40IW_ERR_TIMEOUT;
-
- if (i40iw_sc_ccq_get_cqe_info(ccq, &info)) {
- udelay(I40IW_SLEEP_COUNT);
- continue;
- }
-
- if (info.error) {
- ret_code = I40IW_ERR_CQP_COMPL_ERROR;
- break;
- }
- /* check if opcode is cq create */
- if (op_code != info.op_code) {
- i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
- "%s: opcode mismatch for my op code 0x%x, returned opcode %x\n",
- __func__, op_code, info.op_code);
- }
- /* success, exit out of the loop */
- if (op_code == info.op_code)
- break;
- }
-
- if (compl_info)
- memcpy(compl_info, &info, sizeof(*compl_info));
-
- return ret_code;
-}
-
-/**
- * i40iw_sc_manage_push_page - Handle push page
- * @cqp: struct for cqp hw
- * @info: push page info
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_manage_push_page(
- struct i40iw_sc_cqp *cqp,
- struct i40iw_cqp_manage_push_page_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- u64 header;
-
- if (info->push_idx >= I40IW_MAX_PUSH_PAGE_COUNT)
- return I40IW_ERR_INVALID_PUSH_PAGE_INDEX;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- set_64bit_val(wqe, 16, info->qs_handle);
-
- header = LS_64(info->push_idx, I40IW_CQPSQ_MPP_PPIDX) |
- LS_64(I40IW_CQP_OP_MANAGE_PUSH_PAGES, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
- LS_64(info->free_page, I40IW_CQPSQ_MPP_FREE_PAGE);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_manage_hmc_pm_func_table - manage of function table
- * @cqp: struct for cqp hw
- * @scratch: u64 saved to be used during cqp completion
- * @vf_index: vf index for cqp
- * @free_pm_fcn: function number
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table(
- struct i40iw_sc_cqp *cqp,
- u64 scratch,
- u8 vf_index,
- bool free_pm_fcn,
- bool post_sq)
-{
- u64 *wqe;
- u64 header;
-
- if (vf_index >= I40IW_MAX_VF_PER_PF)
- return I40IW_ERR_INVALID_VF_ID;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- header = LS_64(vf_index, I40IW_CQPSQ_MHMC_VFIDX) |
- LS_64(I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, I40IW_CQPSQ_OPCODE) |
- LS_64(free_pm_fcn, I40IW_CQPSQ_MHMC_FREEPMFN) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_set_hmc_resource_profile - cqp wqe for hmc profile
- * @cqp: struct for cqp hw
- * @scratch: u64 saved to be used during cqp completion
- * @hmc_profile_type: type of profile to set
- * @vf_num: vf number for profile
- * @post_sq: flag for cqp db to ring
- * @poll_registers: flag to poll register for cqp completion
- */
-static enum i40iw_status_code i40iw_sc_set_hmc_resource_profile(
- struct i40iw_sc_cqp *cqp,
- u64 scratch,
- u8 hmc_profile_type,
- u8 vf_num, bool post_sq,
- bool poll_registers)
-{
- u64 *wqe;
- u64 header;
- u32 val, tail, error;
- enum i40iw_status_code ret_code = 0;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- set_64bit_val(wqe, 16,
- (LS_64(hmc_profile_type, I40IW_CQPSQ_SHMCRP_HMC_PROFILE) |
- LS_64(vf_num, I40IW_CQPSQ_SHMCRP_VFNUM)));
-
- header = LS_64(I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
- if (error)
- return I40IW_ERR_CQP_COMPL_ERROR;
-
- if (post_sq) {
- i40iw_sc_cqp_post_sq(cqp);
- if (poll_registers)
- ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000000);
- else
- ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
- I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
- NULL);
- }
-
- return ret_code;
-}
-
-/**
- * i40iw_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table
- * @cqp: struct for cqp hw
- */
-static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table_done(struct i40iw_sc_cqp *cqp)
-{
- return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, NULL);
-}
-
-/**
- * i40iw_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit
- * @cqp: struct for cqp hw
- */
-static enum i40iw_status_code i40iw_sc_commit_fpm_values_done(struct i40iw_sc_cqp *cqp)
-{
- return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_COMMIT_FPM_VALUES, NULL);
-}
-
-/**
- * i40iw_sc_commit_fpm_values - cqp wqe for commit fpm values
- * @cqp: struct for cqp hw
- * @scratch: u64 saved to be used during cqp completion
- * @hmc_fn_id: hmc function id
- * @commit_fpm_mem; Memory for fpm values
- * @post_sq: flag for cqp db to ring
- * @wait_type: poll ccq or cqp registers for cqp completion
- */
-static enum i40iw_status_code i40iw_sc_commit_fpm_values(
- struct i40iw_sc_cqp *cqp,
- u64 scratch,
- u8 hmc_fn_id,
- struct i40iw_dma_mem *commit_fpm_mem,
- bool post_sq,
- u8 wait_type)
-{
- u64 *wqe;
- u64 header;
- u32 tail, val, error;
- enum i40iw_status_code ret_code = 0;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- set_64bit_val(wqe, 16, hmc_fn_id);
- set_64bit_val(wqe, 32, commit_fpm_mem->pa);
-
- header = LS_64(I40IW_CQP_OP_COMMIT_FPM_VALUES, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "COMMIT_FPM_VALUES WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
- if (error)
- return I40IW_ERR_CQP_COMPL_ERROR;
-
- if (post_sq) {
- i40iw_sc_cqp_post_sq(cqp);
-
- if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
- ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
- else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
- ret_code = i40iw_sc_commit_fpm_values_done(cqp);
- }
-
- return ret_code;
-}
-
-/**
- * i40iw_sc_query_rdma_features_done - poll cqp for query features done
- * @cqp: struct for cqp hw
- */
-static enum i40iw_status_code
-i40iw_sc_query_rdma_features_done(struct i40iw_sc_cqp *cqp)
-{
- return i40iw_sc_poll_for_cqp_op_done(
- cqp, I40IW_CQP_OP_QUERY_RDMA_FEATURES, NULL);
-}
-
-/**
- * i40iw_sc_query_rdma_features - query rdma features
- * @cqp: struct for cqp hw
- * @feat_mem: holds PA for HW to use
- * @scratch: u64 saved to be used during cqp completion
- */
-static enum i40iw_status_code
-i40iw_sc_query_rdma_features(struct i40iw_sc_cqp *cqp,
- struct i40iw_dma_mem *feat_mem, u64 scratch)
-{
- u64 *wqe;
- u64 header;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- set_64bit_val(wqe, 32, feat_mem->pa);
-
- header = LS_64(I40IW_CQP_OP_QUERY_RDMA_FEATURES, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) | feat_mem->size;
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY RDMA FEATURES WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- i40iw_sc_cqp_post_sq(cqp);
-
- return 0;
-}
-
-/**
- * i40iw_get_rdma_features - get RDMA features
- * @dev - sc device struct
- */
-enum i40iw_status_code i40iw_get_rdma_features(struct i40iw_sc_dev *dev)
-{
- enum i40iw_status_code ret_code;
- struct i40iw_dma_mem feat_buf;
- u64 temp;
- u16 byte_idx, feat_type, feat_cnt;
-
- ret_code = i40iw_allocate_dma_mem(dev->hw, &feat_buf,
- I40IW_FEATURE_BUF_SIZE,
- I40IW_FEATURE_BUF_ALIGNMENT);
-
- if (ret_code)
- return I40IW_ERR_NO_MEMORY;
-
- ret_code = i40iw_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
- if (!ret_code)
- ret_code = i40iw_sc_query_rdma_features_done(dev->cqp);
-
- if (ret_code)
- goto exit;
-
- get_64bit_val(feat_buf.va, 0, &temp);
- feat_cnt = RS_64(temp, I40IW_FEATURE_CNT);
- if (feat_cnt < I40IW_MAX_FEATURES) {
- ret_code = I40IW_ERR_INVALID_FEAT_CNT;
- goto exit;
- } else if (feat_cnt > I40IW_MAX_FEATURES) {
- i40iw_debug(dev, I40IW_DEBUG_CQP,
- "features buf size insufficient\n");
- }
-
- for (byte_idx = 0, feat_type = 0; feat_type < I40IW_MAX_FEATURES;
- feat_type++, byte_idx += 8) {
- get_64bit_val((u64 *)feat_buf.va, byte_idx, &temp);
- dev->feature_info[feat_type] = RS_64(temp, I40IW_FEATURE_INFO);
- }
-exit:
- i40iw_free_dma_mem(dev->hw, &feat_buf);
-
- return ret_code;
-}
-
-/**
- * i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm
- * @cqp: struct for cqp hw
- */
-static enum i40iw_status_code i40iw_sc_query_fpm_values_done(struct i40iw_sc_cqp *cqp)
-{
- return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_QUERY_FPM_VALUES, NULL);
-}
-
-/**
- * i40iw_sc_query_fpm_values - cqp wqe query fpm values
- * @cqp: struct for cqp hw
- * @scratch: u64 saved to be used during cqp completion
- * @hmc_fn_id: hmc function id
- * @query_fpm_mem: memory for return fpm values
- * @post_sq: flag for cqp db to ring
- * @wait_type: poll ccq or cqp registers for cqp completion
- */
-static enum i40iw_status_code i40iw_sc_query_fpm_values(
- struct i40iw_sc_cqp *cqp,
- u64 scratch,
- u8 hmc_fn_id,
- struct i40iw_dma_mem *query_fpm_mem,
- bool post_sq,
- u8 wait_type)
-{
- u64 *wqe;
- u64 header;
- u32 tail, val, error;
- enum i40iw_status_code ret_code = 0;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- set_64bit_val(wqe, 16, hmc_fn_id);
- set_64bit_val(wqe, 32, query_fpm_mem->pa);
-
- header = LS_64(I40IW_CQP_OP_QUERY_FPM_VALUES, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_FPM WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- /* read the tail from CQP_TAIL register */
- i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
-
- if (error)
- return I40IW_ERR_CQP_COMPL_ERROR;
-
- if (post_sq) {
- i40iw_sc_cqp_post_sq(cqp);
- if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
- ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
- else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
- ret_code = i40iw_sc_query_fpm_values_done(cqp);
- }
-
- return ret_code;
-}
-
-/**
- * i40iw_sc_add_arp_cache_entry - cqp wqe add arp cache entry
- * @cqp: struct for cqp hw
- * @info: arp entry information
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_add_arp_cache_entry(
- struct i40iw_sc_cqp *cqp,
- struct i40iw_add_arp_cache_entry_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- u64 temp, header;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 8, info->reach_max);
-
- temp = info->mac_addr[5] |
- LS_64_1(info->mac_addr[4], 8) |
- LS_64_1(info->mac_addr[3], 16) |
- LS_64_1(info->mac_addr[2], 24) |
- LS_64_1(info->mac_addr[1], 32) |
- LS_64_1(info->mac_addr[0], 40);
-
- set_64bit_val(wqe, 16, temp);
-
- header = info->arp_index |
- LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
- LS_64((info->permanent ? 1 : 0), I40IW_CQPSQ_MAT_PERMANENT) |
- LS_64(1, I40IW_CQPSQ_MAT_ENTRYVALID) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_ENTRY WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_del_arp_cache_entry - dele arp cache entry
- * @cqp: struct for cqp hw
- * @scratch: u64 saved to be used during cqp completion
- * @arp_index: arp index to delete arp entry
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_del_arp_cache_entry(
- struct i40iw_sc_cqp *cqp,
- u64 scratch,
- u16 arp_index,
- bool post_sq)
-{
- u64 *wqe;
- u64 header;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- header = arp_index |
- LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_query_arp_cache_entry - cqp wqe to query arp and arp index
- * @cqp: struct for cqp hw
- * @scratch: u64 saved to be used during cqp completion
- * @arp_index: arp index to delete arp entry
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_query_arp_cache_entry(
- struct i40iw_sc_cqp *cqp,
- u64 scratch,
- u16 arp_index,
- bool post_sq)
-{
- u64 *wqe;
- u64 header;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- header = arp_index |
- LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
- LS_64(1, I40IW_CQPSQ_MAT_QUERY) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_ARP_CACHE_ENTRY WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_manage_apbvt_entry - for adding and deleting apbvt entries
- * @cqp: struct for cqp hw
- * @info: info for apbvt entry to add or delete
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_manage_apbvt_entry(
- struct i40iw_sc_cqp *cqp,
- struct i40iw_apbvt_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- u64 header;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- set_64bit_val(wqe, 16, info->port);
-
- header = LS_64(I40IW_CQP_OP_MANAGE_APBVT, I40IW_CQPSQ_OPCODE) |
- LS_64(info->add, I40IW_CQPSQ_MAPT_ADDPORT) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_APBVT WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_manage_qhash_table_entry - manage quad hash entries
- * @cqp: struct for cqp hw
- * @info: info for quad hash to manage
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- *
- * This is called before connection establishment is started. For passive connections, when
- * listener is created, it will call with entry type of I40IW_QHASH_TYPE_TCP_SYN with local
- * ip address and tcp port. When SYN is received (passive connections) or
- * sent (active connections), this routine is called with entry type of
- * I40IW_QHASH_TYPE_TCP_ESTABLISHED and quad is passed in info.
- *
- * When iwarp connection is done and its state moves to RTS, the quad hash entry in
- * the hardware will point to iwarp's qp number and requires no calls from the driver.
- */
-static enum i40iw_status_code i40iw_sc_manage_qhash_table_entry(
- struct i40iw_sc_cqp *cqp,
- struct i40iw_qhash_table_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- u64 qw1 = 0;
- u64 qw2 = 0;
- u64 temp;
- struct i40iw_sc_vsi *vsi = info->vsi;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- temp = info->mac_addr[5] |
- LS_64_1(info->mac_addr[4], 8) |
- LS_64_1(info->mac_addr[3], 16) |
- LS_64_1(info->mac_addr[2], 24) |
- LS_64_1(info->mac_addr[1], 32) |
- LS_64_1(info->mac_addr[0], 40);
-
- set_64bit_val(wqe, 0, temp);
-
- qw1 = LS_64(info->qp_num, I40IW_CQPSQ_QHASH_QPN) |
- LS_64(info->dest_port, I40IW_CQPSQ_QHASH_DEST_PORT);
- if (info->ipv4_valid) {
- set_64bit_val(wqe,
- 48,
- LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
- } else {
- set_64bit_val(wqe,
- 56,
- LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |
- LS_64(info->dest_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
-
- set_64bit_val(wqe,
- 48,
- LS_64(info->dest_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
- LS_64(info->dest_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
- }
- qw2 = LS_64(vsi->qos[info->user_pri].qs_handle, I40IW_CQPSQ_QHASH_QS_HANDLE);
- if (info->vlan_valid)
- qw2 |= LS_64(info->vlan_id, I40IW_CQPSQ_QHASH_VLANID);
- set_64bit_val(wqe, 16, qw2);
- if (info->entry_type == I40IW_QHASH_TYPE_TCP_ESTABLISHED) {
- qw1 |= LS_64(info->src_port, I40IW_CQPSQ_QHASH_SRC_PORT);
- if (!info->ipv4_valid) {
- set_64bit_val(wqe,
- 40,
- LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |
- LS_64(info->src_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
- set_64bit_val(wqe,
- 32,
- LS_64(info->src_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
- LS_64(info->src_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
- } else {
- set_64bit_val(wqe,
- 32,
- LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
- }
- }
-
- set_64bit_val(wqe, 8, qw1);
- temp = LS_64(cqp->polarity, I40IW_CQPSQ_QHASH_WQEVALID) |
- LS_64(I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY, I40IW_CQPSQ_QHASH_OPCODE) |
- LS_64(info->manage, I40IW_CQPSQ_QHASH_MANAGE) |
- LS_64(info->ipv4_valid, I40IW_CQPSQ_QHASH_IPV4VALID) |
- LS_64(info->vlan_valid, I40IW_CQPSQ_QHASH_VLANVALID) |
- LS_64(info->entry_type, I40IW_CQPSQ_QHASH_ENTRYTYPE);
-
- i40iw_insert_wqe_hdr(wqe, temp);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_QHASH WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_alloc_local_mac_ipaddr_entry - cqp wqe for loc mac entry
- * @cqp: struct for cqp hw
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_alloc_local_mac_ipaddr_entry(
- struct i40iw_sc_cqp *cqp,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- u64 header;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- header = LS_64(I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ALLOCATE_LOCAL_MAC_IPADDR WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_add_local_mac_ipaddr_entry - add mac enry
- * @cqp: struct for cqp hw
- * @info:mac addr info
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_add_local_mac_ipaddr_entry(
- struct i40iw_sc_cqp *cqp,
- struct i40iw_local_mac_ipaddr_entry_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- u64 temp, header;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- temp = info->mac_addr[5] |
- LS_64_1(info->mac_addr[4], 8) |
- LS_64_1(info->mac_addr[3], 16) |
- LS_64_1(info->mac_addr[2], 24) |
- LS_64_1(info->mac_addr[1], 32) |
- LS_64_1(info->mac_addr[0], 40);
-
- set_64bit_val(wqe, 32, temp);
-
- header = LS_64(info->entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |
- LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ADD_LOCAL_MAC_IPADDR WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_del_local_mac_ipaddr_entry - cqp wqe to dele local mac
- * @cqp: struct for cqp hw
- * @scratch: u64 saved to be used during cqp completion
- * @entry_idx: index of mac entry
- * @ ignore_ref_count: to force mac adde delete
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_del_local_mac_ipaddr_entry(
- struct i40iw_sc_cqp *cqp,
- u64 scratch,
- u8 entry_idx,
- u8 ignore_ref_count,
- bool post_sq)
-{
- u64 *wqe;
- u64 header;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- header = LS_64(entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |
- LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |
- LS_64(1, I40IW_CQPSQ_MLIPA_FREEENTRY) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
- LS_64(ignore_ref_count, I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_cqp_nop - send a nop wqe
- * @cqp: struct for cqp hw
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_cqp_nop(struct i40iw_sc_cqp *cqp,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- u64 header;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- header = LS_64(I40IW_CQP_OP_NOP, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
- i40iw_insert_wqe_hdr(wqe, header);
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "NOP WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_ceq_init - initialize ceq
- * @ceq: ceq sc structure
- * @info: ceq initialization info
- */
-static enum i40iw_status_code i40iw_sc_ceq_init(struct i40iw_sc_ceq *ceq,
- struct i40iw_ceq_init_info *info)
-{
- u32 pble_obj_cnt;
-
- if ((info->elem_cnt < I40IW_MIN_CEQ_ENTRIES) ||
- (info->elem_cnt > I40IW_MAX_CEQ_ENTRIES))
- return I40IW_ERR_INVALID_SIZE;
-
- if (info->ceq_id >= I40IW_MAX_CEQID)
- return I40IW_ERR_INVALID_CEQ_ID;
-
- pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
-
- if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
- return I40IW_ERR_INVALID_PBLE_INDEX;
-
- ceq->size = sizeof(*ceq);
- ceq->ceqe_base = (struct i40iw_ceqe *)info->ceqe_base;
- ceq->ceq_id = info->ceq_id;
- ceq->dev = info->dev;
- ceq->elem_cnt = info->elem_cnt;
- ceq->ceq_elem_pa = info->ceqe_pa;
- ceq->virtual_map = info->virtual_map;
-
- ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);
- ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);
- ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);
-
- ceq->tph_en = info->tph_en;
- ceq->tph_val = info->tph_val;
- ceq->polarity = 1;
- I40IW_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);
- ceq->dev->ceq[info->ceq_id] = ceq;
-
- return 0;
-}
-
-/**
- * i40iw_sc_ceq_create - create ceq wqe
- * @ceq: ceq sc structure
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_ceq_create(struct i40iw_sc_ceq *ceq,
- u64 scratch,
- bool post_sq)
-{
- struct i40iw_sc_cqp *cqp;
- u64 *wqe;
- u64 header;
-
- cqp = ceq->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 16, ceq->elem_cnt);
- set_64bit_val(wqe, 32, (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));
- set_64bit_val(wqe, 48, (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
- set_64bit_val(wqe, 56, LS_64(ceq->tph_val, I40IW_CQPSQ_TPHVAL));
-
- header = ceq->ceq_id |
- LS_64(I40IW_CQP_OP_CREATE_CEQ, I40IW_CQPSQ_OPCODE) |
- LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |
- LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) |
- LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_CREATE WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_cceq_create_done - poll for control ceq wqe to complete
- * @ceq: ceq sc structure
- */
-static enum i40iw_status_code i40iw_sc_cceq_create_done(struct i40iw_sc_ceq *ceq)
-{
- struct i40iw_sc_cqp *cqp;
-
- cqp = ceq->dev->cqp;
- return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CEQ, NULL);
-}
-
-/**
- * i40iw_sc_cceq_destroy_done - poll for destroy cceq to complete
- * @ceq: ceq sc structure
- */
-static enum i40iw_status_code i40iw_sc_cceq_destroy_done(struct i40iw_sc_ceq *ceq)
-{
- struct i40iw_sc_cqp *cqp;
-
- cqp = ceq->dev->cqp;
- cqp->process_cqp_sds = i40iw_update_sds_noccq;
- return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_CEQ, NULL);
-}
-
-/**
- * i40iw_sc_cceq_create - create cceq
- * @ceq: ceq sc structure
- * @scratch: u64 saved to be used during cqp completion
- */
-static enum i40iw_status_code i40iw_sc_cceq_create(struct i40iw_sc_ceq *ceq, u64 scratch)
-{
- enum i40iw_status_code ret_code;
-
- ret_code = i40iw_sc_ceq_create(ceq, scratch, true);
- if (!ret_code)
- ret_code = i40iw_sc_cceq_create_done(ceq);
- return ret_code;
-}
-
-/**
- * i40iw_sc_ceq_destroy - destroy ceq
- * @ceq: ceq sc structure
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_ceq_destroy(struct i40iw_sc_ceq *ceq,
- u64 scratch,
- bool post_sq)
-{
- struct i40iw_sc_cqp *cqp;
- u64 *wqe;
- u64 header;
-
- cqp = ceq->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 16, ceq->elem_cnt);
- set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);
- header = ceq->ceq_id |
- LS_64(I40IW_CQP_OP_DESTROY_CEQ, I40IW_CQPSQ_OPCODE) |
- LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |
- LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) |
- LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
- i40iw_insert_wqe_hdr(wqe, header);
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_DESTROY WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_process_ceq - process ceq
- * @dev: sc device struct
- * @ceq: ceq sc structure
- */
-static void *i40iw_sc_process_ceq(struct i40iw_sc_dev *dev, struct i40iw_sc_ceq *ceq)
-{
- u64 temp;
- u64 *ceqe;
- struct i40iw_sc_cq *cq = NULL;
- u8 polarity;
-
- ceqe = (u64 *)I40IW_GET_CURRENT_CEQ_ELEMENT(ceq);
- get_64bit_val(ceqe, 0, &temp);
- polarity = (u8)RS_64(temp, I40IW_CEQE_VALID);
- if (polarity != ceq->polarity)
- return cq;
-
- cq = (struct i40iw_sc_cq *)(unsigned long)LS_64_1(temp, 1);
-
- I40IW_RING_MOVE_TAIL(ceq->ceq_ring);
- if (I40IW_RING_GETCURRENT_TAIL(ceq->ceq_ring) == 0)
- ceq->polarity ^= 1;
-
- if (dev->is_pf)
- i40iw_wr32(dev->hw, I40E_PFPE_CQACK, cq->cq_uk.cq_id);
- else
- i40iw_wr32(dev->hw, I40E_VFPE_CQACK1, cq->cq_uk.cq_id);
-
- return cq;
-}
-
-/**
- * i40iw_sc_aeq_init - initialize aeq
- * @aeq: aeq structure ptr
- * @info: aeq initialization info
- */
-static enum i40iw_status_code i40iw_sc_aeq_init(struct i40iw_sc_aeq *aeq,
- struct i40iw_aeq_init_info *info)
-{
- u32 pble_obj_cnt;
-
- if ((info->elem_cnt < I40IW_MIN_AEQ_ENTRIES) ||
- (info->elem_cnt > I40IW_MAX_AEQ_ENTRIES))
- return I40IW_ERR_INVALID_SIZE;
- pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
-
- if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
- return I40IW_ERR_INVALID_PBLE_INDEX;
-
- aeq->size = sizeof(*aeq);
- aeq->polarity = 1;
- aeq->aeqe_base = (struct i40iw_sc_aeqe *)info->aeqe_base;
- aeq->dev = info->dev;
- aeq->elem_cnt = info->elem_cnt;
-
- aeq->aeq_elem_pa = info->aeq_elem_pa;
- I40IW_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);
- info->dev->aeq = aeq;
-
- aeq->virtual_map = info->virtual_map;
- aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);
- aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);
- aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);
- info->dev->aeq = aeq;
- return 0;
-}
-
-/**
- * i40iw_sc_aeq_create - create aeq
- * @aeq: aeq structure ptr
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_aeq_create(struct i40iw_sc_aeq *aeq,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
-
- cqp = aeq->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 16, aeq->elem_cnt);
- set_64bit_val(wqe, 32,
- (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));
- set_64bit_val(wqe, 48,
- (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
-
- header = LS_64(I40IW_CQP_OP_CREATE_AEQ, I40IW_CQPSQ_OPCODE) |
- LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |
- LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_CREATE WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_aeq_destroy - destroy aeq during close
- * @aeq: aeq structure ptr
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_aeq_destroy(struct i40iw_sc_aeq *aeq,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
-
- cqp = aeq->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 16, aeq->elem_cnt);
- set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);
- header = LS_64(I40IW_CQP_OP_DESTROY_AEQ, I40IW_CQPSQ_OPCODE) |
- LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |
- LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_DESTROY WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_get_next_aeqe - get next aeq entry
- * @aeq: aeq structure ptr
- * @info: aeqe info to be returned
- */
-static enum i40iw_status_code i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq *aeq,
- struct i40iw_aeqe_info *info)
-{
- u64 temp, compl_ctx;
- u64 *aeqe;
- u16 wqe_idx;
- u8 ae_src;
- u8 polarity;
-
- aeqe = (u64 *)I40IW_GET_CURRENT_AEQ_ELEMENT(aeq);
- get_64bit_val(aeqe, 0, &compl_ctx);
- get_64bit_val(aeqe, 8, &temp);
- polarity = (u8)RS_64(temp, I40IW_AEQE_VALID);
-
- if (aeq->polarity != polarity)
- return I40IW_ERR_QUEUE_EMPTY;
-
- i40iw_debug_buf(aeq->dev, I40IW_DEBUG_WQE, "AEQ_ENTRY", aeqe, 16);
-
- ae_src = (u8)RS_64(temp, I40IW_AEQE_AESRC);
- wqe_idx = (u16)RS_64(temp, I40IW_AEQE_WQDESCIDX);
- info->qp_cq_id = (u32)RS_64(temp, I40IW_AEQE_QPCQID);
- info->ae_id = (u16)RS_64(temp, I40IW_AEQE_AECODE);
- info->tcp_state = (u8)RS_64(temp, I40IW_AEQE_TCPSTATE);
- info->iwarp_state = (u8)RS_64(temp, I40IW_AEQE_IWSTATE);
- info->q2_data_written = (u8)RS_64(temp, I40IW_AEQE_Q2DATA);
- info->aeqe_overflow = (bool)RS_64(temp, I40IW_AEQE_OVERFLOW);
-
- switch (info->ae_id) {
- case I40IW_AE_PRIV_OPERATION_DENIED:
- case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG:
- case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT:
- case I40IW_AE_BAD_CLOSE:
- case I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE:
- case I40IW_AE_RDMA_READ_WHILE_ORD_ZERO:
- case I40IW_AE_STAG_ZERO_INVALID:
- case I40IW_AE_IB_RREQ_AND_Q1_FULL:
- case I40IW_AE_WQE_UNEXPECTED_OPCODE:
- case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
- case I40IW_AE_DDP_UBE_INVALID_MO:
- case I40IW_AE_DDP_UBE_INVALID_QN:
- case I40IW_AE_DDP_NO_L_BIT:
- case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
- case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
- case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST:
- case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
- case I40IW_AE_INVALID_ARP_ENTRY:
- case I40IW_AE_INVALID_TCP_OPTION_RCVD:
- case I40IW_AE_STALE_ARP_ENTRY:
- case I40IW_AE_LLP_CLOSE_COMPLETE:
- case I40IW_AE_LLP_CONNECTION_RESET:
- case I40IW_AE_LLP_FIN_RECEIVED:
- case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
- case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
- case I40IW_AE_LLP_SYN_RECEIVED:
- case I40IW_AE_LLP_TERMINATE_RECEIVED:
- case I40IW_AE_LLP_TOO_MANY_RETRIES:
- case I40IW_AE_LLP_DOUBT_REACHABILITY:
- case I40IW_AE_RESET_SENT:
- case I40IW_AE_TERMINATE_SENT:
- case I40IW_AE_RESET_NOT_SENT:
- case I40IW_AE_LCE_QP_CATASTROPHIC:
- case I40IW_AE_QP_SUSPEND_COMPLETE:
- info->qp = true;
- info->compl_ctx = compl_ctx;
- ae_src = I40IW_AE_SOURCE_RSVD;
- break;
- case I40IW_AE_LCE_CQ_CATASTROPHIC:
- info->cq = true;
- info->compl_ctx = LS_64_1(compl_ctx, 1);
- ae_src = I40IW_AE_SOURCE_RSVD;
- break;
- }
-
- switch (ae_src) {
- case I40IW_AE_SOURCE_RQ:
- case I40IW_AE_SOURCE_RQ_0011:
- info->qp = true;
- info->wqe_idx = wqe_idx;
- info->compl_ctx = compl_ctx;
- break;
- case I40IW_AE_SOURCE_CQ:
- case I40IW_AE_SOURCE_CQ_0110:
- case I40IW_AE_SOURCE_CQ_1010:
- case I40IW_AE_SOURCE_CQ_1110:
- info->cq = true;
- info->compl_ctx = LS_64_1(compl_ctx, 1);
- break;
- case I40IW_AE_SOURCE_SQ:
- case I40IW_AE_SOURCE_SQ_0111:
- info->qp = true;
- info->sq = true;
- info->wqe_idx = wqe_idx;
- info->compl_ctx = compl_ctx;
- break;
- case I40IW_AE_SOURCE_IN_RR_WR:
- case I40IW_AE_SOURCE_IN_RR_WR_1011:
- info->qp = true;
- info->compl_ctx = compl_ctx;
- info->in_rdrsp_wr = true;
- break;
- case I40IW_AE_SOURCE_OUT_RR:
- case I40IW_AE_SOURCE_OUT_RR_1111:
- info->qp = true;
- info->compl_ctx = compl_ctx;
- info->out_rdrsp = true;
- break;
- case I40IW_AE_SOURCE_RSVD:
- /* fallthrough */
- default:
- break;
- }
- I40IW_RING_MOVE_TAIL(aeq->aeq_ring);
- if (I40IW_RING_GETCURRENT_TAIL(aeq->aeq_ring) == 0)
- aeq->polarity ^= 1;
- return 0;
-}
-
-/**
- * i40iw_sc_repost_aeq_entries - repost completed aeq entries
- * @dev: sc device struct
- * @count: allocate count
- */
-static enum i40iw_status_code i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev *dev,
- u32 count)
-{
-
- if (dev->is_pf)
- i40iw_wr32(dev->hw, I40E_PFPE_AEQALLOC, count);
- else
- i40iw_wr32(dev->hw, I40E_VFPE_AEQALLOC1, count);
-
- return 0;
-}
-
-/**
- * i40iw_sc_aeq_create_done - create aeq
- * @aeq: aeq structure ptr
- */
-static enum i40iw_status_code i40iw_sc_aeq_create_done(struct i40iw_sc_aeq *aeq)
-{
- struct i40iw_sc_cqp *cqp;
-
- cqp = aeq->dev->cqp;
- return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_AEQ, NULL);
-}
-
-/**
- * i40iw_sc_aeq_destroy_done - destroy of aeq during close
- * @aeq: aeq structure ptr
- */
-static enum i40iw_status_code i40iw_sc_aeq_destroy_done(struct i40iw_sc_aeq *aeq)
-{
- struct i40iw_sc_cqp *cqp;
-
- cqp = aeq->dev->cqp;
- return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_AEQ, NULL);
-}
-
-/**
- * i40iw_sc_ccq_init - initialize control cq
- * @cq: sc's cq ctruct
- * @info: info for control cq initialization
- */
-static enum i40iw_status_code i40iw_sc_ccq_init(struct i40iw_sc_cq *cq,
- struct i40iw_ccq_init_info *info)
-{
- u32 pble_obj_cnt;
-
- if (info->num_elem < I40IW_MIN_CQ_SIZE || info->num_elem > I40IW_MAX_CQ_SIZE)
- return I40IW_ERR_INVALID_SIZE;
-
- if (info->ceq_id > I40IW_MAX_CEQID)
- return I40IW_ERR_INVALID_CEQ_ID;
-
- pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
-
- if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
- return I40IW_ERR_INVALID_PBLE_INDEX;
-
- cq->cq_pa = info->cq_pa;
- cq->cq_uk.cq_base = info->cq_base;
- cq->shadow_area_pa = info->shadow_area_pa;
- cq->cq_uk.shadow_area = info->shadow_area;
- cq->shadow_read_threshold = info->shadow_read_threshold;
- cq->dev = info->dev;
- cq->ceq_id = info->ceq_id;
- cq->cq_uk.cq_size = info->num_elem;
- cq->cq_type = I40IW_CQ_TYPE_CQP;
- cq->ceqe_mask = info->ceqe_mask;
- I40IW_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);
-
- cq->cq_uk.cq_id = 0; /* control cq is id 0 always */
- cq->ceq_id_valid = info->ceq_id_valid;
- cq->tph_en = info->tph_en;
- cq->tph_val = info->tph_val;
- cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;
-
- cq->pbl_list = info->pbl_list;
- cq->virtual_map = info->virtual_map;
- cq->pbl_chunk_size = info->pbl_chunk_size;
- cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
- cq->cq_uk.polarity = true;
-
- /* following are only for iw cqs so initialize them to zero */
- cq->cq_uk.cqe_alloc_reg = NULL;
- info->dev->ccq = cq;
- return 0;
-}
-
-/**
- * i40iw_sc_ccq_create_done - poll cqp for ccq create
- * @ccq: ccq sc struct
- */
-static enum i40iw_status_code i40iw_sc_ccq_create_done(struct i40iw_sc_cq *ccq)
-{
- struct i40iw_sc_cqp *cqp;
-
- cqp = ccq->dev->cqp;
- return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CQ, NULL);
-}
-
-/**
- * i40iw_sc_ccq_create - create control cq
- * @ccq: ccq sc struct
- * @scratch: u64 saved to be used during cqp completion
- * @check_overflow: overlow flag for ccq
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_ccq_create(struct i40iw_sc_cq *ccq,
- u64 scratch,
- bool check_overflow,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
- enum i40iw_status_code ret_code;
-
- cqp = ccq->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
- set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
- set_64bit_val(wqe, 16,
- LS_64(ccq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
- set_64bit_val(wqe, 32, (ccq->virtual_map ? 0 : ccq->cq_pa));
- set_64bit_val(wqe, 40, ccq->shadow_area_pa);
- set_64bit_val(wqe, 48,
- (ccq->virtual_map ? ccq->first_pm_pbl_idx : 0));
- set_64bit_val(wqe, 56,
- LS_64(ccq->tph_val, I40IW_CQPSQ_TPHVAL));
-
- header = ccq->cq_uk.cq_id |
- LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
- LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
- LS_64(ccq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
- LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
- LS_64(ccq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
- LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
- LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) |
- LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_CREATE WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq) {
- i40iw_sc_cqp_post_sq(cqp);
- ret_code = i40iw_sc_ccq_create_done(ccq);
- if (ret_code)
- return ret_code;
- }
- cqp->process_cqp_sds = i40iw_cqp_sds_cmd;
-
- return 0;
-}
-
-/**
- * i40iw_sc_ccq_destroy - destroy ccq during close
- * @ccq: ccq sc struct
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_ccq_destroy(struct i40iw_sc_cq *ccq,
- u64 scratch,
- bool post_sq)
-{
- struct i40iw_sc_cqp *cqp;
- u64 *wqe;
- u64 header;
- enum i40iw_status_code ret_code = 0;
- u32 tail, val, error;
-
- cqp = ccq->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
- set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
- set_64bit_val(wqe, 40, ccq->shadow_area_pa);
-
- header = ccq->cq_uk.cq_id |
- LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
- LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |
- LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
- LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) |
- LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_DESTROY WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
- if (error)
- return I40IW_ERR_CQP_COMPL_ERROR;
-
- if (post_sq) {
- i40iw_sc_cqp_post_sq(cqp);
- ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
- }
-
- cqp->process_cqp_sds = i40iw_update_sds_noccq;
-
- return ret_code;
-}
-
-/**
- * i40iw_sc_cq_init - initialize completion q
- * @cq: cq struct
- * @info: cq initialization info
- */
-static enum i40iw_status_code i40iw_sc_cq_init(struct i40iw_sc_cq *cq,
- struct i40iw_cq_init_info *info)
-{
- u32 __iomem *cqe_alloc_reg = NULL;
- enum i40iw_status_code ret_code;
- u32 pble_obj_cnt;
- u32 arm_offset;
-
- pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
-
- if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
- return I40IW_ERR_INVALID_PBLE_INDEX;
-
- cq->cq_pa = info->cq_base_pa;
- cq->dev = info->dev;
- cq->ceq_id = info->ceq_id;
- arm_offset = (info->dev->is_pf) ? I40E_PFPE_CQARM : I40E_VFPE_CQARM1;
- if (i40iw_get_hw_addr(cq->dev))
- cqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(cq->dev) +
- arm_offset);
- info->cq_uk_init_info.cqe_alloc_reg = cqe_alloc_reg;
- ret_code = i40iw_cq_uk_init(&cq->cq_uk, &info->cq_uk_init_info);
- if (ret_code)
- return ret_code;
- cq->virtual_map = info->virtual_map;
- cq->pbl_chunk_size = info->pbl_chunk_size;
- cq->ceqe_mask = info->ceqe_mask;
- cq->cq_type = (info->type) ? info->type : I40IW_CQ_TYPE_IWARP;
-
- cq->shadow_area_pa = info->shadow_area_pa;
- cq->shadow_read_threshold = info->shadow_read_threshold;
-
- cq->ceq_id_valid = info->ceq_id_valid;
- cq->tph_en = info->tph_en;
- cq->tph_val = info->tph_val;
-
- cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
-
- return 0;
-}
-
-/**
- * i40iw_sc_cq_create - create completion q
- * @cq: cq struct
- * @scratch: u64 saved to be used during cqp completion
- * @check_overflow: flag for overflow check
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_cq_create(struct i40iw_sc_cq *cq,
- u64 scratch,
- bool check_overflow,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
-
- if (cq->cq_uk.cq_id > I40IW_MAX_CQID)
- return I40IW_ERR_INVALID_CQ_ID;
-
- if (cq->ceq_id > I40IW_MAX_CEQID)
- return I40IW_ERR_INVALID_CEQ_ID;
-
- cqp = cq->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
- set_64bit_val(wqe, 8, RS_64_1(cq, 1));
- set_64bit_val(wqe,
- 16,
- LS_64(cq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
-
- set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
-
- set_64bit_val(wqe, 40, cq->shadow_area_pa);
- set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
- set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
-
- header = cq->cq_uk.cq_id |
- LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
- LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
- LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
- LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
- LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
- LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
- LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
- LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_CREATE WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_cq_destroy - destroy completion q
- * @cq: cq struct
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_cq_destroy(struct i40iw_sc_cq *cq,
- u64 scratch,
- bool post_sq)
-{
- struct i40iw_sc_cqp *cqp;
- u64 *wqe;
- u64 header;
-
- cqp = cq->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
- set_64bit_val(wqe, 8, RS_64_1(cq, 1));
- set_64bit_val(wqe, 40, cq->shadow_area_pa);
- set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
-
- header = cq->cq_uk.cq_id |
- LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
- LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |
- LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
- LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
- LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
- LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
- LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_DESTROY WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_cq_modify - modify a Completion Queue
- * @cq: cq struct
- * @info: modification info struct
- * @scratch:
- * @post_sq: flag to post to sq
- */
-static enum i40iw_status_code i40iw_sc_cq_modify(struct i40iw_sc_cq *cq,
- struct i40iw_modify_cq_info *info,
- u64 scratch,
- bool post_sq)
-{
- struct i40iw_sc_cqp *cqp;
- u64 *wqe;
- u64 header;
- u32 cq_size, ceq_id, first_pm_pbl_idx;
- u8 pbl_chunk_size;
- bool virtual_map, ceq_id_valid, check_overflow;
- u32 pble_obj_cnt;
-
- if (info->ceq_valid && (info->ceq_id > I40IW_MAX_CEQID))
- return I40IW_ERR_INVALID_CEQ_ID;
-
- pble_obj_cnt = cq->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
-
- if (info->cq_resize && info->virtual_map &&
- (info->first_pm_pbl_idx >= pble_obj_cnt))
- return I40IW_ERR_INVALID_PBLE_INDEX;
-
- cqp = cq->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- cq->pbl_list = info->pbl_list;
- cq->cq_pa = info->cq_pa;
- cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
-
- cq_size = info->cq_resize ? info->cq_size : cq->cq_uk.cq_size;
- if (info->ceq_change) {
- ceq_id_valid = true;
- ceq_id = info->ceq_id;
- } else {
- ceq_id_valid = cq->ceq_id_valid;
- ceq_id = ceq_id_valid ? cq->ceq_id : 0;
- }
- virtual_map = info->cq_resize ? info->virtual_map : cq->virtual_map;
- first_pm_pbl_idx = (info->cq_resize ?
- (info->virtual_map ? info->first_pm_pbl_idx : 0) :
- (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
- pbl_chunk_size = (info->cq_resize ?
- (info->virtual_map ? info->pbl_chunk_size : 0) :
- (cq->virtual_map ? cq->pbl_chunk_size : 0));
- check_overflow = info->check_overflow_change ? info->check_overflow :
- cq->check_overflow;
- cq->cq_uk.cq_size = cq_size;
- cq->ceq_id_valid = ceq_id_valid;
- cq->ceq_id = ceq_id;
- cq->virtual_map = virtual_map;
- cq->first_pm_pbl_idx = first_pm_pbl_idx;
- cq->pbl_chunk_size = pbl_chunk_size;
- cq->check_overflow = check_overflow;
-
- set_64bit_val(wqe, 0, cq_size);
- set_64bit_val(wqe, 8, RS_64_1(cq, 1));
- set_64bit_val(wqe, 16,
- LS_64(info->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
- set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
- set_64bit_val(wqe, 40, cq->shadow_area_pa);
- set_64bit_val(wqe, 48, (cq->virtual_map ? first_pm_pbl_idx : 0));
- set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
-
- header = cq->cq_uk.cq_id |
- LS_64(ceq_id, I40IW_CQPSQ_CQ_CEQID) |
- LS_64(I40IW_CQP_OP_MODIFY_CQ, I40IW_CQPSQ_OPCODE) |
- LS_64(info->cq_resize, I40IW_CQPSQ_CQ_CQRESIZE) |
- LS_64(pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
- LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
- LS_64(virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
- LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
- LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
- LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_MODIFY WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_qp_init - initialize qp
- * @qp: sc qp
- * @info: initialization qp info
- */
-static enum i40iw_status_code i40iw_sc_qp_init(struct i40iw_sc_qp *qp,
- struct i40iw_qp_init_info *info)
-{
- u32 __iomem *wqe_alloc_reg = NULL;
- enum i40iw_status_code ret_code;
- u32 pble_obj_cnt;
- u8 wqe_size;
- u32 offset;
-
- qp->dev = info->pd->dev;
- qp->vsi = info->vsi;
- qp->sq_pa = info->sq_pa;
- qp->rq_pa = info->rq_pa;
- qp->hw_host_ctx_pa = info->host_ctx_pa;
- qp->q2_pa = info->q2_pa;
- qp->shadow_area_pa = info->shadow_area_pa;
-
- qp->q2_buf = info->q2;
- qp->pd = info->pd;
- qp->hw_host_ctx = info->host_ctx;
- offset = (qp->pd->dev->is_pf) ? I40E_PFPE_WQEALLOC : I40E_VFPE_WQEALLOC1;
- if (i40iw_get_hw_addr(qp->pd->dev))
- wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) +
- offset);
-
- info->qp_uk_init_info.wqe_alloc_reg = wqe_alloc_reg;
- info->qp_uk_init_info.abi_ver = qp->pd->abi_ver;
- ret_code = i40iw_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info);
- if (ret_code)
- return ret_code;
- qp->virtual_map = info->virtual_map;
-
- pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
-
- if ((info->virtual_map && (info->sq_pa >= pble_obj_cnt)) ||
- (info->virtual_map && (info->rq_pa >= pble_obj_cnt)))
- return I40IW_ERR_INVALID_PBLE_INDEX;
-
- qp->llp_stream_handle = (void *)(-1);
- qp->qp_type = (info->type) ? info->type : I40IW_QP_TYPE_IWARP;
-
- qp->hw_sq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,
- false);
- i40iw_debug(qp->dev, I40IW_DEBUG_WQE, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n",
- __func__, qp->hw_sq_size, qp->qp_uk.sq_ring.size);
-
- switch (qp->pd->abi_ver) {
- case 4:
- ret_code = i40iw_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
- &wqe_size);
- if (ret_code)
- return ret_code;
- break;
- case 5: /* fallthrough until next ABI version */
- default:
- if (qp->qp_uk.max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
- return I40IW_ERR_INVALID_FRAG_COUNT;
- wqe_size = I40IW_MAX_WQE_SIZE_RQ;
- break;
- }
- qp->hw_rq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.rq_size *
- (wqe_size / I40IW_QP_WQE_MIN_SIZE), false);
- i40iw_debug(qp->dev, I40IW_DEBUG_WQE,
- "%s: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
- __func__, qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
- qp->sq_tph_val = info->sq_tph_val;
- qp->rq_tph_val = info->rq_tph_val;
- qp->sq_tph_en = info->sq_tph_en;
- qp->rq_tph_en = info->rq_tph_en;
- qp->rcv_tph_en = info->rcv_tph_en;
- qp->xmit_tph_en = info->xmit_tph_en;
- qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
-
- return 0;
-}
-
-/**
- * i40iw_sc_qp_create - create qp
- * @qp: sc qp
- * @info: qp create info
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_qp_create(
- struct i40iw_sc_qp *qp,
- struct i40iw_create_qp_info *info,
- u64 scratch,
- bool post_sq)
-{
- struct i40iw_sc_cqp *cqp;
- u64 *wqe;
- u64 header;
-
- if ((qp->qp_uk.qp_id < I40IW_MIN_IW_QP_ID) ||
- (qp->qp_uk.qp_id > I40IW_MAX_IW_QP_ID))
- return I40IW_ERR_INVALID_QP_ID;
-
- cqp = qp->pd->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
-
- set_64bit_val(wqe, 40, qp->shadow_area_pa);
-
- header = qp->qp_uk.qp_id |
- LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) |
- LS_64((info->ord_valid ? 1 : 0), I40IW_CQPSQ_QP_ORDVALID) |
- LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |
- LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
- LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) |
- LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |
- LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |
- LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_CREATE WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_qp_modify - modify qp cqp wqe
- * @qp: sc qp
- * @info: modify qp info
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_qp_modify(
- struct i40iw_sc_qp *qp,
- struct i40iw_modify_qp_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
- u8 term_actions = 0;
- u8 term_len = 0;
-
- cqp = qp->pd->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- if (info->next_iwarp_state == I40IW_QP_STATE_TERMINATE) {
- if (info->dont_send_fin)
- term_actions += I40IWQP_TERM_SEND_TERM_ONLY;
- if (info->dont_send_term)
- term_actions += I40IWQP_TERM_SEND_FIN_ONLY;
- if ((term_actions == I40IWQP_TERM_SEND_TERM_AND_FIN) ||
- (term_actions == I40IWQP_TERM_SEND_TERM_ONLY))
- term_len = info->termlen;
- }
-
- set_64bit_val(wqe,
- 8,
- LS_64(term_len, I40IW_CQPSQ_QP_TERMLEN));
-
- set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
- set_64bit_val(wqe, 40, qp->shadow_area_pa);
-
- header = qp->qp_uk.qp_id |
- LS_64(I40IW_CQP_OP_MODIFY_QP, I40IW_CQPSQ_OPCODE) |
- LS_64(info->ord_valid, I40IW_CQPSQ_QP_ORDVALID) |
- LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |
- LS_64(info->cached_var_valid, I40IW_CQPSQ_QP_CACHEDVARVALID) |
- LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) |
- LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |
- LS_64(info->force_loopback, I40IW_CQPSQ_QP_FORCELOOPBACK) |
- LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
- LS_64(info->remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |
- LS_64(term_actions, I40IW_CQPSQ_QP_TERMACT) |
- LS_64(info->reset_tcp_conn, I40IW_CQPSQ_QP_RESETCON) |
- LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |
- LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_MODIFY WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_qp_destroy - cqp destroy qp
- * @qp: sc qp
- * @scratch: u64 saved to be used during cqp completion
- * @remove_hash_idx: flag if to remove hash idx
- * @ignore_mw_bnd: memory window bind flag
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_qp_destroy(
- struct i40iw_sc_qp *qp,
- u64 scratch,
- bool remove_hash_idx,
- bool ignore_mw_bnd,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
-
- i40iw_qp_rem_qos(qp);
- cqp = qp->pd->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
- set_64bit_val(wqe, 40, qp->shadow_area_pa);
-
- header = qp->qp_uk.qp_id |
- LS_64(I40IW_CQP_OP_DESTROY_QP, I40IW_CQPSQ_OPCODE) |
- LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
- LS_64(ignore_mw_bnd, I40IW_CQPSQ_QP_IGNOREMWBOUND) |
- LS_64(remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_DESTROY WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_qp_flush_wqes - flush qp's wqe
- * @qp: sc qp
- * @info: dlush information
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_qp_flush_wqes(
- struct i40iw_sc_qp *qp,
- struct i40iw_qp_flush_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 temp = 0;
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
- bool flush_sq = false, flush_rq = false;
-
- if (info->rq && !qp->flush_rq)
- flush_rq = true;
-
- if (info->sq && !qp->flush_sq)
- flush_sq = true;
-
- qp->flush_sq |= flush_sq;
- qp->flush_rq |= flush_rq;
- if (!flush_sq && !flush_rq)
- return 0;
-
- cqp = qp->pd->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- if (info->userflushcode) {
- if (flush_rq) {
- temp |= LS_64(info->rq_minor_code, I40IW_CQPSQ_FWQE_RQMNERR) |
- LS_64(info->rq_major_code, I40IW_CQPSQ_FWQE_RQMJERR);
- }
- if (flush_sq) {
- temp |= LS_64(info->sq_minor_code, I40IW_CQPSQ_FWQE_SQMNERR) |
- LS_64(info->sq_major_code, I40IW_CQPSQ_FWQE_SQMJERR);
- }
- }
- set_64bit_val(wqe, 16, temp);
-
- temp = (info->generate_ae) ?
- info->ae_code | LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE) : 0;
-
- set_64bit_val(wqe, 8, temp);
-
- header = qp->qp_uk.qp_id |
- LS_64(I40IW_CQP_OP_FLUSH_WQES, I40IW_CQPSQ_OPCODE) |
- LS_64(info->generate_ae, I40IW_CQPSQ_FWQE_GENERATE_AE) |
- LS_64(info->userflushcode, I40IW_CQPSQ_FWQE_USERFLCODE) |
- LS_64(flush_sq, I40IW_CQPSQ_FWQE_FLUSHSQ) |
- LS_64(flush_rq, I40IW_CQPSQ_FWQE_FLUSHRQ) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_FLUSH WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_gen_ae - generate AE, currently uses flush WQE CQP OP
- * @qp: sc qp
- * @info: gen ae information
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_gen_ae(
- struct i40iw_sc_qp *qp,
- struct i40iw_gen_ae_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 temp;
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
-
- cqp = qp->pd->dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- temp = info->ae_code |
- LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE);
-
- set_64bit_val(wqe, 8, temp);
-
- header = qp->qp_uk.qp_id |
- LS_64(I40IW_CQP_OP_GEN_AE, I40IW_CQPSQ_OPCODE) |
- LS_64(1, I40IW_CQPSQ_FWQE_GENERATE_AE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "GEN_AE WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_qp_upload_context - upload qp's context
- * @dev: sc device struct
- * @info: upload context info ptr for return
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_qp_upload_context(
- struct i40iw_sc_dev *dev,
- struct i40iw_upload_context_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
-
- cqp = dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 16, info->buf_pa);
-
- header = LS_64(info->qp_id, I40IW_CQPSQ_UCTX_QPID) |
- LS_64(I40IW_CQP_OP_UPLOAD_CONTEXT, I40IW_CQPSQ_OPCODE) |
- LS_64(info->qp_type, I40IW_CQPSQ_UCTX_QPTYPE) |
- LS_64(info->raw_format, I40IW_CQPSQ_UCTX_RAWFORMAT) |
- LS_64(info->freeze_qp, I40IW_CQPSQ_UCTX_FREEZEQP) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QP_UPLOAD_CTX WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_qp_setctx - set qp's context
- * @qp: sc qp
- * @qp_ctx: context ptr
- * @info: ctx info
- */
-static enum i40iw_status_code i40iw_sc_qp_setctx(
- struct i40iw_sc_qp *qp,
- u64 *qp_ctx,
- struct i40iw_qp_host_ctx_info *info)
-{
- struct i40iwarp_offload_info *iw;
- struct i40iw_tcp_offload_info *tcp;
- struct i40iw_sc_vsi *vsi;
- struct i40iw_sc_dev *dev;
- u64 qw0, qw3, qw7 = 0;
-
- iw = info->iwarp_info;
- tcp = info->tcp_info;
- vsi = qp->vsi;
- dev = qp->dev;
- if (info->add_to_qoslist) {
- qp->user_pri = info->user_pri;
- i40iw_qp_add_qos(qp);
- i40iw_debug(qp->dev, I40IW_DEBUG_DCB, "%s qp[%d] UP[%d] qset[%d]\n",
- __func__, qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle);
- }
- qw0 = LS_64(qp->qp_uk.rq_wqe_size, I40IWQPC_RQWQESIZE) |
- LS_64(info->err_rq_idx_valid, I40IWQPC_ERR_RQ_IDX_VALID) |
- LS_64(qp->rcv_tph_en, I40IWQPC_RCVTPHEN) |
- LS_64(qp->xmit_tph_en, I40IWQPC_XMITTPHEN) |
- LS_64(qp->rq_tph_en, I40IWQPC_RQTPHEN) |
- LS_64(qp->sq_tph_en, I40IWQPC_SQTPHEN) |
- LS_64(info->push_idx, I40IWQPC_PPIDX) |
- LS_64(info->push_mode_en, I40IWQPC_PMENA);
-
- set_64bit_val(qp_ctx, 8, qp->sq_pa);
- set_64bit_val(qp_ctx, 16, qp->rq_pa);
-
- qw3 = LS_64(qp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |
- LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) |
- LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE);
-
- set_64bit_val(qp_ctx,
- 128,
- LS_64(info->err_rq_idx, I40IWQPC_ERR_RQ_IDX));
-
- set_64bit_val(qp_ctx,
- 136,
- LS_64(info->send_cq_num, I40IWQPC_TXCQNUM) |
- LS_64(info->rcv_cq_num, I40IWQPC_RXCQNUM));
-
- set_64bit_val(qp_ctx,
- 168,
- LS_64(info->qp_compl_ctx, I40IWQPC_QPCOMPCTX));
- set_64bit_val(qp_ctx,
- 176,
- LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) |
- LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) |
- LS_64(qp->qs_handle, I40IWQPC_QSHANDLE) |
- LS_64(vsi->exception_lan_queue, I40IWQPC_EXCEPTION_LAN_QUEUE));
-
- if (info->iwarp_info_valid) {
- qw0 |= LS_64(iw->ddp_ver, I40IWQPC_DDP_VER) |
- LS_64(iw->rdmap_ver, I40IWQPC_RDMAP_VER);
-
- qw7 |= LS_64(iw->pd_id, I40IWQPC_PDIDX);
- set_64bit_val(qp_ctx,
- 144,
- LS_64(qp->q2_pa, I40IWQPC_Q2ADDR) |
- LS_64(vsi->fcn_id, I40IWQPC_STAT_INDEX));
- set_64bit_val(qp_ctx,
- 152,
- LS_64(iw->last_byte_sent, I40IWQPC_LASTBYTESENT));
-
- set_64bit_val(qp_ctx,
- 160,
- LS_64(iw->ord_size, I40IWQPC_ORDSIZE) |
- LS_64(iw->ird_size, I40IWQPC_IRDSIZE) |
- LS_64(iw->wr_rdresp_en, I40IWQPC_WRRDRSPOK) |
- LS_64(iw->rd_enable, I40IWQPC_RDOK) |
- LS_64(iw->snd_mark_en, I40IWQPC_SNDMARKERS) |
- LS_64(iw->bind_en, I40IWQPC_BINDEN) |
- LS_64(iw->fast_reg_en, I40IWQPC_FASTREGEN) |
- LS_64(iw->priv_mode_en, I40IWQPC_PRIVEN) |
- LS_64((((vsi->stats_fcn_id_alloc) &&
- (dev->is_pf) && (vsi->fcn_id >= I40IW_FIRST_NON_PF_STAT)) ? 1 : 0),
- I40IWQPC_USESTATSINSTANCE) |
- LS_64(1, I40IWQPC_IWARPMODE) |
- LS_64(iw->rcv_mark_en, I40IWQPC_RCVMARKERS) |
- LS_64(iw->align_hdrs, I40IWQPC_ALIGNHDRS) |
- LS_64(iw->rcv_no_mpa_crc, I40IWQPC_RCVNOMPACRC) |
- LS_64(iw->rcv_mark_offset, I40IWQPC_RCVMARKOFFSET) |
- LS_64(iw->snd_mark_offset, I40IWQPC_SNDMARKOFFSET));
- }
- if (info->tcp_info_valid) {
- qw0 |= LS_64(tcp->ipv4, I40IWQPC_IPV4) |
- LS_64(tcp->no_nagle, I40IWQPC_NONAGLE) |
- LS_64(tcp->insert_vlan_tag, I40IWQPC_INSERTVLANTAG) |
- LS_64(tcp->time_stamp, I40IWQPC_TIMESTAMP) |
- LS_64(tcp->cwnd_inc_limit, I40IWQPC_LIMIT) |
- LS_64(tcp->drop_ooo_seg, I40IWQPC_DROPOOOSEG) |
- LS_64(tcp->dup_ack_thresh, I40IWQPC_DUPACK_THRESH);
-
- qw3 |= LS_64(tcp->ttl, I40IWQPC_TTL) |
- LS_64(tcp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |
- LS_64(tcp->avoid_stretch_ack, I40IWQPC_AVOIDSTRETCHACK) |
- LS_64(tcp->tos, I40IWQPC_TOS) |
- LS_64(tcp->src_port, I40IWQPC_SRCPORTNUM) |
- LS_64(tcp->dst_port, I40IWQPC_DESTPORTNUM);
-
- qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
- set_64bit_val(qp_ctx,
- 32,
- LS_64(tcp->dest_ip_addr2, I40IWQPC_DESTIPADDR2) |
- LS_64(tcp->dest_ip_addr3, I40IWQPC_DESTIPADDR3));
-
- set_64bit_val(qp_ctx,
- 40,
- LS_64(tcp->dest_ip_addr0, I40IWQPC_DESTIPADDR0) |
- LS_64(tcp->dest_ip_addr1, I40IWQPC_DESTIPADDR1));
-
- set_64bit_val(qp_ctx,
- 48,
- LS_64(tcp->snd_mss, I40IWQPC_SNDMSS) |
- LS_64(tcp->vlan_tag, I40IWQPC_VLANTAG) |
- LS_64(tcp->arp_idx, I40IWQPC_ARPIDX));
-
- qw7 |= LS_64(tcp->flow_label, I40IWQPC_FLOWLABEL) |
- LS_64(tcp->wscale, I40IWQPC_WSCALE) |
- LS_64(tcp->ignore_tcp_opt, I40IWQPC_IGNORE_TCP_OPT) |
- LS_64(tcp->ignore_tcp_uns_opt, I40IWQPC_IGNORE_TCP_UNS_OPT) |
- LS_64(tcp->tcp_state, I40IWQPC_TCPSTATE) |
- LS_64(tcp->rcv_wscale, I40IWQPC_RCVSCALE) |
- LS_64(tcp->snd_wscale, I40IWQPC_SNDSCALE);
-
- set_64bit_val(qp_ctx,
- 72,
- LS_64(tcp->time_stamp_recent, I40IWQPC_TIMESTAMP_RECENT) |
- LS_64(tcp->time_stamp_age, I40IWQPC_TIMESTAMP_AGE));
- set_64bit_val(qp_ctx,
- 80,
- LS_64(tcp->snd_nxt, I40IWQPC_SNDNXT) |
- LS_64(tcp->snd_wnd, I40IWQPC_SNDWND));
-
- set_64bit_val(qp_ctx,
- 88,
- LS_64(tcp->rcv_nxt, I40IWQPC_RCVNXT) |
- LS_64(tcp->rcv_wnd, I40IWQPC_RCVWND));
- set_64bit_val(qp_ctx,
- 96,
- LS_64(tcp->snd_max, I40IWQPC_SNDMAX) |
- LS_64(tcp->snd_una, I40IWQPC_SNDUNA));
- set_64bit_val(qp_ctx,
- 104,
- LS_64(tcp->srtt, I40IWQPC_SRTT) |
- LS_64(tcp->rtt_var, I40IWQPC_RTTVAR));
- set_64bit_val(qp_ctx,
- 112,
- LS_64(tcp->ss_thresh, I40IWQPC_SSTHRESH) |
- LS_64(tcp->cwnd, I40IWQPC_CWND));
- set_64bit_val(qp_ctx,
- 120,
- LS_64(tcp->snd_wl1, I40IWQPC_SNDWL1) |
- LS_64(tcp->snd_wl2, I40IWQPC_SNDWL2));
- set_64bit_val(qp_ctx,
- 128,
- LS_64(tcp->max_snd_window, I40IWQPC_MAXSNDWND) |
- LS_64(tcp->rexmit_thresh, I40IWQPC_REXMIT_THRESH));
- set_64bit_val(qp_ctx,
- 184,
- LS_64(tcp->local_ipaddr3, I40IWQPC_LOCAL_IPADDR3) |
- LS_64(tcp->local_ipaddr2, I40IWQPC_LOCAL_IPADDR2));
- set_64bit_val(qp_ctx,
- 192,
- LS_64(tcp->local_ipaddr1, I40IWQPC_LOCAL_IPADDR1) |
- LS_64(tcp->local_ipaddr0, I40IWQPC_LOCAL_IPADDR0));
- }
-
- set_64bit_val(qp_ctx, 0, qw0);
- set_64bit_val(qp_ctx, 24, qw3);
- set_64bit_val(qp_ctx, 56, qw7);
-
- i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "QP_HOST)CTX WQE",
- qp_ctx, I40IW_QP_CTX_SIZE);
- return 0;
-}
-
-/**
- * i40iw_sc_alloc_stag - mr stag alloc
- * @dev: sc device struct
- * @info: stag info
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_alloc_stag(
- struct i40iw_sc_dev *dev,
- struct i40iw_allocate_stag_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
- enum i40iw_page_size page_size;
-
- page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
- cqp = dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe,
- 8,
- LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID) |
- LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN));
- set_64bit_val(wqe,
- 16,
- LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
- set_64bit_val(wqe,
- 40,
- LS_64(info->hmc_fcn_index, I40IW_CQPSQ_STAG_HMCFNIDX));
-
- header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |
- LS_64(1, I40IW_CQPSQ_STAG_MR) |
- LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
- LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
- LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
- LS_64(info->remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
- LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
- LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "ALLOC_STAG WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_mr_reg_non_shared - non-shared mr registration
- * @dev: sc device struct
- * @info: mr info
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_mr_reg_non_shared(
- struct i40iw_sc_dev *dev,
- struct i40iw_reg_ns_stag_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- u64 temp;
- struct i40iw_sc_cqp *cqp;
- u64 header;
- u32 pble_obj_cnt;
- bool remote_access;
- u8 addr_type;
- enum i40iw_page_size page_size;
-
- page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
- if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
- I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
- remote_access = true;
- else
- remote_access = false;
-
- pble_obj_cnt = dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
-
- if (info->chunk_size && (info->first_pm_pbl_index >= pble_obj_cnt))
- return I40IW_ERR_INVALID_PBLE_INDEX;
-
- cqp = dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
- set_64bit_val(wqe, 0, temp);
-
- set_64bit_val(wqe,
- 8,
- LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN) |
- LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
-
- set_64bit_val(wqe,
- 16,
- LS_64(info->stag_key, I40IW_CQPSQ_STAG_KEY) |
- LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
- if (!info->chunk_size) {
- set_64bit_val(wqe, 32, info->reg_addr_pa);
- set_64bit_val(wqe, 48, 0);
- } else {
- set_64bit_val(wqe, 32, 0);
- set_64bit_val(wqe, 48, info->first_pm_pbl_index);
- }
- set_64bit_val(wqe, 40, info->hmc_fcn_index);
- set_64bit_val(wqe, 56, 0);
-
- addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
- header = LS_64(I40IW_CQP_OP_REG_MR, I40IW_CQPSQ_OPCODE) |
- LS_64(1, I40IW_CQPSQ_STAG_MR) |
- LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
- LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
- LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
- LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
- LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
- LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
- LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_NS WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_mr_reg_shared - registered shared memory region
- * @dev: sc device struct
- * @info: info for shared memory registeration
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_mr_reg_shared(
- struct i40iw_sc_dev *dev,
- struct i40iw_register_shared_stag *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 temp, va64, fbo, header;
- u32 va32;
- bool remote_access;
- u8 addr_type;
-
- if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
- I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
- remote_access = true;
- else
- remote_access = false;
- cqp = dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- va64 = (uintptr_t)(info->va);
- va32 = (u32)(va64 & 0x00000000FFFFFFFF);
- fbo = (u64)(va32 & (4096 - 1));
-
- set_64bit_val(wqe,
- 0,
- (info->addr_type == I40IW_ADDR_TYPE_VA_BASED ? (uintptr_t)info->va : fbo));
-
- set_64bit_val(wqe,
- 8,
- LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
- temp = LS_64(info->new_stag_key, I40IW_CQPSQ_STAG_KEY) |
- LS_64(info->new_stag_idx, I40IW_CQPSQ_STAG_IDX) |
- LS_64(info->parent_stag_idx, I40IW_CQPSQ_STAG_PARENTSTAGIDX);
- set_64bit_val(wqe, 16, temp);
-
- addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
- header = LS_64(I40IW_CQP_OP_REG_SMR, I40IW_CQPSQ_OPCODE) |
- LS_64(1, I40IW_CQPSQ_STAG_MR) |
- LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
- LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
- LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_SHARED WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_dealloc_stag - deallocate stag
- * @dev: sc device struct
- * @info: dealloc stag info
- * @scratch: u64 saved to be used during cqp completion
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_dealloc_stag(
- struct i40iw_sc_dev *dev,
- struct i40iw_dealloc_stag_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 header;
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
-
- cqp = dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe,
- 8,
- LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
- set_64bit_val(wqe,
- 16,
- LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
-
- header = LS_64(I40IW_CQP_OP_DEALLOC_STAG, I40IW_CQPSQ_OPCODE) |
- LS_64(info->mr, I40IW_CQPSQ_STAG_MR) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "DEALLOC_STAG WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_query_stag - query hardware for stag
- * @dev: sc device struct
- * @scratch: u64 saved to be used during cqp completion
- * @stag_index: stag index for query
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_query_stag(struct i40iw_sc_dev *dev,
- u64 scratch,
- u32 stag_index,
- bool post_sq)
-{
- u64 header;
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
-
- cqp = dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe,
- 16,
- LS_64(stag_index, I40IW_CQPSQ_QUERYSTAG_IDX));
-
- header = LS_64(I40IW_CQP_OP_QUERY_STAG, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QUERY_STAG WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_mw_alloc - mw allocate
- * @dev: sc device struct
- * @scratch: u64 saved to be used during cqp completion
- * @mw_stag_index:stag index
- * @pd_id: pd is for this mw
- * @post_sq: flag for cqp db to ring
- */
-static enum i40iw_status_code i40iw_sc_mw_alloc(
- struct i40iw_sc_dev *dev,
- u64 scratch,
- u32 mw_stag_index,
- u16 pd_id,
- bool post_sq)
-{
- u64 header;
- struct i40iw_sc_cqp *cqp;
- u64 *wqe;
-
- cqp = dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe, 8, LS_64(pd_id, I40IW_CQPSQ_STAG_PDID));
- set_64bit_val(wqe,
- 16,
- LS_64(mw_stag_index, I40IW_CQPSQ_STAG_IDX));
-
- header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MW_ALLOC WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
- * @qp: sc qp struct
- * @info: fast mr info
- * @post_sq: flag for cqp db to ring
- */
-enum i40iw_status_code i40iw_sc_mr_fast_register(
- struct i40iw_sc_qp *qp,
- struct i40iw_fast_reg_stag_info *info,
- bool post_sq)
-{
- u64 temp, header;
- u64 *wqe;
- u32 wqe_idx;
- enum i40iw_page_size page_size;
-
- page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
- wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
- 0, info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
-
- i40iw_debug(qp->dev, I40IW_DEBUG_MR, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
- __func__, info->wr_id, wqe_idx,
- &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
- temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
- set_64bit_val(wqe, 0, temp);
-
- temp = RS_64(info->first_pm_pbl_index >> 16, I40IWQPSQ_FIRSTPMPBLIDXHI);
- set_64bit_val(wqe,
- 8,
- LS_64(temp, I40IWQPSQ_FIRSTPMPBLIDXHI) |
- LS_64(info->reg_addr_pa >> I40IWQPSQ_PBLADDR_SHIFT, I40IWQPSQ_PBLADDR));
-
- set_64bit_val(wqe,
- 16,
- info->total_len |
- LS_64(info->first_pm_pbl_index, I40IWQPSQ_FIRSTPMPBLIDXLO));
-
- header = LS_64(info->stag_key, I40IWQPSQ_STAGKEY) |
- LS_64(info->stag_idx, I40IWQPSQ_STAGINDEX) |
- LS_64(I40IWQP_OP_FAST_REGISTER, I40IWQPSQ_OPCODE) |
- LS_64(info->chunk_size, I40IWQPSQ_LPBLSIZE) |
- LS_64(page_size, I40IWQPSQ_HPAGESIZE) |
- LS_64(info->access_rights, I40IWQPSQ_STAGRIGHTS) |
- LS_64(info->addr_type, I40IWQPSQ_VABASEDTO) |
- LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
- LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "FAST_REG WQE",
- wqe, I40IW_QP_WQE_MIN_SIZE);
-
- if (post_sq)
- i40iw_qp_post_wr(&qp->qp_uk);
- return 0;
-}
-
-/**
- * i40iw_sc_send_lsmm - send last streaming mode message
- * @qp: sc qp struct
- * @lsmm_buf: buffer with lsmm message
- * @size: size of lsmm buffer
- * @stag: stag of lsmm buffer
- */
-static void i40iw_sc_send_lsmm(struct i40iw_sc_qp *qp,
- void *lsmm_buf,
- u32 size,
- i40iw_stag stag)
-{
- u64 *wqe;
- u64 header;
- struct i40iw_qp_uk *qp_uk;
-
- qp_uk = &qp->qp_uk;
- wqe = qp_uk->sq_base->elem;
-
- set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
-
- set_64bit_val(wqe, 8, (size | LS_64(stag, I40IWQPSQ_FRAG_STAG)));
-
- set_64bit_val(wqe, 16, 0);
-
- header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
- LS_64(1, I40IWQPSQ_STREAMMODE) |
- LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |
- LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(qp->dev, I40IW_DEBUG_QP, "SEND_LSMM WQE",
- wqe, I40IW_QP_WQE_MIN_SIZE);
-}
-
-/**
- * i40iw_sc_send_lsmm_nostag - for privilege qp
- * @qp: sc qp struct
- * @lsmm_buf: buffer with lsmm message
- * @size: size of lsmm buffer
- */
-static void i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp *qp,
- void *lsmm_buf,
- u32 size)
-{
- u64 *wqe;
- u64 header;
- struct i40iw_qp_uk *qp_uk;
-
- qp_uk = &qp->qp_uk;
- wqe = qp_uk->sq_base->elem;
-
- set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
-
- set_64bit_val(wqe, 8, size);
-
- set_64bit_val(wqe, 16, 0);
-
- header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
- LS_64(1, I40IWQPSQ_STREAMMODE) |
- LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |
- LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE",
- wqe, I40IW_QP_WQE_MIN_SIZE);
-}
-
-/**
- * i40iw_sc_send_rtt - send last read0 or write0
- * @qp: sc qp struct
- * @read: Do read0 or write0
- */
-static void i40iw_sc_send_rtt(struct i40iw_sc_qp *qp, bool read)
-{
- u64 *wqe;
- u64 header;
- struct i40iw_qp_uk *qp_uk;
-
- qp_uk = &qp->qp_uk;
- wqe = qp_uk->sq_base->elem;
-
- set_64bit_val(wqe, 0, 0);
- set_64bit_val(wqe, 8, 0);
- set_64bit_val(wqe, 16, 0);
- if (read) {
- header = LS_64(0x1234, I40IWQPSQ_REMSTAG) |
- LS_64(I40IWQP_OP_RDMA_READ, I40IWQPSQ_OPCODE) |
- LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
- set_64bit_val(wqe, 8, ((u64)0xabcd << 32));
- } else {
- header = LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |
- LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
- }
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "RTR WQE",
- wqe, I40IW_QP_WQE_MIN_SIZE);
-}
-
-/**
- * i40iw_sc_post_wqe0 - send wqe with opcode
- * @qp: sc qp struct
- * @opcode: opcode to use for wqe0
- */
-static enum i40iw_status_code i40iw_sc_post_wqe0(struct i40iw_sc_qp *qp, u8 opcode)
-{
- u64 *wqe;
- u64 header;
- struct i40iw_qp_uk *qp_uk;
-
- qp_uk = &qp->qp_uk;
- wqe = qp_uk->sq_base->elem;
-
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- switch (opcode) {
- case I40IWQP_OP_NOP:
- set_64bit_val(wqe, 0, 0);
- set_64bit_val(wqe, 8, 0);
- set_64bit_val(wqe, 16, 0);
- header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |
- LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
- break;
- case I40IWQP_OP_RDMA_SEND:
- set_64bit_val(wqe, 0, 0);
- set_64bit_val(wqe, 8, 0);
- set_64bit_val(wqe, 16, 0);
- header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
- LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID) |
- LS_64(1, I40IWQPSQ_STREAMMODE) |
- LS_64(1, I40IWQPSQ_WAITFORRCVPDU);
-
- i40iw_insert_wqe_hdr(wqe, header);
- break;
- default:
- i40iw_debug(qp->dev, I40IW_DEBUG_QP, "%s: Invalid WQE zero opcode\n",
- __func__);
- break;
- }
- return 0;
-}
-
-/**
- * i40iw_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
- * @dev : ptr to i40iw_dev struct
- * @hmc_fn_id: hmc function id
- */
-enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, u8 hmc_fn_id)
-{
- struct i40iw_hmc_info *hmc_info;
- struct i40iw_dma_mem query_fpm_mem;
- struct i40iw_virt_mem virt_mem;
- struct i40iw_vfdev *vf_dev = NULL;
- u32 mem_size;
- enum i40iw_status_code ret_code = 0;
- bool poll_registers = true;
- u16 iw_vf_idx;
- u8 wait_type;
-
- if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID ||
- (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
- return I40IW_ERR_INVALID_HMCFN_ID;
-
- i40iw_debug(dev, I40IW_DEBUG_HMC, "hmc_fn_id %u, dev->hmc_fn_id %u\n", hmc_fn_id,
- dev->hmc_fn_id);
- if (hmc_fn_id == dev->hmc_fn_id) {
- hmc_info = dev->hmc_info;
- query_fpm_mem.pa = dev->fpm_query_buf_pa;
- query_fpm_mem.va = dev->fpm_query_buf;
- } else {
- vf_dev = i40iw_vfdev_from_fpm(dev, hmc_fn_id);
- if (!vf_dev)
- return I40IW_ERR_INVALID_VF_ID;
-
- hmc_info = &vf_dev->hmc_info;
- iw_vf_idx = vf_dev->iw_vf_idx;
- i40iw_debug(dev, I40IW_DEBUG_HMC, "vf_dev %p, hmc_info %p, hmc_obj %p\n", vf_dev,
- hmc_info, hmc_info->hmc_obj);
- if (!vf_dev->fpm_query_buf) {
- if (!dev->vf_fpm_query_buf[iw_vf_idx].va) {
- ret_code = i40iw_alloc_query_fpm_buf(dev,
- &dev->vf_fpm_query_buf[iw_vf_idx]);
- if (ret_code)
- return ret_code;
- }
- vf_dev->fpm_query_buf = dev->vf_fpm_query_buf[iw_vf_idx].va;
- vf_dev->fpm_query_buf_pa = dev->vf_fpm_query_buf[iw_vf_idx].pa;
- }
- query_fpm_mem.pa = vf_dev->fpm_query_buf_pa;
- query_fpm_mem.va = vf_dev->fpm_query_buf;
- /**
- * It is HARDWARE specific:
- * this call is done by PF for VF and
- * i40iw_sc_query_fpm_values needs ccq poll
- * because PF ccq is already created.
- */
- poll_registers = false;
- }
-
- hmc_info->hmc_fn_id = hmc_fn_id;
-
- if (hmc_fn_id != dev->hmc_fn_id) {
- ret_code =
- i40iw_cqp_query_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id);
- } else {
- wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
- (u8)I40IW_CQP_WAIT_POLL_CQ;
-
- ret_code = i40iw_sc_query_fpm_values(
- dev->cqp,
- 0,
- hmc_info->hmc_fn_id,
- &query_fpm_mem,
- true,
- wait_type);
- }
- if (ret_code)
- return ret_code;
-
- /* parse the fpm_query_buf and fill hmc obj info */
- ret_code =
- i40iw_sc_parse_fpm_query_buf((u64 *)query_fpm_mem.va,
- hmc_info,
- &dev->hmc_fpm_misc);
- if (ret_code)
- return ret_code;
- i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "QUERY FPM BUFFER",
- query_fpm_mem.va, I40IW_QUERY_FPM_BUF_SIZE);
-
- if (hmc_fn_id != dev->hmc_fn_id) {
- i40iw_cqp_commit_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id);
-
- /* parse the fpm_commit_buf and fill hmc obj info */
- i40iw_sc_parse_fpm_commit_buf((u64 *)query_fpm_mem.va, hmc_info->hmc_obj, &hmc_info->sd_table.sd_cnt);
- mem_size = sizeof(struct i40iw_hmc_sd_entry) *
- (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index);
- ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
- if (ret_code)
- return ret_code;
- hmc_info->sd_table.sd_entry = virt_mem.va;
- }
-
- return ret_code;
-}
-
-/**
- * i40iw_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and
- * populates fpm base address in hmc_info
- * @dev : ptr to i40iw_dev struct
- * @hmc_fn_id: hmc function id
- */
-static enum i40iw_status_code i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev *dev,
- u8 hmc_fn_id)
-{
- struct i40iw_hmc_info *hmc_info;
- struct i40iw_hmc_obj_info *obj_info;
- u64 *buf;
- struct i40iw_dma_mem commit_fpm_mem;
- u32 i, j;
- enum i40iw_status_code ret_code = 0;
- bool poll_registers = true;
- u8 wait_type;
-
- if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID ||
- (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
- return I40IW_ERR_INVALID_HMCFN_ID;
-
- if (hmc_fn_id == dev->hmc_fn_id) {
- hmc_info = dev->hmc_info;
- } else {
- hmc_info = i40iw_vf_hmcinfo_from_fpm(dev, hmc_fn_id);
- poll_registers = false;
- }
- if (!hmc_info)
- return I40IW_ERR_BAD_PTR;
-
- obj_info = hmc_info->hmc_obj;
- buf = dev->fpm_commit_buf;
-
- /* copy cnt values in commit buf */
- for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE;
- i++, j += 8)
- set_64bit_val(buf, j, (u64)obj_info[i].cnt);
-
- set_64bit_val(buf, 40, 0); /* APBVT rsvd */
-
- commit_fpm_mem.pa = dev->fpm_commit_buf_pa;
- commit_fpm_mem.va = dev->fpm_commit_buf;
- wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
- (u8)I40IW_CQP_WAIT_POLL_CQ;
- ret_code = i40iw_sc_commit_fpm_values(
- dev->cqp,
- 0,
- hmc_info->hmc_fn_id,
- &commit_fpm_mem,
- true,
- wait_type);
-
- /* parse the fpm_commit_buf and fill hmc obj info */
- if (!ret_code)
- ret_code = i40iw_sc_parse_fpm_commit_buf(dev->fpm_commit_buf,
- hmc_info->hmc_obj,
- &hmc_info->sd_table.sd_cnt);
-
- i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "COMMIT FPM BUFFER",
- commit_fpm_mem.va, I40IW_COMMIT_FPM_BUF_SIZE);
-
- return ret_code;
-}
-
-/**
- * cqp_sds_wqe_fill - fill cqp wqe doe sd
- * @cqp: struct for cqp hw
- * @info; sd info for wqe
- * @scratch: u64 saved to be used during cqp completion
- */
-static enum i40iw_status_code cqp_sds_wqe_fill(struct i40iw_sc_cqp *cqp,
- struct i40iw_update_sds_info *info,
- u64 scratch)
-{
- u64 data;
- u64 header;
- u64 *wqe;
- int mem_entries, wqe_entries;
- struct i40iw_dma_mem *sdbuf = &cqp->sdbuf;
- u64 offset;
- u32 wqe_idx;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- I40IW_CQP_INIT_WQE(wqe);
- wqe_entries = (info->cnt > 3) ? 3 : info->cnt;
- mem_entries = info->cnt - wqe_entries;
-
- header = LS_64(I40IW_CQP_OP_UPDATE_PE_SDS, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
- LS_64(mem_entries, I40IW_CQPSQ_UPESD_ENTRY_COUNT);
-
- if (mem_entries) {
- offset = wqe_idx * I40IW_UPDATE_SD_BUF_SIZE;
- memcpy((char *)sdbuf->va + offset, &info->entry[3],
- mem_entries << 4);
- data = (u64)sdbuf->pa + offset;
- } else {
- data = 0;
- }
- data |= LS_64(info->hmc_fn_id, I40IW_CQPSQ_UPESD_HMCFNID);
-
- set_64bit_val(wqe, 16, data);
-
- switch (wqe_entries) {
- case 3:
- set_64bit_val(wqe, 48,
- (LS_64(info->entry[2].cmd, I40IW_CQPSQ_UPESD_SDCMD) |
- LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
-
- set_64bit_val(wqe, 56, info->entry[2].data);
- /* fallthrough */
- case 2:
- set_64bit_val(wqe, 32,
- (LS_64(info->entry[1].cmd, I40IW_CQPSQ_UPESD_SDCMD) |
- LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
-
- set_64bit_val(wqe, 40, info->entry[1].data);
- /* fallthrough */
- case 1:
- set_64bit_val(wqe, 0,
- LS_64(info->entry[0].cmd, I40IW_CQPSQ_UPESD_SDCMD));
-
- set_64bit_val(wqe, 8, info->entry[0].data);
- break;
- default:
- break;
- }
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "UPDATE_PE_SDS WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
- return 0;
-}
-
-/**
- * i40iw_update_pe_sds - cqp wqe for sd
- * @dev: ptr to i40iw_dev struct
- * @info: sd info for sd's
- * @scratch: u64 saved to be used during cqp completion
- */
-static enum i40iw_status_code i40iw_update_pe_sds(struct i40iw_sc_dev *dev,
- struct i40iw_update_sds_info *info,
- u64 scratch)
-{
- struct i40iw_sc_cqp *cqp = dev->cqp;
- enum i40iw_status_code ret_code;
-
- ret_code = cqp_sds_wqe_fill(cqp, info, scratch);
- if (!ret_code)
- i40iw_sc_cqp_post_sq(cqp);
-
- return ret_code;
-}
-
-/**
- * i40iw_update_sds_noccq - update sd before ccq created
- * @dev: sc device struct
- * @info: sd info for sd's
- */
-enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev,
- struct i40iw_update_sds_info *info)
-{
- u32 error, val, tail;
- struct i40iw_sc_cqp *cqp = dev->cqp;
- enum i40iw_status_code ret_code;
-
- ret_code = cqp_sds_wqe_fill(cqp, info, 0);
- if (ret_code)
- return ret_code;
- i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
- if (error)
- return I40IW_ERR_CQP_COMPL_ERROR;
-
- i40iw_sc_cqp_post_sq(cqp);
- ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
-
- return ret_code;
-}
-
-/**
- * i40iw_sc_suspend_qp - suspend qp for param change
- * @cqp: struct for cqp hw
- * @qp: sc qp struct
- * @scratch: u64 saved to be used during cqp completion
- */
-enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp,
- struct i40iw_sc_qp *qp,
- u64 scratch)
-{
- u64 header;
- u64 *wqe;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_SUSPENDQP_QPID) |
- LS_64(I40IW_CQP_OP_SUSPEND_QP, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SUSPEND_QP WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_resume_qp - resume qp after suspend
- * @cqp: struct for cqp hw
- * @qp: sc qp struct
- * @scratch: u64 saved to be used during cqp completion
- */
-enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp,
- struct i40iw_sc_qp *qp,
- u64 scratch)
-{
- u64 header;
- u64 *wqe;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe,
- 16,
- LS_64(qp->qs_handle, I40IW_CQPSQ_RESUMEQP_QSHANDLE));
-
- header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_RESUMEQP_QPID) |
- LS_64(I40IW_CQP_OP_RESUME_QP, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "RESUME_QP WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-/**
- * i40iw_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
- * @cqp: struct for cqp hw
- * @scratch: u64 saved to be used during cqp completion
- * @hmc_fn_id: hmc function id
- * @post_sq: flag for cqp db to ring
- * @poll_registers: flag to poll register for cqp completion
- */
-enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(
- struct i40iw_sc_cqp *cqp,
- u64 scratch,
- u8 hmc_fn_id,
- bool post_sq,
- bool poll_registers)
-{
- u64 header;
- u64 *wqe;
- u32 tail, val, error;
- enum i40iw_status_code ret_code = 0;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
- set_64bit_val(wqe,
- 16,
- LS_64(hmc_fn_id, I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID));
-
- header = LS_64(I40IW_CQP_OP_SHMC_PAGES_ALLOCATED, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
- i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
- if (error) {
- ret_code = I40IW_ERR_CQP_COMPL_ERROR;
- return ret_code;
- }
- if (post_sq) {
- i40iw_sc_cqp_post_sq(cqp);
- if (poll_registers)
- /* check for cqp sq tail update */
- ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
- else
- ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
- I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
- NULL);
- }
-
- return ret_code;
-}
-
-/**
- * i40iw_ring_full - check if cqp ring is full
- * @cqp: struct for cqp hw
- */
-static bool i40iw_ring_full(struct i40iw_sc_cqp *cqp)
-{
- return I40IW_RING_FULL_ERR(cqp->sq_ring);
-}
-
-/**
- * i40iw_est_sd - returns approximate number of SDs for HMC
- * @dev: sc device struct
- * @hmc_info: hmc structure, size and count for HMC objects
- */
-static u64 i40iw_est_sd(struct i40iw_sc_dev *dev, struct i40iw_hmc_info *hmc_info)
-{
- int i;
- u64 size = 0;
- u64 sd;
-
- for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_PBLE; i++)
- size += hmc_info->hmc_obj[i].cnt * hmc_info->hmc_obj[i].size;
-
- if (dev->is_pf)
- size += hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
-
- if (size & 0x1FFFFF)
- sd = (size >> 21) + 1; /* add 1 for remainder */
- else
- sd = size >> 21;
-
- if (!dev->is_pf) {
- /* 2MB alignment for VF PBLE HMC */
- size = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
- if (size & 0x1FFFFF)
- sd += (size >> 21) + 1; /* add 1 for remainder */
- else
- sd += size >> 21;
- }
-
- return sd;
-}
-
-/**
- * i40iw_config_fpm_values - configure HMC objects
- * @dev: sc device struct
- * @qp_count: desired qp count
- */
-enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count)
-{
- struct i40iw_virt_mem virt_mem;
- u32 i, mem_size;
- u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted;
- u64 sd_needed;
- u32 loop_count = 0;
-
- struct i40iw_hmc_info *hmc_info;
- struct i40iw_hmc_fpm_misc *hmc_fpm_misc;
- enum i40iw_status_code ret_code = 0;
-
- hmc_info = dev->hmc_info;
- hmc_fpm_misc = &dev->hmc_fpm_misc;
-
- ret_code = i40iw_sc_init_iw_hmc(dev, dev->hmc_fn_id);
- if (ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "i40iw_sc_init_iw_hmc returned error_code = %d\n",
- ret_code);
- return ret_code;
- }
-
- for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++)
- hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
- sd_needed = i40iw_est_sd(dev, hmc_info);
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "%s: FW initial max sd_count[%08lld] first_sd_index[%04d]\n",
- __func__, sd_needed, hmc_info->first_sd_index);
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "%s: sd count %d where max sd is %d\n",
- __func__, hmc_info->sd_table.sd_cnt,
- hmc_fpm_misc->max_sds);
-
- qpwanted = min(qp_count, hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt);
- qpwantedoriginal = qpwanted;
- mrwanted = hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt;
- pblewanted = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt;
-
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d\n",
- qp_count, hmc_fpm_misc->max_sds,
- hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt,
- hmc_info->hmc_obj[I40IW_HMC_IW_CQ].max_cnt,
- hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt,
- hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt);
-
- do {
- ++loop_count;
- hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt = qpwanted;
- hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt =
- min(2 * qpwanted, hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt);
- hmc_info->hmc_obj[I40IW_HMC_IW_SRQ].cnt = 0x00; /* Reserved */
- hmc_info->hmc_obj[I40IW_HMC_IW_HTE].cnt =
- qpwanted * hmc_fpm_misc->ht_multiplier;
- hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt =
- hmc_info->hmc_obj[I40IW_HMC_IW_ARP].max_cnt;
- hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].cnt = 1;
- hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt = mrwanted;
-
- hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt =
- roundup_pow_of_two(I40IW_MAX_WQ_ENTRIES * qpwanted);
- hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt =
- roundup_pow_of_two(2 * I40IW_MAX_IRD_SIZE * qpwanted);
- hmc_info->hmc_obj[I40IW_HMC_IW_XFFL].cnt =
- hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
- hmc_info->hmc_obj[I40IW_HMC_IW_Q1FL].cnt =
- hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;
- hmc_info->hmc_obj[I40IW_HMC_IW_TIMER].cnt =
- ((qpwanted) / 512 + 1) * hmc_fpm_misc->timer_bucket;
- hmc_info->hmc_obj[I40IW_HMC_IW_FSIMC].cnt = 0x00;
- hmc_info->hmc_obj[I40IW_HMC_IW_FSIAV].cnt = 0x00;
- hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt = pblewanted;
-
- /* How much memory is needed for all the objects. */
- sd_needed = i40iw_est_sd(dev, hmc_info);
- if ((loop_count > 1000) ||
- ((!(loop_count % 10)) &&
- (qpwanted > qpwantedoriginal * 2 / 3))) {
- if (qpwanted > FPM_MULTIPLIER)
- qpwanted = roundup_pow_of_two(qpwanted -
- FPM_MULTIPLIER);
- qpwanted >>= 1;
- }
- if (mrwanted > FPM_MULTIPLIER * 10)
- mrwanted -= FPM_MULTIPLIER * 10;
- if (pblewanted > FPM_MULTIPLIER * 1000)
- pblewanted -= FPM_MULTIPLIER * 1000;
- } while (sd_needed > hmc_fpm_misc->max_sds && loop_count < 2000);
-
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n",
- loop_count, sd_needed,
- hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt,
- hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt,
- hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt,
- hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt);
-
- ret_code = i40iw_sc_configure_iw_fpm(dev, dev->hmc_fn_id);
- if (ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "configure_iw_fpm returned error_code[x%08X]\n",
- i40iw_rd32(dev->hw, dev->is_pf ? I40E_PFPE_CQPERRCODES : I40E_VFPE_CQPERRCODES1));
- return ret_code;
- }
-
- mem_size = sizeof(struct i40iw_hmc_sd_entry) *
- (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
- ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
- if (ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "%s: failed to allocate memory for sd_entry buffer\n",
- __func__);
- return ret_code;
- }
- hmc_info->sd_table.sd_entry = virt_mem.va;
-
- return ret_code;
-}
-
-/**
- * i40iw_exec_cqp_cmd - execute cqp cmd when wqe are available
- * @dev: rdma device
- * @pcmdinfo: cqp command info
- */
-static enum i40iw_status_code i40iw_exec_cqp_cmd(struct i40iw_sc_dev *dev,
- struct cqp_commands_info *pcmdinfo)
-{
- enum i40iw_status_code status;
- struct i40iw_dma_mem values_mem;
-
- dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;
- switch (pcmdinfo->cqp_cmd) {
- case OP_DELETE_LOCAL_MAC_IPADDR_ENTRY:
- status = i40iw_sc_del_local_mac_ipaddr_entry(
- pcmdinfo->in.u.del_local_mac_ipaddr_entry.cqp,
- pcmdinfo->in.u.del_local_mac_ipaddr_entry.scratch,
- pcmdinfo->in.u.del_local_mac_ipaddr_entry.entry_idx,
- pcmdinfo->in.u.del_local_mac_ipaddr_entry.ignore_ref_count,
- pcmdinfo->post_sq);
- break;
- case OP_CEQ_DESTROY:
- status = i40iw_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,
- pcmdinfo->in.u.ceq_destroy.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_AEQ_DESTROY:
- status = i40iw_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
- pcmdinfo->in.u.aeq_destroy.scratch,
- pcmdinfo->post_sq);
-
- break;
- case OP_DELETE_ARP_CACHE_ENTRY:
- status = i40iw_sc_del_arp_cache_entry(
- pcmdinfo->in.u.del_arp_cache_entry.cqp,
- pcmdinfo->in.u.del_arp_cache_entry.scratch,
- pcmdinfo->in.u.del_arp_cache_entry.arp_index,
- pcmdinfo->post_sq);
- break;
- case OP_MANAGE_APBVT_ENTRY:
- status = i40iw_sc_manage_apbvt_entry(
- pcmdinfo->in.u.manage_apbvt_entry.cqp,
- &pcmdinfo->in.u.manage_apbvt_entry.info,
- pcmdinfo->in.u.manage_apbvt_entry.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_CEQ_CREATE:
- status = i40iw_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
- pcmdinfo->in.u.ceq_create.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_AEQ_CREATE:
- status = i40iw_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,
- pcmdinfo->in.u.aeq_create.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY:
- status = i40iw_sc_alloc_local_mac_ipaddr_entry(
- pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.cqp,
- pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_ADD_LOCAL_MAC_IPADDR_ENTRY:
- status = i40iw_sc_add_local_mac_ipaddr_entry(
- pcmdinfo->in.u.add_local_mac_ipaddr_entry.cqp,
- &pcmdinfo->in.u.add_local_mac_ipaddr_entry.info,
- pcmdinfo->in.u.add_local_mac_ipaddr_entry.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_MANAGE_QHASH_TABLE_ENTRY:
- status = i40iw_sc_manage_qhash_table_entry(
- pcmdinfo->in.u.manage_qhash_table_entry.cqp,
- &pcmdinfo->in.u.manage_qhash_table_entry.info,
- pcmdinfo->in.u.manage_qhash_table_entry.scratch,
- pcmdinfo->post_sq);
-
- break;
- case OP_QP_MODIFY:
- status = i40iw_sc_qp_modify(
- pcmdinfo->in.u.qp_modify.qp,
- &pcmdinfo->in.u.qp_modify.info,
- pcmdinfo->in.u.qp_modify.scratch,
- pcmdinfo->post_sq);
-
- break;
- case OP_QP_UPLOAD_CONTEXT:
- status = i40iw_sc_qp_upload_context(
- pcmdinfo->in.u.qp_upload_context.dev,
- &pcmdinfo->in.u.qp_upload_context.info,
- pcmdinfo->in.u.qp_upload_context.scratch,
- pcmdinfo->post_sq);
-
- break;
- case OP_CQ_CREATE:
- status = i40iw_sc_cq_create(
- pcmdinfo->in.u.cq_create.cq,
- pcmdinfo->in.u.cq_create.scratch,
- pcmdinfo->in.u.cq_create.check_overflow,
- pcmdinfo->post_sq);
- break;
- case OP_CQ_DESTROY:
- status = i40iw_sc_cq_destroy(
- pcmdinfo->in.u.cq_destroy.cq,
- pcmdinfo->in.u.cq_destroy.scratch,
- pcmdinfo->post_sq);
-
- break;
- case OP_QP_CREATE:
- status = i40iw_sc_qp_create(
- pcmdinfo->in.u.qp_create.qp,
- &pcmdinfo->in.u.qp_create.info,
- pcmdinfo->in.u.qp_create.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_QP_DESTROY:
- status = i40iw_sc_qp_destroy(
- pcmdinfo->in.u.qp_destroy.qp,
- pcmdinfo->in.u.qp_destroy.scratch,
- pcmdinfo->in.u.qp_destroy.remove_hash_idx,
- pcmdinfo->in.u.qp_destroy.
- ignore_mw_bnd,
- pcmdinfo->post_sq);
-
- break;
- case OP_ALLOC_STAG:
- status = i40iw_sc_alloc_stag(
- pcmdinfo->in.u.alloc_stag.dev,
- &pcmdinfo->in.u.alloc_stag.info,
- pcmdinfo->in.u.alloc_stag.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_MR_REG_NON_SHARED:
- status = i40iw_sc_mr_reg_non_shared(
- pcmdinfo->in.u.mr_reg_non_shared.dev,
- &pcmdinfo->in.u.mr_reg_non_shared.info,
- pcmdinfo->in.u.mr_reg_non_shared.scratch,
- pcmdinfo->post_sq);
-
- break;
- case OP_DEALLOC_STAG:
- status = i40iw_sc_dealloc_stag(
- pcmdinfo->in.u.dealloc_stag.dev,
- &pcmdinfo->in.u.dealloc_stag.info,
- pcmdinfo->in.u.dealloc_stag.scratch,
- pcmdinfo->post_sq);
-
- break;
- case OP_MW_ALLOC:
- status = i40iw_sc_mw_alloc(
- pcmdinfo->in.u.mw_alloc.dev,
- pcmdinfo->in.u.mw_alloc.scratch,
- pcmdinfo->in.u.mw_alloc.mw_stag_index,
- pcmdinfo->in.u.mw_alloc.pd_id,
- pcmdinfo->post_sq);
-
- break;
- case OP_QP_FLUSH_WQES:
- status = i40iw_sc_qp_flush_wqes(
- pcmdinfo->in.u.qp_flush_wqes.qp,
- &pcmdinfo->in.u.qp_flush_wqes.info,
- pcmdinfo->in.u.qp_flush_wqes.
- scratch, pcmdinfo->post_sq);
- break;
- case OP_GEN_AE:
- status = i40iw_sc_gen_ae(
- pcmdinfo->in.u.gen_ae.qp,
- &pcmdinfo->in.u.gen_ae.info,
- pcmdinfo->in.u.gen_ae.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_ADD_ARP_CACHE_ENTRY:
- status = i40iw_sc_add_arp_cache_entry(
- pcmdinfo->in.u.add_arp_cache_entry.cqp,
- &pcmdinfo->in.u.add_arp_cache_entry.info,
- pcmdinfo->in.u.add_arp_cache_entry.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_MANAGE_PUSH_PAGE:
- status = i40iw_sc_manage_push_page(
- pcmdinfo->in.u.manage_push_page.cqp,
- &pcmdinfo->in.u.manage_push_page.info,
- pcmdinfo->in.u.manage_push_page.scratch,
- pcmdinfo->post_sq);
- break;
- case OP_UPDATE_PE_SDS:
- /* case I40IW_CQP_OP_UPDATE_PE_SDS */
- status = i40iw_update_pe_sds(
- pcmdinfo->in.u.update_pe_sds.dev,
- &pcmdinfo->in.u.update_pe_sds.info,
- pcmdinfo->in.u.update_pe_sds.
- scratch);
-
- break;
- case OP_MANAGE_HMC_PM_FUNC_TABLE:
- status = i40iw_sc_manage_hmc_pm_func_table(
- pcmdinfo->in.u.manage_hmc_pm.dev->cqp,
- pcmdinfo->in.u.manage_hmc_pm.scratch,
- (u8)pcmdinfo->in.u.manage_hmc_pm.info.vf_id,
- pcmdinfo->in.u.manage_hmc_pm.info.free_fcn,
- true);
- break;
- case OP_SUSPEND:
- status = i40iw_sc_suspend_qp(
- pcmdinfo->in.u.suspend_resume.cqp,
- pcmdinfo->in.u.suspend_resume.qp,
- pcmdinfo->in.u.suspend_resume.scratch);
- break;
- case OP_RESUME:
- status = i40iw_sc_resume_qp(
- pcmdinfo->in.u.suspend_resume.cqp,
- pcmdinfo->in.u.suspend_resume.qp,
- pcmdinfo->in.u.suspend_resume.scratch);
- break;
- case OP_MANAGE_VF_PBLE_BP:
- status = i40iw_manage_vf_pble_bp(
- pcmdinfo->in.u.manage_vf_pble_bp.cqp,
- &pcmdinfo->in.u.manage_vf_pble_bp.info,
- pcmdinfo->in.u.manage_vf_pble_bp.scratch, true);
- break;
- case OP_QUERY_FPM_VALUES:
- values_mem.pa = pcmdinfo->in.u.query_fpm_values.fpm_values_pa;
- values_mem.va = pcmdinfo->in.u.query_fpm_values.fpm_values_va;
- status = i40iw_sc_query_fpm_values(
- pcmdinfo->in.u.query_fpm_values.cqp,
- pcmdinfo->in.u.query_fpm_values.scratch,
- pcmdinfo->in.u.query_fpm_values.hmc_fn_id,
- &values_mem, true, I40IW_CQP_WAIT_EVENT);
- break;
- case OP_COMMIT_FPM_VALUES:
- values_mem.pa = pcmdinfo->in.u.commit_fpm_values.fpm_values_pa;
- values_mem.va = pcmdinfo->in.u.commit_fpm_values.fpm_values_va;
- status = i40iw_sc_commit_fpm_values(
- pcmdinfo->in.u.commit_fpm_values.cqp,
- pcmdinfo->in.u.commit_fpm_values.scratch,
- pcmdinfo->in.u.commit_fpm_values.hmc_fn_id,
- &values_mem,
- true,
- I40IW_CQP_WAIT_EVENT);
- break;
- case OP_QUERY_RDMA_FEATURES:
- values_mem.pa = pcmdinfo->in.u.query_rdma_features.cap_pa;
- values_mem.va = pcmdinfo->in.u.query_rdma_features.cap_va;
- status = i40iw_sc_query_rdma_features(
- pcmdinfo->in.u.query_rdma_features.cqp, &values_mem,
- pcmdinfo->in.u.query_rdma_features.scratch);
- break;
- default:
- status = I40IW_NOT_SUPPORTED;
- break;
- }
-
- return status;
-}
-
-/**
- * i40iw_process_cqp_cmd - process all cqp commands
- * @dev: sc device struct
- * @pcmdinfo: cqp command info
- */
-enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,
- struct cqp_commands_info *pcmdinfo)
-{
- enum i40iw_status_code status = 0;
- unsigned long flags;
-
- spin_lock_irqsave(&dev->cqp_lock, flags);
- if (list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp))
- status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
- else
- list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);
- spin_unlock_irqrestore(&dev->cqp_lock, flags);
- return status;
-}
-
-/**
- * i40iw_process_bh - called from tasklet for cqp list
- * @dev: sc device struct
- */
-enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev)
-{
- enum i40iw_status_code status = 0;
- struct cqp_commands_info *pcmdinfo;
- unsigned long flags;
-
- spin_lock_irqsave(&dev->cqp_lock, flags);
- while (!list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp)) {
- pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&dev->cqp_cmd_head);
-
- status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
- if (status)
- break;
- }
- spin_unlock_irqrestore(&dev->cqp_lock, flags);
- return status;
-}
-
-/**
- * i40iw_iwarp_opcode - determine if incoming is rdma layer
- * @info: aeq info for the packet
- * @pkt: packet for error
- */
-static u32 i40iw_iwarp_opcode(struct i40iw_aeqe_info *info, u8 *pkt)
-{
- __be16 *mpa;
- u32 opcode = 0xffffffff;
-
- if (info->q2_data_written) {
- mpa = (__be16 *)pkt;
- opcode = ntohs(mpa[1]) & 0xf;
- }
- return opcode;
-}
-
-/**
- * i40iw_locate_mpa - return pointer to mpa in the pkt
- * @pkt: packet with data
- */
-static u8 *i40iw_locate_mpa(u8 *pkt)
-{
- /* skip over ethernet header */
- pkt += I40IW_MAC_HLEN;
-
- /* Skip over IP and TCP headers */
- pkt += 4 * (pkt[0] & 0x0f);
- pkt += 4 * ((pkt[12] >> 4) & 0x0f);
- return pkt;
-}
-
-/**
- * i40iw_setup_termhdr - termhdr for terminate pkt
- * @qp: sc qp ptr for pkt
- * @hdr: term hdr
- * @opcode: flush opcode for termhdr
- * @layer_etype: error layer + error type
- * @err: error cod ein the header
- */
-static void i40iw_setup_termhdr(struct i40iw_sc_qp *qp,
- struct i40iw_terminate_hdr *hdr,
- enum i40iw_flush_opcode opcode,
- u8 layer_etype,
- u8 err)
-{
- qp->flush_code = opcode;
- hdr->layer_etype = layer_etype;
- hdr->error_code = err;
-}
-
-/**
- * i40iw_bld_terminate_hdr - build terminate message header
- * @qp: qp associated with received terminate AE
- * @info: the struct contiaing AE information
- */
-static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp,
- struct i40iw_aeqe_info *info)
-{
- u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
- u16 ddp_seg_len;
- int copy_len = 0;
- u8 is_tagged = 0;
- u32 opcode;
- struct i40iw_terminate_hdr *termhdr;
-
- termhdr = (struct i40iw_terminate_hdr *)qp->q2_buf;
- memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
-
- if (info->q2_data_written) {
- /* Use data from offending packet to fill in ddp & rdma hdrs */
- pkt = i40iw_locate_mpa(pkt);
- ddp_seg_len = ntohs(*(__be16 *)pkt);
- if (ddp_seg_len) {
- copy_len = 2;
- termhdr->hdrct = DDP_LEN_FLAG;
- if (pkt[2] & 0x80) {
- is_tagged = 1;
- if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
- copy_len += TERM_DDP_LEN_TAGGED;
- termhdr->hdrct |= DDP_HDR_FLAG;
- }
- } else {
- if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
- copy_len += TERM_DDP_LEN_UNTAGGED;
- termhdr->hdrct |= DDP_HDR_FLAG;
- }
-
- if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
- if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
- copy_len += TERM_RDMA_LEN;
- termhdr->hdrct |= RDMA_HDR_FLAG;
- }
- }
- }
- }
- }
-
- opcode = i40iw_iwarp_opcode(info, pkt);
-
- switch (info->ae_id) {
- case I40IW_AE_AMP_UNALLOCATED_STAG:
- qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
- if (opcode == I40IW_OP_TYPE_RDMA_WRITE)
- i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
- (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_STAG);
- else
- i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
- break;
- case I40IW_AE_AMP_BOUNDS_VIOLATION:
- qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
- if (info->q2_data_written)
- i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
- (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_BOUNDS);
- else
- i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_BOUNDS);
- break;
- case I40IW_AE_AMP_BAD_PD:
- switch (opcode) {
- case I40IW_OP_TYPE_RDMA_WRITE:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
- (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_UNASSOC_STAG);
- break;
- case I40IW_OP_TYPE_SEND_INV:
- case I40IW_OP_TYPE_SEND_SOL_INV:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_CANT_INV_STAG);
- break;
- default:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_UNASSOC_STAG);
- }
- break;
- case I40IW_AE_AMP_INVALID_STAG:
- qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
- i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
- break;
- case I40IW_AE_AMP_BAD_QP:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
- (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
- break;
- case I40IW_AE_AMP_BAD_STAG_KEY:
- case I40IW_AE_AMP_BAD_STAG_INDEX:
- qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
- switch (opcode) {
- case I40IW_OP_TYPE_SEND_INV:
- case I40IW_OP_TYPE_SEND_SOL_INV:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_CANT_INV_STAG);
- break;
- default:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_STAG);
- }
- break;
- case I40IW_AE_AMP_RIGHTS_VIOLATION:
- case I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
- case I40IW_AE_PRIV_OPERATION_DENIED:
- qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
- i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_ACCESS);
- break;
- case I40IW_AE_AMP_TO_WRAP:
- qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
- i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_TO_WRAP);
- break;
- case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
- (LAYER_MPA << 4) | DDP_LLP, MPA_CRC);
- break;
- case I40IW_AE_LLP_SEGMENT_TOO_LARGE:
- case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
- (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
- break;
- case I40IW_AE_LCE_QP_CATASTROPHIC:
- case I40IW_AE_DDP_NO_L_BIT:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
- (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
- break;
- case I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
- (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_RANGE);
- break;
- case I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
- qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
- i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
- (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_TOO_LONG);
- break;
- case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
- if (is_tagged)
- i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
- (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_DDP_VER);
- else
- i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
- (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_DDP_VER);
- break;
- case I40IW_AE_DDP_UBE_INVALID_MO:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
- (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MO);
- break;
- case I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
- (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_NO_BUF);
- break;
- case I40IW_AE_DDP_UBE_INVALID_QN:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
- (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
- break;
- case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_RDMAP_VER);
- break;
- case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNEXPECTED_OP);
- break;
- default:
- i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
- (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNSPECIFIED);
- break;
- }
-
- if (copy_len)
- memcpy(termhdr + 1, pkt, copy_len);
-
- return sizeof(struct i40iw_terminate_hdr) + copy_len;
-}
-
-/**
- * i40iw_terminate_send_fin() - Send fin for terminate message
- * @qp: qp associated with received terminate AE
- */
-void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp)
-{
- /* Send the fin only */
- i40iw_term_modify_qp(qp,
- I40IW_QP_STATE_TERMINATE,
- I40IWQP_TERM_SEND_FIN_ONLY,
- 0);
-}
-
-/**
- * i40iw_terminate_connection() - Bad AE and send terminate to remote QP
- * @qp: qp associated with received terminate AE
- * @info: the struct contiaing AE information
- */
-void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)
-{
- u8 termlen = 0;
-
- if (qp->term_flags & I40IW_TERM_SENT)
- return; /* Sanity check */
-
- /* Eventtype can change from bld_terminate_hdr */
- qp->eventtype = TERM_EVENT_QP_FATAL;
- termlen = i40iw_bld_terminate_hdr(qp, info);
- i40iw_terminate_start_timer(qp);
- qp->term_flags |= I40IW_TERM_SENT;
- i40iw_term_modify_qp(qp, I40IW_QP_STATE_TERMINATE,
- I40IWQP_TERM_SEND_TERM_ONLY, termlen);
-}
-
-/**
- * i40iw_terminate_received - handle terminate received AE
- * @qp: qp associated with received terminate AE
- * @info: the struct contiaing AE information
- */
-void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)
-{
- u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
- __be32 *mpa;
- u8 ddp_ctl;
- u8 rdma_ctl;
- u16 aeq_id = 0;
- struct i40iw_terminate_hdr *termhdr;
-
- mpa = (__be32 *)i40iw_locate_mpa(pkt);
- if (info->q2_data_written) {
- /* did not validate the frame - do it now */
- ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;
- rdma_ctl = ntohl(mpa[0]) & 0xff;
- if ((ddp_ctl & 0xc0) != 0x40)
- aeq_id = I40IW_AE_LCE_QP_CATASTROPHIC;
- else if ((ddp_ctl & 0x03) != 1)
- aeq_id = I40IW_AE_DDP_UBE_INVALID_DDP_VERSION;
- else if (ntohl(mpa[2]) != 2)
- aeq_id = I40IW_AE_DDP_UBE_INVALID_QN;
- else if (ntohl(mpa[3]) != 1)
- aeq_id = I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN;
- else if (ntohl(mpa[4]) != 0)
- aeq_id = I40IW_AE_DDP_UBE_INVALID_MO;
- else if ((rdma_ctl & 0xc0) != 0x40)
- aeq_id = I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
-
- info->ae_id = aeq_id;
- if (info->ae_id) {
- /* Bad terminate recvd - send back a terminate */
- i40iw_terminate_connection(qp, info);
- return;
- }
- }
-
- qp->term_flags |= I40IW_TERM_RCVD;
- qp->eventtype = TERM_EVENT_QP_FATAL;
- termhdr = (struct i40iw_terminate_hdr *)&mpa[5];
- if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||
- termhdr->layer_etype == RDMAP_REMOTE_OP) {
- i40iw_terminate_done(qp, 0);
- } else {
- i40iw_terminate_start_timer(qp);
- i40iw_terminate_send_fin(qp);
- }
-}
-
-/**
- * i40iw_sc_vsi_init - Initialize virtual device
- * @vsi: pointer to the vsi structure
- * @info: parameters to initialize vsi
- **/
-void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info)
-{
- int i;
-
- vsi->dev = info->dev;
- vsi->back_vsi = info->back_vsi;
- vsi->mtu = info->params->mtu;
- vsi->exception_lan_queue = info->exception_lan_queue;
- i40iw_fill_qos_list(info->params->qs_handle_list);
-
- for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
- vsi->qos[i].qs_handle = info->params->qs_handle_list[i];
- i40iw_debug(vsi->dev, I40IW_DEBUG_DCB, "qset[%d]: %d\n", i,
- vsi->qos[i].qs_handle);
- spin_lock_init(&vsi->qos[i].lock);
- INIT_LIST_HEAD(&vsi->qos[i].qplist);
- }
-}
-
-/**
- * i40iw_hw_stats_init - Initiliaze HW stats table
- * @stats: pestat struct
- * @fcn_idx: PCI fn id
- * @is_pf: Is it a PF?
- *
- * Populate the HW stats table with register offset addr for each
- * stats. And start the perioidic stats timer.
- */
-void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 fcn_idx, bool is_pf)
-{
- u32 stats_reg_offset;
- u32 stats_index;
- struct i40iw_dev_hw_stats_offsets *stats_table =
- &stats->hw_stats_offsets;
- struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
-
- if (is_pf) {
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
- I40E_GLPES_PFIP4RXDISCARD(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
- I40E_GLPES_PFIP4RXTRUNC(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
- I40E_GLPES_PFIP4TXNOROUTE(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
- I40E_GLPES_PFIP6RXDISCARD(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
- I40E_GLPES_PFIP6RXTRUNC(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
- I40E_GLPES_PFIP6TXNOROUTE(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
- I40E_GLPES_PFTCPRTXSEG(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
- I40E_GLPES_PFTCPRXOPTERR(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
- I40E_GLPES_PFTCPRXPROTOERR(fcn_idx);
-
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
- I40E_GLPES_PFIP4RXOCTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
- I40E_GLPES_PFIP4RXPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
- I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
- I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
- I40E_GLPES_PFIP4TXOCTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
- I40E_GLPES_PFIP4TXPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
- I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
- I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
- I40E_GLPES_PFIP6RXOCTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
- I40E_GLPES_PFIP6RXPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
- I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
- I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
- I40E_GLPES_PFIP6TXOCTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
- I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
- I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
- I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
- I40E_GLPES_PFTCPRXSEGSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
- I40E_GLPES_PFTCPTXSEGLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
- I40E_GLPES_PFRDMARXRDSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
- I40E_GLPES_PFRDMARXSNDSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
- I40E_GLPES_PFRDMARXWRSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
- I40E_GLPES_PFRDMATXRDSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
- I40E_GLPES_PFRDMATXSNDSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
- I40E_GLPES_PFRDMATXWRSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
- I40E_GLPES_PFRDMAVBNDLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
- I40E_GLPES_PFRDMAVINVLO(fcn_idx);
- } else {
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
- I40E_GLPES_VFIP4RXDISCARD(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
- I40E_GLPES_VFIP4RXTRUNC(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
- I40E_GLPES_VFIP4TXNOROUTE(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
- I40E_GLPES_VFIP6RXDISCARD(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
- I40E_GLPES_VFIP6RXTRUNC(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
- I40E_GLPES_VFIP6TXNOROUTE(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
- I40E_GLPES_VFTCPRTXSEG(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
- I40E_GLPES_VFTCPRXOPTERR(fcn_idx);
- stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
- I40E_GLPES_VFTCPRXPROTOERR(fcn_idx);
-
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
- I40E_GLPES_VFIP4RXOCTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
- I40E_GLPES_VFIP4RXPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
- I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
- I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
- I40E_GLPES_VFIP4TXOCTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
- I40E_GLPES_VFIP4TXPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
- I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
- I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
- I40E_GLPES_VFIP6RXOCTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
- I40E_GLPES_VFIP6RXPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
- I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
- I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
- I40E_GLPES_VFIP6TXOCTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
- I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
- I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
- I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
- I40E_GLPES_VFTCPRXSEGSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
- I40E_GLPES_VFTCPTXSEGLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
- I40E_GLPES_VFRDMARXRDSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
- I40E_GLPES_VFRDMARXSNDSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
- I40E_GLPES_VFRDMARXWRSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
- I40E_GLPES_VFRDMATXRDSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
- I40E_GLPES_VFRDMATXSNDSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
- I40E_GLPES_VFRDMATXWRSLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
- I40E_GLPES_VFRDMAVBNDLO(fcn_idx);
- stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
- I40E_GLPES_VFRDMAVINVLO(fcn_idx);
- }
-
- for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
- stats_index++) {
- stats_reg_offset = stats_table->stats_offset_64[stats_index];
- last_rd_stats->stats_value_64[stats_index] =
- readq(stats->hw->hw_addr + stats_reg_offset);
- }
-
- for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
- stats_index++) {
- stats_reg_offset = stats_table->stats_offset_32[stats_index];
- last_rd_stats->stats_value_32[stats_index] =
- i40iw_rd32(stats->hw, stats_reg_offset);
- }
-}
-
-/**
- * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs.
- * @stat: pestat struct
- * @index: index in HW stats table which contains offset reg-addr
- * @value: hw stats value
- */
-void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,
- enum i40iw_hw_stats_index_32b index,
- u64 *value)
-{
- struct i40iw_dev_hw_stats_offsets *stats_table =
- &stats->hw_stats_offsets;
- struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
- struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
- u64 new_stats_value = 0;
- u32 stats_reg_offset = stats_table->stats_offset_32[index];
-
- new_stats_value = i40iw_rd32(stats->hw, stats_reg_offset);
- /*roll-over case */
- if (new_stats_value < last_rd_stats->stats_value_32[index])
- hw_stats->stats_value_32[index] += new_stats_value;
- else
- hw_stats->stats_value_32[index] +=
- new_stats_value - last_rd_stats->stats_value_32[index];
- last_rd_stats->stats_value_32[index] = new_stats_value;
- *value = hw_stats->stats_value_32[index];
-}
-
-/**
- * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs.
- * @stats: pestat struct
- * @index: index in HW stats table which contains offset reg-addr
- * @value: hw stats value
- */
-void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,
- enum i40iw_hw_stats_index_64b index,
- u64 *value)
-{
- struct i40iw_dev_hw_stats_offsets *stats_table =
- &stats->hw_stats_offsets;
- struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
- struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
- u64 new_stats_value = 0;
- u32 stats_reg_offset = stats_table->stats_offset_64[index];
-
- new_stats_value = readq(stats->hw->hw_addr + stats_reg_offset);
- /*roll-over case */
- if (new_stats_value < last_rd_stats->stats_value_64[index])
- hw_stats->stats_value_64[index] += new_stats_value;
- else
- hw_stats->stats_value_64[index] +=
- new_stats_value - last_rd_stats->stats_value_64[index];
- last_rd_stats->stats_value_64[index] = new_stats_value;
- *value = hw_stats->stats_value_64[index];
-}
-
-/**
- * i40iw_hw_stats_read_all - read all HW stat counters
- * @stats: pestat struct
- * @stats_values: hw stats structure
- *
- * Read all the HW stat counters and populates hw_stats structure
- * of passed-in vsi's pestat as well as copy created in stat_values.
- */
-void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats,
- struct i40iw_dev_hw_stats *stats_values)
-{
- u32 stats_index;
- unsigned long flags;
-
- spin_lock_irqsave(&stats->lock, flags);
-
- for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
- stats_index++)
- i40iw_hw_stats_read_32(stats, stats_index,
- &stats_values->stats_value_32[stats_index]);
- for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
- stats_index++)
- i40iw_hw_stats_read_64(stats, stats_index,
- &stats_values->stats_value_64[stats_index]);
- spin_unlock_irqrestore(&stats->lock, flags);
-}
-
-/**
- * i40iw_hw_stats_refresh_all - Update all HW stats structs
- * @stats: pestat struct
- *
- * Read all the HW stats counters to refresh values in hw_stats structure
- * of passed-in dev's pestat
- */
-void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats)
-{
- u64 stats_value;
- u32 stats_index;
- unsigned long flags;
-
- spin_lock_irqsave(&stats->lock, flags);
-
- for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
- stats_index++)
- i40iw_hw_stats_read_32(stats, stats_index, &stats_value);
- for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
- stats_index++)
- i40iw_hw_stats_read_64(stats, stats_index, &stats_value);
- spin_unlock_irqrestore(&stats->lock, flags);
-}
-
-/**
- * i40iw_get_fcn_id - Return the function id
- * @dev: pointer to the device
- */
-static u8 i40iw_get_fcn_id(struct i40iw_sc_dev *dev)
-{
- u8 fcn_id = I40IW_INVALID_FCN_ID;
- u8 i;
-
- for (i = I40IW_FIRST_NON_PF_STAT; i < I40IW_MAX_STATS_COUNT; i++)
- if (!dev->fcn_id_array[i]) {
- fcn_id = i;
- dev->fcn_id_array[i] = true;
- break;
- }
- return fcn_id;
-}
-
-/**
- * i40iw_vsi_stats_init - Initialize the vsi statistics
- * @vsi: pointer to the vsi structure
- * @info: The info structure used for initialization
- */
-enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info)
-{
- u8 fcn_id = info->fcn_id;
-
- if (info->alloc_fcn_id)
- fcn_id = i40iw_get_fcn_id(vsi->dev);
-
- if (fcn_id == I40IW_INVALID_FCN_ID)
- return I40IW_ERR_NOT_READY;
-
- vsi->pestat = info->pestat;
- vsi->pestat->hw = vsi->dev->hw;
- vsi->pestat->vsi = vsi;
-
- if (info->stats_initialize) {
- i40iw_hw_stats_init(vsi->pestat, fcn_id, true);
- spin_lock_init(&vsi->pestat->lock);
- i40iw_hw_stats_start_timer(vsi);
- }
- vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
- vsi->fcn_id = fcn_id;
- return I40IW_SUCCESS;
-}
-
-/**
- * i40iw_vsi_stats_free - Free the vsi stats
- * @vsi: pointer to the vsi structure
- */
-void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi)
-{
- u8 fcn_id = vsi->fcn_id;
-
- if (vsi->stats_fcn_id_alloc && fcn_id < I40IW_MAX_STATS_COUNT)
- vsi->dev->fcn_id_array[fcn_id] = false;
- i40iw_hw_stats_stop_timer(vsi);
-}
-
-static struct i40iw_cqp_ops iw_cqp_ops = {
- .cqp_init = i40iw_sc_cqp_init,
- .cqp_create = i40iw_sc_cqp_create,
- .cqp_post_sq = i40iw_sc_cqp_post_sq,
- .cqp_get_next_send_wqe = i40iw_sc_cqp_get_next_send_wqe,
- .cqp_destroy = i40iw_sc_cqp_destroy,
- .poll_for_cqp_op_done = i40iw_sc_poll_for_cqp_op_done
-};
-
-static struct i40iw_ccq_ops iw_ccq_ops = {
- .ccq_init = i40iw_sc_ccq_init,
- .ccq_create = i40iw_sc_ccq_create,
- .ccq_destroy = i40iw_sc_ccq_destroy,
- .ccq_create_done = i40iw_sc_ccq_create_done,
- .ccq_get_cqe_info = i40iw_sc_ccq_get_cqe_info,
- .ccq_arm = i40iw_sc_ccq_arm
-};
-
-static struct i40iw_ceq_ops iw_ceq_ops = {
- .ceq_init = i40iw_sc_ceq_init,
- .ceq_create = i40iw_sc_ceq_create,
- .cceq_create_done = i40iw_sc_cceq_create_done,
- .cceq_destroy_done = i40iw_sc_cceq_destroy_done,
- .cceq_create = i40iw_sc_cceq_create,
- .ceq_destroy = i40iw_sc_ceq_destroy,
- .process_ceq = i40iw_sc_process_ceq
-};
-
-static struct i40iw_aeq_ops iw_aeq_ops = {
- .aeq_init = i40iw_sc_aeq_init,
- .aeq_create = i40iw_sc_aeq_create,
- .aeq_destroy = i40iw_sc_aeq_destroy,
- .get_next_aeqe = i40iw_sc_get_next_aeqe,
- .repost_aeq_entries = i40iw_sc_repost_aeq_entries,
- .aeq_create_done = i40iw_sc_aeq_create_done,
- .aeq_destroy_done = i40iw_sc_aeq_destroy_done
-};
-
-/* iwarp pd ops */
-static struct i40iw_pd_ops iw_pd_ops = {
- .pd_init = i40iw_sc_pd_init,
-};
-
-static struct i40iw_priv_qp_ops iw_priv_qp_ops = {
- .qp_init = i40iw_sc_qp_init,
- .qp_create = i40iw_sc_qp_create,
- .qp_modify = i40iw_sc_qp_modify,
- .qp_destroy = i40iw_sc_qp_destroy,
- .qp_flush_wqes = i40iw_sc_qp_flush_wqes,
- .qp_upload_context = i40iw_sc_qp_upload_context,
- .qp_setctx = i40iw_sc_qp_setctx,
- .qp_send_lsmm = i40iw_sc_send_lsmm,
- .qp_send_lsmm_nostag = i40iw_sc_send_lsmm_nostag,
- .qp_send_rtt = i40iw_sc_send_rtt,
- .qp_post_wqe0 = i40iw_sc_post_wqe0,
- .iw_mr_fast_register = i40iw_sc_mr_fast_register
-};
-
-static struct i40iw_priv_cq_ops iw_priv_cq_ops = {
- .cq_init = i40iw_sc_cq_init,
- .cq_create = i40iw_sc_cq_create,
- .cq_destroy = i40iw_sc_cq_destroy,
- .cq_modify = i40iw_sc_cq_modify,
-};
-
-static struct i40iw_mr_ops iw_mr_ops = {
- .alloc_stag = i40iw_sc_alloc_stag,
- .mr_reg_non_shared = i40iw_sc_mr_reg_non_shared,
- .mr_reg_shared = i40iw_sc_mr_reg_shared,
- .dealloc_stag = i40iw_sc_dealloc_stag,
- .query_stag = i40iw_sc_query_stag,
- .mw_alloc = i40iw_sc_mw_alloc
-};
-
-static struct i40iw_cqp_misc_ops iw_cqp_misc_ops = {
- .manage_push_page = i40iw_sc_manage_push_page,
- .manage_hmc_pm_func_table = i40iw_sc_manage_hmc_pm_func_table,
- .set_hmc_resource_profile = i40iw_sc_set_hmc_resource_profile,
- .commit_fpm_values = i40iw_sc_commit_fpm_values,
- .query_fpm_values = i40iw_sc_query_fpm_values,
- .static_hmc_pages_allocated = i40iw_sc_static_hmc_pages_allocated,
- .add_arp_cache_entry = i40iw_sc_add_arp_cache_entry,
- .del_arp_cache_entry = i40iw_sc_del_arp_cache_entry,
- .query_arp_cache_entry = i40iw_sc_query_arp_cache_entry,
- .manage_apbvt_entry = i40iw_sc_manage_apbvt_entry,
- .manage_qhash_table_entry = i40iw_sc_manage_qhash_table_entry,
- .alloc_local_mac_ipaddr_table_entry = i40iw_sc_alloc_local_mac_ipaddr_entry,
- .add_local_mac_ipaddr_entry = i40iw_sc_add_local_mac_ipaddr_entry,
- .del_local_mac_ipaddr_entry = i40iw_sc_del_local_mac_ipaddr_entry,
- .cqp_nop = i40iw_sc_cqp_nop,
- .commit_fpm_values_done = i40iw_sc_commit_fpm_values_done,
- .query_fpm_values_done = i40iw_sc_query_fpm_values_done,
- .manage_hmc_pm_func_table_done = i40iw_sc_manage_hmc_pm_func_table_done,
- .update_suspend_qp = i40iw_sc_suspend_qp,
- .update_resume_qp = i40iw_sc_resume_qp
-};
-
-static struct i40iw_hmc_ops iw_hmc_ops = {
- .init_iw_hmc = i40iw_sc_init_iw_hmc,
- .parse_fpm_query_buf = i40iw_sc_parse_fpm_query_buf,
- .configure_iw_fpm = i40iw_sc_configure_iw_fpm,
- .parse_fpm_commit_buf = i40iw_sc_parse_fpm_commit_buf,
- .create_hmc_object = i40iw_sc_create_hmc_obj,
- .del_hmc_object = i40iw_sc_del_hmc_obj
-};
-
-/**
- * i40iw_device_init - Initialize IWARP device
- * @dev: IWARP device pointer
- * @info: IWARP init info
- */
-enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,
- struct i40iw_device_init_info *info)
-{
- u32 val;
- u32 vchnl_ver = 0;
- u16 hmc_fcn = 0;
- enum i40iw_status_code ret_code = 0;
- u8 db_size;
-
- spin_lock_init(&dev->cqp_lock);
-
- i40iw_device_init_uk(&dev->dev_uk);
-
- dev->debug_mask = info->debug_mask;
-
- dev->hmc_fn_id = info->hmc_fn_id;
- dev->is_pf = info->is_pf;
-
- dev->fpm_query_buf_pa = info->fpm_query_buf_pa;
- dev->fpm_query_buf = info->fpm_query_buf;
-
- dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;
- dev->fpm_commit_buf = info->fpm_commit_buf;
-
- dev->hw = info->hw;
- dev->hw->hw_addr = info->bar0;
-
- if (dev->is_pf) {
- val = i40iw_rd32(dev->hw, I40E_GLPCI_DREVID);
- dev->hw_rev = (u8)RS_32(val, I40E_GLPCI_DREVID_DEFAULT_REVID);
-
- val = i40iw_rd32(dev->hw, I40E_GLPCI_LBARCTRL);
- db_size = (u8)RS_32(val, I40E_GLPCI_LBARCTRL_PE_DB_SIZE);
- if ((db_size != I40IW_PE_DB_SIZE_4M) &&
- (db_size != I40IW_PE_DB_SIZE_8M)) {
- i40iw_debug(dev, I40IW_DEBUG_DEV,
- "%s: PE doorbell is not enabled in CSR val 0x%x\n",
- __func__, val);
- ret_code = I40IW_ERR_PE_DOORBELL_NOT_ENABLED;
- return ret_code;
- }
- dev->db_addr = dev->hw->hw_addr + I40IW_DB_ADDR_OFFSET;
- dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_pf;
- } else {
- dev->db_addr = dev->hw->hw_addr + I40IW_VF_DB_ADDR_OFFSET;
- }
-
- dev->cqp_ops = &iw_cqp_ops;
- dev->ccq_ops = &iw_ccq_ops;
- dev->ceq_ops = &iw_ceq_ops;
- dev->aeq_ops = &iw_aeq_ops;
- dev->cqp_misc_ops = &iw_cqp_misc_ops;
- dev->iw_pd_ops = &iw_pd_ops;
- dev->iw_priv_qp_ops = &iw_priv_qp_ops;
- dev->iw_priv_cq_ops = &iw_priv_cq_ops;
- dev->mr_ops = &iw_mr_ops;
- dev->hmc_ops = &iw_hmc_ops;
- dev->vchnl_if.vchnl_send = info->vchnl_send;
- if (dev->vchnl_if.vchnl_send)
- dev->vchnl_up = true;
- else
- dev->vchnl_up = false;
- if (!dev->is_pf) {
- dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_vf;
- ret_code = i40iw_vchnl_vf_get_ver(dev, &vchnl_ver);
- if (!ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_DEV,
- "%s: Get Channel version rc = 0x%0x, version is %u\n",
- __func__, ret_code, vchnl_ver);
- ret_code = i40iw_vchnl_vf_get_hmc_fcn(dev, &hmc_fcn);
- if (!ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_DEV,
- "%s Get HMC function rc = 0x%0x, hmc fcn is %u\n",
- __func__, ret_code, hmc_fcn);
- dev->hmc_fn_id = (u8)hmc_fcn;
- }
- }
- }
- dev->iw_vf_cqp_ops = &iw_vf_cqp_ops;
-
- return ret_code;
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_d.h b/drivers/infiniband/hw/i40iw/i40iw_d.h
deleted file mode 100644
index e8367d67575d..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_d.h
+++ /dev/null
@@ -1,1757 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_D_H
-#define I40IW_D_H
-
-#define I40IW_FIRST_USER_QP_ID 2
-
-#define I40IW_DB_ADDR_OFFSET (4 * 1024 * 1024 - 64 * 1024)
-#define I40IW_VF_DB_ADDR_OFFSET (64 * 1024)
-
-#define I40IW_PUSH_OFFSET (4 * 1024 * 1024)
-#define I40IW_PF_FIRST_PUSH_PAGE_INDEX 16
-#define I40IW_VF_PUSH_OFFSET ((8 + 64) * 1024)
-#define I40IW_VF_FIRST_PUSH_PAGE_INDEX 2
-
-#define I40IW_PE_DB_SIZE_4M 1
-#define I40IW_PE_DB_SIZE_8M 2
-
-#define I40IW_DDP_VER 1
-#define I40IW_RDMAP_VER 1
-
-#define I40IW_RDMA_MODE_RDMAC 0
-#define I40IW_RDMA_MODE_IETF 1
-
-#define I40IW_QP_STATE_INVALID 0
-#define I40IW_QP_STATE_IDLE 1
-#define I40IW_QP_STATE_RTS 2
-#define I40IW_QP_STATE_CLOSING 3
-#define I40IW_QP_STATE_RESERVED 4
-#define I40IW_QP_STATE_TERMINATE 5
-#define I40IW_QP_STATE_ERROR 6
-
-#define I40IW_STAG_STATE_INVALID 0
-#define I40IW_STAG_STATE_VALID 1
-
-#define I40IW_STAG_TYPE_SHARED 0
-#define I40IW_STAG_TYPE_NONSHARED 1
-
-#define I40IW_MAX_USER_PRIORITY 8
-#define I40IW_MAX_STATS_COUNT 16
-#define I40IW_FIRST_NON_PF_STAT 4
-
-
-#define I40IW_MTU_TO_MSS_IPV4 40
-#define I40IW_MTU_TO_MSS_IPV6 60
-#define I40IW_DEFAULT_MTU 1500
-
-#define LS_64_1(val, bits) ((u64)(uintptr_t)val << bits)
-#define RS_64_1(val, bits) ((u64)(uintptr_t)val >> bits)
-#define LS_32_1(val, bits) (u32)(val << bits)
-#define RS_32_1(val, bits) (u32)(val >> bits)
-#define I40E_HI_DWORD(x) ((u32)((((x) >> 16) >> 16) & 0xFFFFFFFF))
-
-#define QS_HANDLE_UNKNOWN 0xffff
-
-#define LS_64(val, field) (((u64)val << field ## _SHIFT) & (field ## _MASK))
-
-#define RS_64(val, field) ((u64)(val & field ## _MASK) >> field ## _SHIFT)
-#define LS_32(val, field) ((val << field ## _SHIFT) & (field ## _MASK))
-#define RS_32(val, field) ((val & field ## _MASK) >> field ## _SHIFT)
-
-#define TERM_DDP_LEN_TAGGED 14
-#define TERM_DDP_LEN_UNTAGGED 18
-#define TERM_RDMA_LEN 28
-#define RDMA_OPCODE_MASK 0x0f
-#define RDMA_READ_REQ_OPCODE 1
-#define Q2_BAD_FRAME_OFFSET 72
-#define Q2_FPSN_OFFSET 64
-#define CQE_MAJOR_DRV 0x8000
-
-#define I40IW_TERM_SENT 0x01
-#define I40IW_TERM_RCVD 0x02
-#define I40IW_TERM_DONE 0x04
-#define I40IW_MAC_HLEN 14
-
-#define I40IW_INVALID_WQE_INDEX 0xffffffff
-
-#define I40IW_CQP_WAIT_POLL_REGS 1
-#define I40IW_CQP_WAIT_POLL_CQ 2
-#define I40IW_CQP_WAIT_EVENT 3
-
-#define I40IW_CQP_INIT_WQE(wqe) memset(wqe, 0, 64)
-
-#define I40IW_GET_CURRENT_CQ_ELEMENT(_cq) \
- ( \
- &((_cq)->cq_base[I40IW_RING_GETCURRENT_HEAD((_cq)->cq_ring)]) \
- )
-#define I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(_cq) \
- ( \
- &(((struct i40iw_extended_cqe *) \
- ((_cq)->cq_base))[I40IW_RING_GETCURRENT_HEAD((_cq)->cq_ring)]) \
- )
-
-#define I40IW_GET_CURRENT_AEQ_ELEMENT(_aeq) \
- ( \
- &_aeq->aeqe_base[I40IW_RING_GETCURRENT_TAIL(_aeq->aeq_ring)] \
- )
-
-#define I40IW_GET_CURRENT_CEQ_ELEMENT(_ceq) \
- ( \
- &_ceq->ceqe_base[I40IW_RING_GETCURRENT_TAIL(_ceq->ceq_ring)] \
- )
-
-#define I40IW_AE_SOURCE_RSVD 0x0
-#define I40IW_AE_SOURCE_RQ 0x1
-#define I40IW_AE_SOURCE_RQ_0011 0x3
-
-#define I40IW_AE_SOURCE_CQ 0x2
-#define I40IW_AE_SOURCE_CQ_0110 0x6
-#define I40IW_AE_SOURCE_CQ_1010 0xA
-#define I40IW_AE_SOURCE_CQ_1110 0xE
-
-#define I40IW_AE_SOURCE_SQ 0x5
-#define I40IW_AE_SOURCE_SQ_0111 0x7
-
-#define I40IW_AE_SOURCE_IN_RR_WR 0x9
-#define I40IW_AE_SOURCE_IN_RR_WR_1011 0xB
-#define I40IW_AE_SOURCE_OUT_RR 0xD
-#define I40IW_AE_SOURCE_OUT_RR_1111 0xF
-
-#define I40IW_TCP_STATE_NON_EXISTENT 0
-#define I40IW_TCP_STATE_CLOSED 1
-#define I40IW_TCP_STATE_LISTEN 2
-#define I40IW_STATE_SYN_SEND 3
-#define I40IW_TCP_STATE_SYN_RECEIVED 4
-#define I40IW_TCP_STATE_ESTABLISHED 5
-#define I40IW_TCP_STATE_CLOSE_WAIT 6
-#define I40IW_TCP_STATE_FIN_WAIT_1 7
-#define I40IW_TCP_STATE_CLOSING 8
-#define I40IW_TCP_STATE_LAST_ACK 9
-#define I40IW_TCP_STATE_FIN_WAIT_2 10
-#define I40IW_TCP_STATE_TIME_WAIT 11
-#define I40IW_TCP_STATE_RESERVED_1 12
-#define I40IW_TCP_STATE_RESERVED_2 13
-#define I40IW_TCP_STATE_RESERVED_3 14
-#define I40IW_TCP_STATE_RESERVED_4 15
-
-/* ILQ CQP hash table fields */
-#define I40IW_CQPSQ_QHASH_VLANID_SHIFT 32
-#define I40IW_CQPSQ_QHASH_VLANID_MASK \
- ((u64)0xfff << I40IW_CQPSQ_QHASH_VLANID_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_QPN_SHIFT 32
-#define I40IW_CQPSQ_QHASH_QPN_MASK \
- ((u64)0x3ffff << I40IW_CQPSQ_QHASH_QPN_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_QS_HANDLE_SHIFT 0
-#define I40IW_CQPSQ_QHASH_QS_HANDLE_MASK ((u64)0x3ff << I40IW_CQPSQ_QHASH_QS_HANDLE_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_SRC_PORT_SHIFT 16
-#define I40IW_CQPSQ_QHASH_SRC_PORT_MASK \
- ((u64)0xffff << I40IW_CQPSQ_QHASH_SRC_PORT_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_DEST_PORT_SHIFT 0
-#define I40IW_CQPSQ_QHASH_DEST_PORT_MASK \
- ((u64)0xffff << I40IW_CQPSQ_QHASH_DEST_PORT_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_ADDR0_SHIFT 32
-#define I40IW_CQPSQ_QHASH_ADDR0_MASK \
- ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR0_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_ADDR1_SHIFT 0
-#define I40IW_CQPSQ_QHASH_ADDR1_MASK \
- ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR1_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_ADDR2_SHIFT 32
-#define I40IW_CQPSQ_QHASH_ADDR2_MASK \
- ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR2_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_ADDR3_SHIFT 0
-#define I40IW_CQPSQ_QHASH_ADDR3_MASK \
- ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR3_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_WQEVALID_SHIFT 63
-#define I40IW_CQPSQ_QHASH_WQEVALID_MASK \
- ((u64)0x1 << I40IW_CQPSQ_QHASH_WQEVALID_SHIFT)
-#define I40IW_CQPSQ_QHASH_OPCODE_SHIFT 32
-#define I40IW_CQPSQ_QHASH_OPCODE_MASK \
- ((u64)0x3f << I40IW_CQPSQ_QHASH_OPCODE_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_MANAGE_SHIFT 61
-#define I40IW_CQPSQ_QHASH_MANAGE_MASK \
- ((u64)0x3 << I40IW_CQPSQ_QHASH_MANAGE_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_IPV4VALID_SHIFT 60
-#define I40IW_CQPSQ_QHASH_IPV4VALID_MASK \
- ((u64)0x1 << I40IW_CQPSQ_QHASH_IPV4VALID_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_VLANVALID_SHIFT 59
-#define I40IW_CQPSQ_QHASH_VLANVALID_MASK \
- ((u64)0x1 << I40IW_CQPSQ_QHASH_VLANVALID_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_ENTRYTYPE_SHIFT 42
-#define I40IW_CQPSQ_QHASH_ENTRYTYPE_MASK \
- ((u64)0x7 << I40IW_CQPSQ_QHASH_ENTRYTYPE_SHIFT)
-/* CQP Host Context */
-#define I40IW_CQPHC_EN_DC_TCP_SHIFT 0
-#define I40IW_CQPHC_EN_DC_TCP_MASK (1UL << I40IW_CQPHC_EN_DC_TCP_SHIFT)
-
-#define I40IW_CQPHC_SQSIZE_SHIFT 8
-#define I40IW_CQPHC_SQSIZE_MASK (0xfUL << I40IW_CQPHC_SQSIZE_SHIFT)
-
-#define I40IW_CQPHC_DISABLE_PFPDUS_SHIFT 1
-#define I40IW_CQPHC_DISABLE_PFPDUS_MASK (0x1UL << I40IW_CQPHC_DISABLE_PFPDUS_SHIFT)
-
-#define I40IW_CQPHC_ENABLED_VFS_SHIFT 32
-#define I40IW_CQPHC_ENABLED_VFS_MASK (0x3fULL << I40IW_CQPHC_ENABLED_VFS_SHIFT)
-
-#define I40IW_CQPHC_HMC_PROFILE_SHIFT 0
-#define I40IW_CQPHC_HMC_PROFILE_MASK (0x7ULL << I40IW_CQPHC_HMC_PROFILE_SHIFT)
-
-#define I40IW_CQPHC_SVER_SHIFT 24
-#define I40IW_CQPHC_SVER_MASK (0xffUL << I40IW_CQPHC_SVER_SHIFT)
-
-#define I40IW_CQPHC_SQBASE_SHIFT 9
-#define I40IW_CQPHC_SQBASE_MASK \
- (0xfffffffffffffeULL << I40IW_CQPHC_SQBASE_SHIFT)
-
-#define I40IW_CQPHC_QPCTX_SHIFT 0
-#define I40IW_CQPHC_QPCTX_MASK \
- (0xffffffffffffffffULL << I40IW_CQPHC_QPCTX_SHIFT)
-#define I40IW_CQPHC_SVER 1
-
-#define I40IW_CQP_SW_SQSIZE_4 4
-#define I40IW_CQP_SW_SQSIZE_2048 2048
-
-/* iWARP QP Doorbell shadow area */
-#define I40IW_QP_DBSA_HW_SQ_TAIL_SHIFT 0
-#define I40IW_QP_DBSA_HW_SQ_TAIL_MASK \
- (0x3fffUL << I40IW_QP_DBSA_HW_SQ_TAIL_SHIFT)
-
-/* Completion Queue Doorbell shadow area */
-#define I40IW_CQ_DBSA_CQEIDX_SHIFT 0
-#define I40IW_CQ_DBSA_CQEIDX_MASK (0xfffffUL << I40IW_CQ_DBSA_CQEIDX_SHIFT)
-
-#define I40IW_CQ_DBSA_SW_CQ_SELECT_SHIFT 0
-#define I40IW_CQ_DBSA_SW_CQ_SELECT_MASK \
- (0x3fffUL << I40IW_CQ_DBSA_SW_CQ_SELECT_SHIFT)
-
-#define I40IW_CQ_DBSA_ARM_NEXT_SHIFT 14
-#define I40IW_CQ_DBSA_ARM_NEXT_MASK (1UL << I40IW_CQ_DBSA_ARM_NEXT_SHIFT)
-
-#define I40IW_CQ_DBSA_ARM_NEXT_SE_SHIFT 15
-#define I40IW_CQ_DBSA_ARM_NEXT_SE_MASK (1UL << I40IW_CQ_DBSA_ARM_NEXT_SE_SHIFT)
-
-#define I40IW_CQ_DBSA_ARM_SEQ_NUM_SHIFT 16
-#define I40IW_CQ_DBSA_ARM_SEQ_NUM_MASK \
- (0x3UL << I40IW_CQ_DBSA_ARM_SEQ_NUM_SHIFT)
-
-/* CQP and iWARP Completion Queue */
-#define I40IW_CQ_QPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQ_QPCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CCQ_OPRETVAL_SHIFT 0
-#define I40IW_CCQ_OPRETVAL_MASK (0xffffffffUL << I40IW_CCQ_OPRETVAL_SHIFT)
-
-#define I40IW_CQ_MINERR_SHIFT 0
-#define I40IW_CQ_MINERR_MASK (0xffffUL << I40IW_CQ_MINERR_SHIFT)
-
-#define I40IW_CQ_MAJERR_SHIFT 16
-#define I40IW_CQ_MAJERR_MASK (0xffffUL << I40IW_CQ_MAJERR_SHIFT)
-
-#define I40IW_CQ_WQEIDX_SHIFT 32
-#define I40IW_CQ_WQEIDX_MASK (0x3fffULL << I40IW_CQ_WQEIDX_SHIFT)
-
-#define I40IW_CQ_ERROR_SHIFT 55
-#define I40IW_CQ_ERROR_MASK (1ULL << I40IW_CQ_ERROR_SHIFT)
-
-#define I40IW_CQ_SQ_SHIFT 62
-#define I40IW_CQ_SQ_MASK (1ULL << I40IW_CQ_SQ_SHIFT)
-
-#define I40IW_CQ_VALID_SHIFT 63
-#define I40IW_CQ_VALID_MASK (1ULL << I40IW_CQ_VALID_SHIFT)
-
-#define I40IWCQ_PAYLDLEN_SHIFT 0
-#define I40IWCQ_PAYLDLEN_MASK (0xffffffffUL << I40IWCQ_PAYLDLEN_SHIFT)
-
-#define I40IWCQ_TCPSEQNUM_SHIFT 32
-#define I40IWCQ_TCPSEQNUM_MASK (0xffffffffULL << I40IWCQ_TCPSEQNUM_SHIFT)
-
-#define I40IWCQ_INVSTAG_SHIFT 0
-#define I40IWCQ_INVSTAG_MASK (0xffffffffUL << I40IWCQ_INVSTAG_SHIFT)
-
-#define I40IWCQ_QPID_SHIFT 32
-#define I40IWCQ_QPID_MASK (0x3ffffULL << I40IWCQ_QPID_SHIFT)
-
-#define I40IWCQ_PSHDROP_SHIFT 51
-#define I40IWCQ_PSHDROP_MASK (1ULL << I40IWCQ_PSHDROP_SHIFT)
-
-#define I40IWCQ_SRQ_SHIFT 52
-#define I40IWCQ_SRQ_MASK (1ULL << I40IWCQ_SRQ_SHIFT)
-
-#define I40IWCQ_STAG_SHIFT 53
-#define I40IWCQ_STAG_MASK (1ULL << I40IWCQ_STAG_SHIFT)
-
-#define I40IWCQ_SOEVENT_SHIFT 54
-#define I40IWCQ_SOEVENT_MASK (1ULL << I40IWCQ_SOEVENT_SHIFT)
-
-#define I40IWCQ_OP_SHIFT 56
-#define I40IWCQ_OP_MASK (0x3fULL << I40IWCQ_OP_SHIFT)
-
-/* CEQE format */
-#define I40IW_CEQE_CQCTX_SHIFT 0
-#define I40IW_CEQE_CQCTX_MASK \
- (0x7fffffffffffffffULL << I40IW_CEQE_CQCTX_SHIFT)
-
-#define I40IW_CEQE_VALID_SHIFT 63
-#define I40IW_CEQE_VALID_MASK (1ULL << I40IW_CEQE_VALID_SHIFT)
-
-/* AEQE format */
-#define I40IW_AEQE_COMPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_AEQE_COMPCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_AEQE_QPCQID_SHIFT 0
-#define I40IW_AEQE_QPCQID_MASK (0x3ffffUL << I40IW_AEQE_QPCQID_SHIFT)
-
-#define I40IW_AEQE_WQDESCIDX_SHIFT 18
-#define I40IW_AEQE_WQDESCIDX_MASK (0x3fffULL << I40IW_AEQE_WQDESCIDX_SHIFT)
-
-#define I40IW_AEQE_OVERFLOW_SHIFT 33
-#define I40IW_AEQE_OVERFLOW_MASK (1ULL << I40IW_AEQE_OVERFLOW_SHIFT)
-
-#define I40IW_AEQE_AECODE_SHIFT 34
-#define I40IW_AEQE_AECODE_MASK (0xffffULL << I40IW_AEQE_AECODE_SHIFT)
-
-#define I40IW_AEQE_AESRC_SHIFT 50
-#define I40IW_AEQE_AESRC_MASK (0xfULL << I40IW_AEQE_AESRC_SHIFT)
-
-#define I40IW_AEQE_IWSTATE_SHIFT 54
-#define I40IW_AEQE_IWSTATE_MASK (0x7ULL << I40IW_AEQE_IWSTATE_SHIFT)
-
-#define I40IW_AEQE_TCPSTATE_SHIFT 57
-#define I40IW_AEQE_TCPSTATE_MASK (0xfULL << I40IW_AEQE_TCPSTATE_SHIFT)
-
-#define I40IW_AEQE_Q2DATA_SHIFT 61
-#define I40IW_AEQE_Q2DATA_MASK (0x3ULL << I40IW_AEQE_Q2DATA_SHIFT)
-
-#define I40IW_AEQE_VALID_SHIFT 63
-#define I40IW_AEQE_VALID_MASK (1ULL << I40IW_AEQE_VALID_SHIFT)
-
-/* CQP SQ WQES */
-#define I40IW_QP_TYPE_IWARP 1
-#define I40IW_QP_TYPE_UDA 2
-#define I40IW_QP_TYPE_CQP 4
-
-#define I40IW_CQ_TYPE_IWARP 1
-#define I40IW_CQ_TYPE_ILQ 2
-#define I40IW_CQ_TYPE_IEQ 3
-#define I40IW_CQ_TYPE_CQP 4
-
-#define I40IWQP_TERM_SEND_TERM_AND_FIN 0
-#define I40IWQP_TERM_SEND_TERM_ONLY 1
-#define I40IWQP_TERM_SEND_FIN_ONLY 2
-#define I40IWQP_TERM_DONOT_SEND_TERM_OR_FIN 3
-
-#define I40IW_CQP_OP_CREATE_QP 0
-#define I40IW_CQP_OP_MODIFY_QP 0x1
-#define I40IW_CQP_OP_DESTROY_QP 0x02
-#define I40IW_CQP_OP_CREATE_CQ 0x03
-#define I40IW_CQP_OP_MODIFY_CQ 0x04
-#define I40IW_CQP_OP_DESTROY_CQ 0x05
-#define I40IW_CQP_OP_CREATE_SRQ 0x06
-#define I40IW_CQP_OP_MODIFY_SRQ 0x07
-#define I40IW_CQP_OP_DESTROY_SRQ 0x08
-#define I40IW_CQP_OP_ALLOC_STAG 0x09
-#define I40IW_CQP_OP_REG_MR 0x0a
-#define I40IW_CQP_OP_QUERY_STAG 0x0b
-#define I40IW_CQP_OP_REG_SMR 0x0c
-#define I40IW_CQP_OP_DEALLOC_STAG 0x0d
-#define I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE 0x0e
-#define I40IW_CQP_OP_MANAGE_ARP 0x0f
-#define I40IW_CQP_OP_MANAGE_VF_PBLE_BP 0x10
-#define I40IW_CQP_OP_MANAGE_PUSH_PAGES 0x11
-#define I40IW_CQP_OP_QUERY_RDMA_FEATURES 0x12
-#define I40IW_CQP_OP_UPLOAD_CONTEXT 0x13
-#define I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY 0x14
-#define I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE 0x15
-#define I40IW_CQP_OP_CREATE_CEQ 0x16
-#define I40IW_CQP_OP_DESTROY_CEQ 0x18
-#define I40IW_CQP_OP_CREATE_AEQ 0x19
-#define I40IW_CQP_OP_DESTROY_AEQ 0x1b
-#define I40IW_CQP_OP_CREATE_ADDR_VECT 0x1c
-#define I40IW_CQP_OP_MODIFY_ADDR_VECT 0x1d
-#define I40IW_CQP_OP_DESTROY_ADDR_VECT 0x1e
-#define I40IW_CQP_OP_UPDATE_PE_SDS 0x1f
-#define I40IW_CQP_OP_QUERY_FPM_VALUES 0x20
-#define I40IW_CQP_OP_COMMIT_FPM_VALUES 0x21
-#define I40IW_CQP_OP_FLUSH_WQES 0x22
-/* I40IW_CQP_OP_GEN_AE is the same value as I40IW_CQP_OP_FLUSH_WQES */
-#define I40IW_CQP_OP_GEN_AE 0x22
-#define I40IW_CQP_OP_MANAGE_APBVT 0x23
-#define I40IW_CQP_OP_NOP 0x24
-#define I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY 0x25
-#define I40IW_CQP_OP_CREATE_UDA_MCAST_GROUP 0x26
-#define I40IW_CQP_OP_MODIFY_UDA_MCAST_GROUP 0x27
-#define I40IW_CQP_OP_DESTROY_UDA_MCAST_GROUP 0x28
-#define I40IW_CQP_OP_SUSPEND_QP 0x29
-#define I40IW_CQP_OP_RESUME_QP 0x2a
-#define I40IW_CQP_OP_SHMC_PAGES_ALLOCATED 0x2b
-#define I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE 0x2d
-
-#define I40IW_FEATURE_BUF_SIZE (8 * I40IW_MAX_FEATURES)
-
-#define I40IW_FW_VER_MINOR_SHIFT 0
-#define I40IW_FW_VER_MINOR_MASK \
- (0xffffULL << I40IW_FW_VER_MINOR_SHIFT)
-
-#define I40IW_FW_VER_MAJOR_SHIFT 16
-#define I40IW_FW_VER_MAJOR_MASK \
- (0xffffULL << I40IW_FW_VER_MAJOR_SHIFT)
-
-#define I40IW_FEATURE_INFO_SHIFT 0
-#define I40IW_FEATURE_INFO_MASK \
- (0xffffULL << I40IW_FEATURE_INFO_SHIFT)
-
-#define I40IW_FEATURE_CNT_SHIFT 32
-#define I40IW_FEATURE_CNT_MASK \
- (0xffffULL << I40IW_FEATURE_CNT_SHIFT)
-
-#define I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT 16
-#define I40IW_UDA_QPSQ_NEXT_HEADER_MASK ((u64)0xff << I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT)
-
-#define I40IW_UDA_QPSQ_OPCODE_SHIFT 32
-#define I40IW_UDA_QPSQ_OPCODE_MASK ((u64)0x3f << I40IW_UDA_QPSQ_OPCODE_SHIFT)
-
-#define I40IW_UDA_QPSQ_MACLEN_SHIFT 56
-#define I40IW_UDA_QPSQ_MACLEN_MASK \
- ((u64)0x7f << I40IW_UDA_QPSQ_MACLEN_SHIFT)
-
-#define I40IW_UDA_QPSQ_IPLEN_SHIFT 48
-#define I40IW_UDA_QPSQ_IPLEN_MASK \
- ((u64)0x7f << I40IW_UDA_QPSQ_IPLEN_SHIFT)
-
-#define I40IW_UDA_QPSQ_L4T_SHIFT 30
-#define I40IW_UDA_QPSQ_L4T_MASK \
- ((u64)0x3 << I40IW_UDA_QPSQ_L4T_SHIFT)
-
-#define I40IW_UDA_QPSQ_IIPT_SHIFT 28
-#define I40IW_UDA_QPSQ_IIPT_MASK \
- ((u64)0x3 << I40IW_UDA_QPSQ_IIPT_SHIFT)
-
-#define I40IW_UDA_QPSQ_L4LEN_SHIFT 24
-#define I40IW_UDA_QPSQ_L4LEN_MASK ((u64)0xf << I40IW_UDA_QPSQ_L4LEN_SHIFT)
-
-#define I40IW_UDA_QPSQ_AVIDX_SHIFT 0
-#define I40IW_UDA_QPSQ_AVIDX_MASK ((u64)0xffff << I40IW_UDA_QPSQ_AVIDX_SHIFT)
-
-#define I40IW_UDA_QPSQ_VALID_SHIFT 63
-#define I40IW_UDA_QPSQ_VALID_MASK \
- ((u64)0x1 << I40IW_UDA_QPSQ_VALID_SHIFT)
-
-#define I40IW_UDA_QPSQ_SIGCOMPL_SHIFT 62
-#define I40IW_UDA_QPSQ_SIGCOMPL_MASK ((u64)0x1 << I40IW_UDA_QPSQ_SIGCOMPL_SHIFT)
-
-#define I40IW_UDA_PAYLOADLEN_SHIFT 0
-#define I40IW_UDA_PAYLOADLEN_MASK ((u64)0x3fff << I40IW_UDA_PAYLOADLEN_SHIFT)
-
-#define I40IW_UDA_HDRLEN_SHIFT 16
-#define I40IW_UDA_HDRLEN_MASK ((u64)0x1ff << I40IW_UDA_HDRLEN_SHIFT)
-
-#define I40IW_VLAN_TAG_VALID_SHIFT 50
-#define I40IW_VLAN_TAG_VALID_MASK ((u64)0x1 << I40IW_VLAN_TAG_VALID_SHIFT)
-
-#define I40IW_UDA_L3PROTO_SHIFT 0
-#define I40IW_UDA_L3PROTO_MASK ((u64)0x3 << I40IW_UDA_L3PROTO_SHIFT)
-
-#define I40IW_UDA_L4PROTO_SHIFT 16
-#define I40IW_UDA_L4PROTO_MASK ((u64)0x3 << I40IW_UDA_L4PROTO_SHIFT)
-
-#define I40IW_UDA_QPSQ_DOLOOPBACK_SHIFT 44
-#define I40IW_UDA_QPSQ_DOLOOPBACK_MASK \
- ((u64)0x1 << I40IW_UDA_QPSQ_DOLOOPBACK_SHIFT)
-
-/* CQP SQ WQE common fields */
-#define I40IW_CQPSQ_OPCODE_SHIFT 32
-#define I40IW_CQPSQ_OPCODE_MASK (0x3fULL << I40IW_CQPSQ_OPCODE_SHIFT)
-
-#define I40IW_CQPSQ_WQEVALID_SHIFT 63
-#define I40IW_CQPSQ_WQEVALID_MASK (1ULL << I40IW_CQPSQ_WQEVALID_SHIFT)
-
-#define I40IW_CQPSQ_TPHVAL_SHIFT 0
-#define I40IW_CQPSQ_TPHVAL_MASK (0xffUL << I40IW_CQPSQ_TPHVAL_SHIFT)
-
-#define I40IW_CQPSQ_TPHEN_SHIFT 60
-#define I40IW_CQPSQ_TPHEN_MASK (1ULL << I40IW_CQPSQ_TPHEN_SHIFT)
-
-#define I40IW_CQPSQ_PBUFADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_PBUFADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-/* Create/Modify/Destroy QP */
-
-#define I40IW_CQPSQ_QP_NEWMSS_SHIFT 32
-#define I40IW_CQPSQ_QP_NEWMSS_MASK (0x3fffULL << I40IW_CQPSQ_QP_NEWMSS_SHIFT)
-
-#define I40IW_CQPSQ_QP_TERMLEN_SHIFT 48
-#define I40IW_CQPSQ_QP_TERMLEN_MASK (0xfULL << I40IW_CQPSQ_QP_TERMLEN_SHIFT)
-
-#define I40IW_CQPSQ_QP_QPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_QP_QPCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_QP_QPID_SHIFT 0
-#define I40IW_CQPSQ_QP_QPID_MASK (0x3FFFFUL)
-/* I40IWCQ_QPID_MASK */
-
-#define I40IW_CQPSQ_QP_OP_SHIFT 32
-#define I40IW_CQPSQ_QP_OP_MASK I40IWCQ_OP_MASK
-
-#define I40IW_CQPSQ_QP_ORDVALID_SHIFT 42
-#define I40IW_CQPSQ_QP_ORDVALID_MASK (1ULL << I40IW_CQPSQ_QP_ORDVALID_SHIFT)
-
-#define I40IW_CQPSQ_QP_TOECTXVALID_SHIFT 43
-#define I40IW_CQPSQ_QP_TOECTXVALID_MASK \
- (1ULL << I40IW_CQPSQ_QP_TOECTXVALID_SHIFT)
-
-#define I40IW_CQPSQ_QP_CACHEDVARVALID_SHIFT 44
-#define I40IW_CQPSQ_QP_CACHEDVARVALID_MASK \
- (1ULL << I40IW_CQPSQ_QP_CACHEDVARVALID_SHIFT)
-
-#define I40IW_CQPSQ_QP_VQ_SHIFT 45
-#define I40IW_CQPSQ_QP_VQ_MASK (1ULL << I40IW_CQPSQ_QP_VQ_SHIFT)
-
-#define I40IW_CQPSQ_QP_FORCELOOPBACK_SHIFT 46
-#define I40IW_CQPSQ_QP_FORCELOOPBACK_MASK \
- (1ULL << I40IW_CQPSQ_QP_FORCELOOPBACK_SHIFT)
-
-#define I40IW_CQPSQ_QP_CQNUMVALID_SHIFT 47
-#define I40IW_CQPSQ_QP_CQNUMVALID_MASK \
- (1ULL << I40IW_CQPSQ_QP_CQNUMVALID_SHIFT)
-
-#define I40IW_CQPSQ_QP_QPTYPE_SHIFT 48
-#define I40IW_CQPSQ_QP_QPTYPE_MASK (0x3ULL << I40IW_CQPSQ_QP_QPTYPE_SHIFT)
-
-#define I40IW_CQPSQ_QP_MSSCHANGE_SHIFT 52
-#define I40IW_CQPSQ_QP_MSSCHANGE_MASK (1ULL << I40IW_CQPSQ_QP_MSSCHANGE_SHIFT)
-
-#define I40IW_CQPSQ_QP_IGNOREMWBOUND_SHIFT 54
-#define I40IW_CQPSQ_QP_IGNOREMWBOUND_MASK \
- (1ULL << I40IW_CQPSQ_QP_IGNOREMWBOUND_SHIFT)
-
-#define I40IW_CQPSQ_QP_REMOVEHASHENTRY_SHIFT 55
-#define I40IW_CQPSQ_QP_REMOVEHASHENTRY_MASK \
- (1ULL << I40IW_CQPSQ_QP_REMOVEHASHENTRY_SHIFT)
-
-#define I40IW_CQPSQ_QP_TERMACT_SHIFT 56
-#define I40IW_CQPSQ_QP_TERMACT_MASK (0x3ULL << I40IW_CQPSQ_QP_TERMACT_SHIFT)
-
-#define I40IW_CQPSQ_QP_RESETCON_SHIFT 58
-#define I40IW_CQPSQ_QP_RESETCON_MASK (1ULL << I40IW_CQPSQ_QP_RESETCON_SHIFT)
-
-#define I40IW_CQPSQ_QP_ARPTABIDXVALID_SHIFT 59
-#define I40IW_CQPSQ_QP_ARPTABIDXVALID_MASK \
- (1ULL << I40IW_CQPSQ_QP_ARPTABIDXVALID_SHIFT)
-
-#define I40IW_CQPSQ_QP_NEXTIWSTATE_SHIFT 60
-#define I40IW_CQPSQ_QP_NEXTIWSTATE_MASK \
- (0x7ULL << I40IW_CQPSQ_QP_NEXTIWSTATE_SHIFT)
-
-#define I40IW_CQPSQ_QP_DBSHADOWADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_QP_DBSHADOWADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-/* Create/Modify/Destroy CQ */
-#define I40IW_CQPSQ_CQ_CQSIZE_SHIFT 0
-#define I40IW_CQPSQ_CQ_CQSIZE_MASK (0x3ffffUL << I40IW_CQPSQ_CQ_CQSIZE_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CQCTX_SHIFT 0
-#define I40IW_CQPSQ_CQ_CQCTX_MASK \
- (0x7fffffffffffffffULL << I40IW_CQPSQ_CQ_CQCTX_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CQCTX_SHIFT 0
-#define I40IW_CQPSQ_CQ_CQCTX_MASK \
- (0x7fffffffffffffffULL << I40IW_CQPSQ_CQ_CQCTX_SHIFT)
-
-#define I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_SHIFT 0
-#define I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_MASK \
- (0x3ffff << I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CEQID_SHIFT 24
-#define I40IW_CQPSQ_CQ_CEQID_MASK (0x7fUL << I40IW_CQPSQ_CQ_CEQID_SHIFT)
-
-#define I40IW_CQPSQ_CQ_OP_SHIFT 32
-#define I40IW_CQPSQ_CQ_OP_MASK (0x3fULL << I40IW_CQPSQ_CQ_OP_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CQRESIZE_SHIFT 43
-#define I40IW_CQPSQ_CQ_CQRESIZE_MASK (1ULL << I40IW_CQPSQ_CQ_CQRESIZE_SHIFT)
-
-#define I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT 44
-#define I40IW_CQPSQ_CQ_LPBLSIZE_MASK (3ULL << I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CHKOVERFLOW_SHIFT 46
-#define I40IW_CQPSQ_CQ_CHKOVERFLOW_MASK \
- (1ULL << I40IW_CQPSQ_CQ_CHKOVERFLOW_SHIFT)
-
-#define I40IW_CQPSQ_CQ_VIRTMAP_SHIFT 47
-#define I40IW_CQPSQ_CQ_VIRTMAP_MASK (1ULL << I40IW_CQPSQ_CQ_VIRTMAP_SHIFT)
-
-#define I40IW_CQPSQ_CQ_ENCEQEMASK_SHIFT 48
-#define I40IW_CQPSQ_CQ_ENCEQEMASK_MASK \
- (1ULL << I40IW_CQPSQ_CQ_ENCEQEMASK_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CEQIDVALID_SHIFT 49
-#define I40IW_CQPSQ_CQ_CEQIDVALID_MASK \
- (1ULL << I40IW_CQPSQ_CQ_CEQIDVALID_SHIFT)
-
-#define I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_SHIFT 61
-#define I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_MASK \
- (1ULL << I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_SHIFT)
-
-/* Create/Modify/Destroy Shared Receive Queue */
-
-#define I40IW_CQPSQ_SRQ_RQSIZE_SHIFT 0
-#define I40IW_CQPSQ_SRQ_RQSIZE_MASK (0xfUL << I40IW_CQPSQ_SRQ_RQSIZE_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_RQWQESIZE_SHIFT 4
-#define I40IW_CQPSQ_SRQ_RQWQESIZE_MASK \
- (0x7UL << I40IW_CQPSQ_SRQ_RQWQESIZE_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_SRQLIMIT_SHIFT 32
-#define I40IW_CQPSQ_SRQ_SRQLIMIT_MASK \
- (0xfffULL << I40IW_CQPSQ_SRQ_SRQLIMIT_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_SRQCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_SRQ_SRQCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_SRQ_PDID_SHIFT 16
-#define I40IW_CQPSQ_SRQ_PDID_MASK \
- (0x7fffULL << I40IW_CQPSQ_SRQ_PDID_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_SRQID_SHIFT 0
-#define I40IW_CQPSQ_SRQ_SRQID_MASK (0x7fffUL << I40IW_CQPSQ_SRQ_SRQID_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
-#define I40IW_CQPSQ_SRQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
-
-#define I40IW_CQPSQ_SRQ_VIRTMAP_SHIFT I40IW_CQPSQ_CQ_VIRTMAP_SHIFT
-#define I40IW_CQPSQ_SRQ_VIRTMAP_MASK I40IW_CQPSQ_CQ_VIRTMAP_MASK
-
-#define I40IW_CQPSQ_SRQ_TPHEN_SHIFT I40IW_CQPSQ_TPHEN_SHIFT
-#define I40IW_CQPSQ_SRQ_TPHEN_MASK I40IW_CQPSQ_TPHEN_MASK
-
-#define I40IW_CQPSQ_SRQ_ARMLIMITEVENT_SHIFT 61
-#define I40IW_CQPSQ_SRQ_ARMLIMITEVENT_MASK \
- (1ULL << I40IW_CQPSQ_SRQ_ARMLIMITEVENT_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_DBSHADOWAREA_SHIFT 6
-#define I40IW_CQPSQ_SRQ_DBSHADOWAREA_MASK \
- (0x3ffffffffffffffULL << I40IW_CQPSQ_SRQ_DBSHADOWAREA_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_SHIFT 0
-#define I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_MASK \
- (0xfffffffUL << I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_SHIFT)
-
-/* Allocate/Register/Register Shared/Deallocate Stag */
-#define I40IW_CQPSQ_STAG_VA_FBO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_STAG_VA_FBO_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_STAG_STAGLEN_SHIFT 0
-#define I40IW_CQPSQ_STAG_STAGLEN_MASK \
- (0x3fffffffffffULL << I40IW_CQPSQ_STAG_STAGLEN_SHIFT)
-
-#define I40IW_CQPSQ_STAG_PDID_SHIFT 48
-#define I40IW_CQPSQ_STAG_PDID_MASK (0x7fffULL << I40IW_CQPSQ_STAG_PDID_SHIFT)
-
-#define I40IW_CQPSQ_STAG_KEY_SHIFT 0
-#define I40IW_CQPSQ_STAG_KEY_MASK (0xffUL << I40IW_CQPSQ_STAG_KEY_SHIFT)
-
-#define I40IW_CQPSQ_STAG_IDX_SHIFT 8
-#define I40IW_CQPSQ_STAG_IDX_MASK (0xffffffUL << I40IW_CQPSQ_STAG_IDX_SHIFT)
-
-#define I40IW_CQPSQ_STAG_PARENTSTAGIDX_SHIFT 32
-#define I40IW_CQPSQ_STAG_PARENTSTAGIDX_MASK \
- (0xffffffULL << I40IW_CQPSQ_STAG_PARENTSTAGIDX_SHIFT)
-
-#define I40IW_CQPSQ_STAG_MR_SHIFT 43
-#define I40IW_CQPSQ_STAG_MR_MASK (1ULL << I40IW_CQPSQ_STAG_MR_SHIFT)
-
-#define I40IW_CQPSQ_STAG_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
-#define I40IW_CQPSQ_STAG_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
-
-#define I40IW_CQPSQ_STAG_HPAGESIZE_SHIFT 46
-#define I40IW_CQPSQ_STAG_HPAGESIZE_MASK \
- (1ULL << I40IW_CQPSQ_STAG_HPAGESIZE_SHIFT)
-
-#define I40IW_CQPSQ_STAG_ARIGHTS_SHIFT 48
-#define I40IW_CQPSQ_STAG_ARIGHTS_MASK \
- (0x1fULL << I40IW_CQPSQ_STAG_ARIGHTS_SHIFT)
-
-#define I40IW_CQPSQ_STAG_REMACCENABLED_SHIFT 53
-#define I40IW_CQPSQ_STAG_REMACCENABLED_MASK \
- (1ULL << I40IW_CQPSQ_STAG_REMACCENABLED_SHIFT)
-
-#define I40IW_CQPSQ_STAG_VABASEDTO_SHIFT 59
-#define I40IW_CQPSQ_STAG_VABASEDTO_MASK \
- (1ULL << I40IW_CQPSQ_STAG_VABASEDTO_SHIFT)
-
-#define I40IW_CQPSQ_STAG_USEHMCFNIDX_SHIFT 60
-#define I40IW_CQPSQ_STAG_USEHMCFNIDX_MASK \
- (1ULL << I40IW_CQPSQ_STAG_USEHMCFNIDX_SHIFT)
-
-#define I40IW_CQPSQ_STAG_USEPFRID_SHIFT 61
-#define I40IW_CQPSQ_STAG_USEPFRID_MASK \
- (1ULL << I40IW_CQPSQ_STAG_USEPFRID_SHIFT)
-
-#define I40IW_CQPSQ_STAG_PBA_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_STAG_PBA_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_STAG_HMCFNIDX_SHIFT 0
-#define I40IW_CQPSQ_STAG_HMCFNIDX_MASK \
- (0x3fUL << I40IW_CQPSQ_STAG_HMCFNIDX_SHIFT)
-
-#define I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_SHIFT 0
-#define I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_MASK \
- (0xfffffffUL << I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_SHIFT)
-
-/* Query stag */
-#define I40IW_CQPSQ_QUERYSTAG_IDX_SHIFT I40IW_CQPSQ_STAG_IDX_SHIFT
-#define I40IW_CQPSQ_QUERYSTAG_IDX_MASK I40IW_CQPSQ_STAG_IDX_MASK
-
-/* Allocate Local IP Address Entry */
-
-/* Manage Local IP Address Table - MLIPA */
-#define I40IW_CQPSQ_MLIPA_IPV6LO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_MLIPA_IPV6LO_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_MLIPA_IPV6HI_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_MLIPA_IPV6HI_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_MLIPA_IPV4_SHIFT 0
-#define I40IW_CQPSQ_MLIPA_IPV4_MASK \
- (0xffffffffUL << I40IW_CQPSQ_MLIPA_IPV4_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_IPTABLEIDX_SHIFT 0
-#define I40IW_CQPSQ_MLIPA_IPTABLEIDX_MASK \
- (0x3fUL << I40IW_CQPSQ_MLIPA_IPTABLEIDX_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_IPV4VALID_SHIFT 42
-#define I40IW_CQPSQ_MLIPA_IPV4VALID_MASK \
- (1ULL << I40IW_CQPSQ_MLIPA_IPV4VALID_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_IPV6VALID_SHIFT 43
-#define I40IW_CQPSQ_MLIPA_IPV6VALID_MASK \
- (1ULL << I40IW_CQPSQ_MLIPA_IPV6VALID_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_FREEENTRY_SHIFT 62
-#define I40IW_CQPSQ_MLIPA_FREEENTRY_MASK \
- (1ULL << I40IW_CQPSQ_MLIPA_FREEENTRY_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_SHIFT 61
-#define I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_MASK \
- (1ULL << I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC0_SHIFT 0
-#define I40IW_CQPSQ_MLIPA_MAC0_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC0_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC1_SHIFT 8
-#define I40IW_CQPSQ_MLIPA_MAC1_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC1_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC2_SHIFT 16
-#define I40IW_CQPSQ_MLIPA_MAC2_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC2_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC3_SHIFT 24
-#define I40IW_CQPSQ_MLIPA_MAC3_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC3_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC4_SHIFT 32
-#define I40IW_CQPSQ_MLIPA_MAC4_MASK (0xffULL << I40IW_CQPSQ_MLIPA_MAC4_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC5_SHIFT 40
-#define I40IW_CQPSQ_MLIPA_MAC5_MASK (0xffULL << I40IW_CQPSQ_MLIPA_MAC5_SHIFT)
-
-/* Manage ARP Table - MAT */
-#define I40IW_CQPSQ_MAT_REACHMAX_SHIFT 0
-#define I40IW_CQPSQ_MAT_REACHMAX_MASK \
- (0xffffffffUL << I40IW_CQPSQ_MAT_REACHMAX_SHIFT)
-
-#define I40IW_CQPSQ_MAT_MACADDR_SHIFT 0
-#define I40IW_CQPSQ_MAT_MACADDR_MASK \
- (0xffffffffffffULL << I40IW_CQPSQ_MAT_MACADDR_SHIFT)
-
-#define I40IW_CQPSQ_MAT_ARPENTRYIDX_SHIFT 0
-#define I40IW_CQPSQ_MAT_ARPENTRYIDX_MASK \
- (0xfffUL << I40IW_CQPSQ_MAT_ARPENTRYIDX_SHIFT)
-
-#define I40IW_CQPSQ_MAT_ENTRYVALID_SHIFT 42
-#define I40IW_CQPSQ_MAT_ENTRYVALID_MASK \
- (1ULL << I40IW_CQPSQ_MAT_ENTRYVALID_SHIFT)
-
-#define I40IW_CQPSQ_MAT_PERMANENT_SHIFT 43
-#define I40IW_CQPSQ_MAT_PERMANENT_MASK \
- (1ULL << I40IW_CQPSQ_MAT_PERMANENT_SHIFT)
-
-#define I40IW_CQPSQ_MAT_QUERY_SHIFT 44
-#define I40IW_CQPSQ_MAT_QUERY_MASK (1ULL << I40IW_CQPSQ_MAT_QUERY_SHIFT)
-
-/* Manage VF PBLE Backing Pages - MVPBP*/
-#define I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_SHIFT 0
-#define I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_MASK \
- (0x3ffULL << I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_SHIFT)
-
-#define I40IW_CQPSQ_MVPBP_FIRST_PD_INX_SHIFT 16
-#define I40IW_CQPSQ_MVPBP_FIRST_PD_INX_MASK \
- (0x1ffULL << I40IW_CQPSQ_MVPBP_FIRST_PD_INX_SHIFT)
-
-#define I40IW_CQPSQ_MVPBP_SD_INX_SHIFT 32
-#define I40IW_CQPSQ_MVPBP_SD_INX_MASK \
- (0xfffULL << I40IW_CQPSQ_MVPBP_SD_INX_SHIFT)
-
-#define I40IW_CQPSQ_MVPBP_INV_PD_ENT_SHIFT 62
-#define I40IW_CQPSQ_MVPBP_INV_PD_ENT_MASK \
- (0x1ULL << I40IW_CQPSQ_MVPBP_INV_PD_ENT_SHIFT)
-
-#define I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT 3
-#define I40IW_CQPSQ_MVPBP_PD_PLPBA_MASK \
- (0x1fffffffffffffffULL << I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT)
-
-/* Manage Push Page - MPP */
-#define I40IW_INVALID_PUSH_PAGE_INDEX 0xffff
-
-#define I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT 0
-#define I40IW_CQPSQ_MPP_QS_HANDLE_MASK (0xffffUL << \
- I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT)
-
-#define I40IW_CQPSQ_MPP_PPIDX_SHIFT 0
-#define I40IW_CQPSQ_MPP_PPIDX_MASK (0x3ffUL << I40IW_CQPSQ_MPP_PPIDX_SHIFT)
-
-#define I40IW_CQPSQ_MPP_FREE_PAGE_SHIFT 62
-#define I40IW_CQPSQ_MPP_FREE_PAGE_MASK (1ULL << I40IW_CQPSQ_MPP_FREE_PAGE_SHIFT)
-
-/* Upload Context - UCTX */
-#define I40IW_CQPSQ_UCTX_QPCTXADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_UCTX_QPCTXADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_UCTX_QPID_SHIFT 0
-#define I40IW_CQPSQ_UCTX_QPID_MASK (0x3ffffUL << I40IW_CQPSQ_UCTX_QPID_SHIFT)
-
-#define I40IW_CQPSQ_UCTX_QPTYPE_SHIFT 48
-#define I40IW_CQPSQ_UCTX_QPTYPE_MASK (0xfULL << I40IW_CQPSQ_UCTX_QPTYPE_SHIFT)
-
-#define I40IW_CQPSQ_UCTX_RAWFORMAT_SHIFT 61
-#define I40IW_CQPSQ_UCTX_RAWFORMAT_MASK \
- (1ULL << I40IW_CQPSQ_UCTX_RAWFORMAT_SHIFT)
-
-#define I40IW_CQPSQ_UCTX_FREEZEQP_SHIFT 62
-#define I40IW_CQPSQ_UCTX_FREEZEQP_MASK \
- (1ULL << I40IW_CQPSQ_UCTX_FREEZEQP_SHIFT)
-
-/* Manage HMC PM Function Table - MHMC */
-#define I40IW_CQPSQ_MHMC_VFIDX_SHIFT 0
-#define I40IW_CQPSQ_MHMC_VFIDX_MASK (0x7fUL << I40IW_CQPSQ_MHMC_VFIDX_SHIFT)
-
-#define I40IW_CQPSQ_MHMC_FREEPMFN_SHIFT 62
-#define I40IW_CQPSQ_MHMC_FREEPMFN_MASK \
- (1ULL << I40IW_CQPSQ_MHMC_FREEPMFN_SHIFT)
-
-/* Set HMC Resource Profile - SHMCRP */
-#define I40IW_CQPSQ_SHMCRP_HMC_PROFILE_SHIFT 0
-#define I40IW_CQPSQ_SHMCRP_HMC_PROFILE_MASK \
- (0x7ULL << I40IW_CQPSQ_SHMCRP_HMC_PROFILE_SHIFT)
-#define I40IW_CQPSQ_SHMCRP_VFNUM_SHIFT 32
-#define I40IW_CQPSQ_SHMCRP_VFNUM_MASK (0x3fULL << I40IW_CQPSQ_SHMCRP_VFNUM_SHIFT)
-
-/* Create/Destroy CEQ */
-#define I40IW_CQPSQ_CEQ_CEQSIZE_SHIFT 0
-#define I40IW_CQPSQ_CEQ_CEQSIZE_MASK \
- (0x1ffffUL << I40IW_CQPSQ_CEQ_CEQSIZE_SHIFT)
-
-#define I40IW_CQPSQ_CEQ_CEQID_SHIFT 0
-#define I40IW_CQPSQ_CEQ_CEQID_MASK (0x7fUL << I40IW_CQPSQ_CEQ_CEQID_SHIFT)
-
-#define I40IW_CQPSQ_CEQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
-#define I40IW_CQPSQ_CEQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
-
-#define I40IW_CQPSQ_CEQ_VMAP_SHIFT 47
-#define I40IW_CQPSQ_CEQ_VMAP_MASK (1ULL << I40IW_CQPSQ_CEQ_VMAP_SHIFT)
-
-#define I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_SHIFT 0
-#define I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_MASK \
- (0xfffffffUL << I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_SHIFT)
-
-/* Create/Destroy AEQ */
-#define I40IW_CQPSQ_AEQ_AEQECNT_SHIFT 0
-#define I40IW_CQPSQ_AEQ_AEQECNT_MASK \
- (0x7ffffUL << I40IW_CQPSQ_AEQ_AEQECNT_SHIFT)
-
-#define I40IW_CQPSQ_AEQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
-#define I40IW_CQPSQ_AEQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
-
-#define I40IW_CQPSQ_AEQ_VMAP_SHIFT 47
-#define I40IW_CQPSQ_AEQ_VMAP_MASK (1ULL << I40IW_CQPSQ_AEQ_VMAP_SHIFT)
-
-#define I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_SHIFT 0
-#define I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_MASK \
- (0xfffffffUL << I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_SHIFT)
-
-/* Commit FPM Values - CFPM */
-#define I40IW_CQPSQ_CFPM_HMCFNID_SHIFT 0
-#define I40IW_CQPSQ_CFPM_HMCFNID_MASK (0x3fUL << I40IW_CQPSQ_CFPM_HMCFNID_SHIFT)
-
-/* Flush WQEs - FWQE */
-#define I40IW_CQPSQ_FWQE_AECODE_SHIFT 0
-#define I40IW_CQPSQ_FWQE_AECODE_MASK (0xffffUL << I40IW_CQPSQ_FWQE_AECODE_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_AESOURCE_SHIFT 16
-#define I40IW_CQPSQ_FWQE_AESOURCE_MASK \
- (0xfUL << I40IW_CQPSQ_FWQE_AESOURCE_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_RQMNERR_SHIFT 0
-#define I40IW_CQPSQ_FWQE_RQMNERR_MASK \
- (0xffffUL << I40IW_CQPSQ_FWQE_RQMNERR_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_RQMJERR_SHIFT 16
-#define I40IW_CQPSQ_FWQE_RQMJERR_MASK \
- (0xffffUL << I40IW_CQPSQ_FWQE_RQMJERR_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_SQMNERR_SHIFT 32
-#define I40IW_CQPSQ_FWQE_SQMNERR_MASK \
- (0xffffULL << I40IW_CQPSQ_FWQE_SQMNERR_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_SQMJERR_SHIFT 48
-#define I40IW_CQPSQ_FWQE_SQMJERR_MASK \
- (0xffffULL << I40IW_CQPSQ_FWQE_SQMJERR_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_QPID_SHIFT 0
-#define I40IW_CQPSQ_FWQE_QPID_MASK (0x3ffffULL << I40IW_CQPSQ_FWQE_QPID_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_GENERATE_AE_SHIFT 59
-#define I40IW_CQPSQ_FWQE_GENERATE_AE_MASK (1ULL << \
- I40IW_CQPSQ_FWQE_GENERATE_AE_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_USERFLCODE_SHIFT 60
-#define I40IW_CQPSQ_FWQE_USERFLCODE_MASK \
- (1ULL << I40IW_CQPSQ_FWQE_USERFLCODE_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_FLUSHSQ_SHIFT 61
-#define I40IW_CQPSQ_FWQE_FLUSHSQ_MASK (1ULL << I40IW_CQPSQ_FWQE_FLUSHSQ_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_FLUSHRQ_SHIFT 62
-#define I40IW_CQPSQ_FWQE_FLUSHRQ_MASK (1ULL << I40IW_CQPSQ_FWQE_FLUSHRQ_SHIFT)
-
-/* Manage Accelerated Port Table - MAPT */
-#define I40IW_CQPSQ_MAPT_PORT_SHIFT 0
-#define I40IW_CQPSQ_MAPT_PORT_MASK (0xffffUL << I40IW_CQPSQ_MAPT_PORT_SHIFT)
-
-#define I40IW_CQPSQ_MAPT_ADDPORT_SHIFT 62
-#define I40IW_CQPSQ_MAPT_ADDPORT_MASK (1ULL << I40IW_CQPSQ_MAPT_ADDPORT_SHIFT)
-
-/* Update Protocol Engine SDs */
-#define I40IW_CQPSQ_UPESD_SDCMD_SHIFT 0
-#define I40IW_CQPSQ_UPESD_SDCMD_MASK (0xffffffffUL << I40IW_CQPSQ_UPESD_SDCMD_SHIFT)
-
-#define I40IW_CQPSQ_UPESD_SDDATALOW_SHIFT 0
-#define I40IW_CQPSQ_UPESD_SDDATALOW_MASK \
- (0xffffffffUL << I40IW_CQPSQ_UPESD_SDDATALOW_SHIFT)
-
-#define I40IW_CQPSQ_UPESD_SDDATAHI_SHIFT 32
-#define I40IW_CQPSQ_UPESD_SDDATAHI_MASK \
- (0xffffffffULL << I40IW_CQPSQ_UPESD_SDDATAHI_SHIFT)
-#define I40IW_CQPSQ_UPESD_HMCFNID_SHIFT 0
-#define I40IW_CQPSQ_UPESD_HMCFNID_MASK \
- (0x3fUL << I40IW_CQPSQ_UPESD_HMCFNID_SHIFT)
-
-#define I40IW_CQPSQ_UPESD_ENTRY_VALID_SHIFT 63
-#define I40IW_CQPSQ_UPESD_ENTRY_VALID_MASK \
- ((u64)1 << I40IW_CQPSQ_UPESD_ENTRY_VALID_SHIFT)
-
-#define I40IW_CQPSQ_UPESD_ENTRY_COUNT_SHIFT 0
-#define I40IW_CQPSQ_UPESD_ENTRY_COUNT_MASK \
- (0xfUL << I40IW_CQPSQ_UPESD_ENTRY_COUNT_SHIFT)
-
-#define I40IW_CQPSQ_UPESD_SKIP_ENTRY_SHIFT 7
-#define I40IW_CQPSQ_UPESD_SKIP_ENTRY_MASK \
- (0x1UL << I40IW_CQPSQ_UPESD_SKIP_ENTRY_SHIFT)
-
-/* Suspend QP */
-#define I40IW_CQPSQ_SUSPENDQP_QPID_SHIFT 0
-#define I40IW_CQPSQ_SUSPENDQP_QPID_MASK (0x3FFFFUL)
-/* I40IWCQ_QPID_MASK */
-
-/* Resume QP */
-#define I40IW_CQPSQ_RESUMEQP_QSHANDLE_SHIFT 0
-#define I40IW_CQPSQ_RESUMEQP_QSHANDLE_MASK \
- (0xffffffffUL << I40IW_CQPSQ_RESUMEQP_QSHANDLE_SHIFT)
-
-#define I40IW_CQPSQ_RESUMEQP_QPID_SHIFT 0
-#define I40IW_CQPSQ_RESUMEQP_QPID_MASK (0x3FFFFUL)
-/* I40IWCQ_QPID_MASK */
-
-/* IW QP Context */
-#define I40IWQPC_DDP_VER_SHIFT 0
-#define I40IWQPC_DDP_VER_MASK (3UL << I40IWQPC_DDP_VER_SHIFT)
-
-#define I40IWQPC_SNAP_SHIFT 2
-#define I40IWQPC_SNAP_MASK (1UL << I40IWQPC_SNAP_SHIFT)
-
-#define I40IWQPC_IPV4_SHIFT 3
-#define I40IWQPC_IPV4_MASK (1UL << I40IWQPC_IPV4_SHIFT)
-
-#define I40IWQPC_NONAGLE_SHIFT 4
-#define I40IWQPC_NONAGLE_MASK (1UL << I40IWQPC_NONAGLE_SHIFT)
-
-#define I40IWQPC_INSERTVLANTAG_SHIFT 5
-#define I40IWQPC_INSERTVLANTAG_MASK (1 << I40IWQPC_INSERTVLANTAG_SHIFT)
-
-#define I40IWQPC_USESRQ_SHIFT 6
-#define I40IWQPC_USESRQ_MASK (1UL << I40IWQPC_USESRQ_SHIFT)
-
-#define I40IWQPC_TIMESTAMP_SHIFT 7
-#define I40IWQPC_TIMESTAMP_MASK (1UL << I40IWQPC_TIMESTAMP_SHIFT)
-
-#define I40IWQPC_RQWQESIZE_SHIFT 8
-#define I40IWQPC_RQWQESIZE_MASK (3UL << I40IWQPC_RQWQESIZE_SHIFT)
-
-#define I40IWQPC_INSERTL2TAG2_SHIFT 11
-#define I40IWQPC_INSERTL2TAG2_MASK (1UL << I40IWQPC_INSERTL2TAG2_SHIFT)
-
-#define I40IWQPC_LIMIT_SHIFT 12
-#define I40IWQPC_LIMIT_MASK (3UL << I40IWQPC_LIMIT_SHIFT)
-
-#define I40IWQPC_DROPOOOSEG_SHIFT 15
-#define I40IWQPC_DROPOOOSEG_MASK (1UL << I40IWQPC_DROPOOOSEG_SHIFT)
-
-#define I40IWQPC_DUPACK_THRESH_SHIFT 16
-#define I40IWQPC_DUPACK_THRESH_MASK (7UL << I40IWQPC_DUPACK_THRESH_SHIFT)
-
-#define I40IWQPC_ERR_RQ_IDX_VALID_SHIFT 19
-#define I40IWQPC_ERR_RQ_IDX_VALID_MASK (1UL << I40IWQPC_ERR_RQ_IDX_VALID_SHIFT)
-
-#define I40IWQPC_DIS_VLAN_CHECKS_SHIFT 19
-#define I40IWQPC_DIS_VLAN_CHECKS_MASK (7UL << I40IWQPC_DIS_VLAN_CHECKS_SHIFT)
-
-#define I40IWQPC_RCVTPHEN_SHIFT 28
-#define I40IWQPC_RCVTPHEN_MASK (1UL << I40IWQPC_RCVTPHEN_SHIFT)
-
-#define I40IWQPC_XMITTPHEN_SHIFT 29
-#define I40IWQPC_XMITTPHEN_MASK (1ULL << I40IWQPC_XMITTPHEN_SHIFT)
-
-#define I40IWQPC_RQTPHEN_SHIFT 30
-#define I40IWQPC_RQTPHEN_MASK (1UL << I40IWQPC_RQTPHEN_SHIFT)
-
-#define I40IWQPC_SQTPHEN_SHIFT 31
-#define I40IWQPC_SQTPHEN_MASK (1ULL << I40IWQPC_SQTPHEN_SHIFT)
-
-#define I40IWQPC_PPIDX_SHIFT 32
-#define I40IWQPC_PPIDX_MASK (0x3ffULL << I40IWQPC_PPIDX_SHIFT)
-
-#define I40IWQPC_PMENA_SHIFT 47
-#define I40IWQPC_PMENA_MASK (1ULL << I40IWQPC_PMENA_SHIFT)
-
-#define I40IWQPC_RDMAP_VER_SHIFT 62
-#define I40IWQPC_RDMAP_VER_MASK (3ULL << I40IWQPC_RDMAP_VER_SHIFT)
-
-#define I40IWQPC_SQADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPC_SQADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPC_RQADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPC_RQADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPC_TTL_SHIFT 0
-#define I40IWQPC_TTL_MASK (0xffUL << I40IWQPC_TTL_SHIFT)
-
-#define I40IWQPC_RQSIZE_SHIFT 8
-#define I40IWQPC_RQSIZE_MASK (0xfUL << I40IWQPC_RQSIZE_SHIFT)
-
-#define I40IWQPC_SQSIZE_SHIFT 12
-#define I40IWQPC_SQSIZE_MASK (0xfUL << I40IWQPC_SQSIZE_SHIFT)
-
-#define I40IWQPC_SRCMACADDRIDX_SHIFT 16
-#define I40IWQPC_SRCMACADDRIDX_MASK (0x3fUL << I40IWQPC_SRCMACADDRIDX_SHIFT)
-
-#define I40IWQPC_AVOIDSTRETCHACK_SHIFT 23
-#define I40IWQPC_AVOIDSTRETCHACK_MASK (1UL << I40IWQPC_AVOIDSTRETCHACK_SHIFT)
-
-#define I40IWQPC_TOS_SHIFT 24
-#define I40IWQPC_TOS_MASK (0xffUL << I40IWQPC_TOS_SHIFT)
-
-#define I40IWQPC_SRCPORTNUM_SHIFT 32
-#define I40IWQPC_SRCPORTNUM_MASK (0xffffULL << I40IWQPC_SRCPORTNUM_SHIFT)
-
-#define I40IWQPC_DESTPORTNUM_SHIFT 48
-#define I40IWQPC_DESTPORTNUM_MASK (0xffffULL << I40IWQPC_DESTPORTNUM_SHIFT)
-
-#define I40IWQPC_DESTIPADDR0_SHIFT 32
-#define I40IWQPC_DESTIPADDR0_MASK \
- (0xffffffffULL << I40IWQPC_DESTIPADDR0_SHIFT)
-
-#define I40IWQPC_DESTIPADDR1_SHIFT 0
-#define I40IWQPC_DESTIPADDR1_MASK \
- (0xffffffffULL << I40IWQPC_DESTIPADDR1_SHIFT)
-
-#define I40IWQPC_DESTIPADDR2_SHIFT 32
-#define I40IWQPC_DESTIPADDR2_MASK \
- (0xffffffffULL << I40IWQPC_DESTIPADDR2_SHIFT)
-
-#define I40IWQPC_DESTIPADDR3_SHIFT 0
-#define I40IWQPC_DESTIPADDR3_MASK \
- (0xffffffffULL << I40IWQPC_DESTIPADDR3_SHIFT)
-
-#define I40IWQPC_SNDMSS_SHIFT 16
-#define I40IWQPC_SNDMSS_MASK (0x3fffUL << I40IWQPC_SNDMSS_SHIFT)
-
-#define I40IW_UDA_QPC_MAXFRAMESIZE_SHIFT 16
-#define I40IW_UDA_QPC_MAXFRAMESIZE_MASK (0x3fffUL << I40IW_UDA_QPC_MAXFRAMESIZE_SHIFT)
-
-#define I40IWQPC_VLANTAG_SHIFT 32
-#define I40IWQPC_VLANTAG_MASK (0xffffULL << I40IWQPC_VLANTAG_SHIFT)
-
-#define I40IWQPC_ARPIDX_SHIFT 48
-#define I40IWQPC_ARPIDX_MASK (0xffffULL << I40IWQPC_ARPIDX_SHIFT)
-
-#define I40IWQPC_FLOWLABEL_SHIFT 0
-#define I40IWQPC_FLOWLABEL_MASK (0xfffffUL << I40IWQPC_FLOWLABEL_SHIFT)
-
-#define I40IWQPC_WSCALE_SHIFT 20
-#define I40IWQPC_WSCALE_MASK (1UL << I40IWQPC_WSCALE_SHIFT)
-
-#define I40IWQPC_KEEPALIVE_SHIFT 21
-#define I40IWQPC_KEEPALIVE_MASK (1UL << I40IWQPC_KEEPALIVE_SHIFT)
-
-#define I40IWQPC_IGNORE_TCP_OPT_SHIFT 22
-#define I40IWQPC_IGNORE_TCP_OPT_MASK (1UL << I40IWQPC_IGNORE_TCP_OPT_SHIFT)
-
-#define I40IWQPC_IGNORE_TCP_UNS_OPT_SHIFT 23
-#define I40IWQPC_IGNORE_TCP_UNS_OPT_MASK \
- (1UL << I40IWQPC_IGNORE_TCP_UNS_OPT_SHIFT)
-
-#define I40IWQPC_TCPSTATE_SHIFT 28
-#define I40IWQPC_TCPSTATE_MASK (0xfUL << I40IWQPC_TCPSTATE_SHIFT)
-
-#define I40IWQPC_RCVSCALE_SHIFT 32
-#define I40IWQPC_RCVSCALE_MASK (0xfULL << I40IWQPC_RCVSCALE_SHIFT)
-
-#define I40IWQPC_SNDSCALE_SHIFT 40
-#define I40IWQPC_SNDSCALE_MASK (0xfULL << I40IWQPC_SNDSCALE_SHIFT)
-
-#define I40IWQPC_PDIDX_SHIFT 48
-#define I40IWQPC_PDIDX_MASK (0x7fffULL << I40IWQPC_PDIDX_SHIFT)
-
-#define I40IWQPC_KALIVE_TIMER_MAX_PROBES_SHIFT 16
-#define I40IWQPC_KALIVE_TIMER_MAX_PROBES_MASK \
- (0xffUL << I40IWQPC_KALIVE_TIMER_MAX_PROBES_SHIFT)
-
-#define I40IWQPC_KEEPALIVE_INTERVAL_SHIFT 24
-#define I40IWQPC_KEEPALIVE_INTERVAL_MASK \
- (0xffUL << I40IWQPC_KEEPALIVE_INTERVAL_SHIFT)
-
-#define I40IWQPC_TIMESTAMP_RECENT_SHIFT 0
-#define I40IWQPC_TIMESTAMP_RECENT_MASK \
- (0xffffffffUL << I40IWQPC_TIMESTAMP_RECENT_SHIFT)
-
-#define I40IWQPC_TIMESTAMP_AGE_SHIFT 32
-#define I40IWQPC_TIMESTAMP_AGE_MASK \
- (0xffffffffULL << I40IWQPC_TIMESTAMP_AGE_SHIFT)
-
-#define I40IWQPC_SNDNXT_SHIFT 0
-#define I40IWQPC_SNDNXT_MASK (0xffffffffUL << I40IWQPC_SNDNXT_SHIFT)
-
-#define I40IWQPC_SNDWND_SHIFT 32
-#define I40IWQPC_SNDWND_MASK (0xffffffffULL << I40IWQPC_SNDWND_SHIFT)
-
-#define I40IWQPC_RCVNXT_SHIFT 0
-#define I40IWQPC_RCVNXT_MASK (0xffffffffUL << I40IWQPC_RCVNXT_SHIFT)
-
-#define I40IWQPC_RCVWND_SHIFT 32
-#define I40IWQPC_RCVWND_MASK (0xffffffffULL << I40IWQPC_RCVWND_SHIFT)
-
-#define I40IWQPC_SNDMAX_SHIFT 0
-#define I40IWQPC_SNDMAX_MASK (0xffffffffUL << I40IWQPC_SNDMAX_SHIFT)
-
-#define I40IWQPC_SNDUNA_SHIFT 32
-#define I40IWQPC_SNDUNA_MASK (0xffffffffULL << I40IWQPC_SNDUNA_SHIFT)
-
-#define I40IWQPC_SRTT_SHIFT 0
-#define I40IWQPC_SRTT_MASK (0xffffffffUL << I40IWQPC_SRTT_SHIFT)
-
-#define I40IWQPC_RTTVAR_SHIFT 32
-#define I40IWQPC_RTTVAR_MASK (0xffffffffULL << I40IWQPC_RTTVAR_SHIFT)
-
-#define I40IWQPC_SSTHRESH_SHIFT 0
-#define I40IWQPC_SSTHRESH_MASK (0xffffffffUL << I40IWQPC_SSTHRESH_SHIFT)
-
-#define I40IWQPC_CWND_SHIFT 32
-#define I40IWQPC_CWND_MASK (0xffffffffULL << I40IWQPC_CWND_SHIFT)
-
-#define I40IWQPC_SNDWL1_SHIFT 0
-#define I40IWQPC_SNDWL1_MASK (0xffffffffUL << I40IWQPC_SNDWL1_SHIFT)
-
-#define I40IWQPC_SNDWL2_SHIFT 32
-#define I40IWQPC_SNDWL2_MASK (0xffffffffULL << I40IWQPC_SNDWL2_SHIFT)
-
-#define I40IWQPC_ERR_RQ_IDX_SHIFT 32
-#define I40IWQPC_ERR_RQ_IDX_MASK (0x3fffULL << I40IWQPC_ERR_RQ_IDX_SHIFT)
-
-#define I40IWQPC_MAXSNDWND_SHIFT 0
-#define I40IWQPC_MAXSNDWND_MASK (0xffffffffUL << I40IWQPC_MAXSNDWND_SHIFT)
-
-#define I40IWQPC_REXMIT_THRESH_SHIFT 48
-#define I40IWQPC_REXMIT_THRESH_MASK (0x3fULL << I40IWQPC_REXMIT_THRESH_SHIFT)
-
-#define I40IWQPC_TXCQNUM_SHIFT 0
-#define I40IWQPC_TXCQNUM_MASK (0x1ffffUL << I40IWQPC_TXCQNUM_SHIFT)
-
-#define I40IWQPC_RXCQNUM_SHIFT 32
-#define I40IWQPC_RXCQNUM_MASK (0x1ffffULL << I40IWQPC_RXCQNUM_SHIFT)
-
-#define I40IWQPC_STAT_INDEX_SHIFT 0
-#define I40IWQPC_STAT_INDEX_MASK (0x1fULL << I40IWQPC_STAT_INDEX_SHIFT)
-
-#define I40IWQPC_Q2ADDR_SHIFT 0
-#define I40IWQPC_Q2ADDR_MASK (0xffffffffffffff00ULL << I40IWQPC_Q2ADDR_SHIFT)
-
-#define I40IWQPC_LASTBYTESENT_SHIFT 0
-#define I40IWQPC_LASTBYTESENT_MASK (0xffUL << I40IWQPC_LASTBYTESENT_SHIFT)
-
-#define I40IWQPC_SRQID_SHIFT 32
-#define I40IWQPC_SRQID_MASK (0xffULL << I40IWQPC_SRQID_SHIFT)
-
-#define I40IWQPC_ORDSIZE_SHIFT 0
-#define I40IWQPC_ORDSIZE_MASK (0x7fUL << I40IWQPC_ORDSIZE_SHIFT)
-
-#define I40IWQPC_IRDSIZE_SHIFT 16
-#define I40IWQPC_IRDSIZE_MASK (0x3UL << I40IWQPC_IRDSIZE_SHIFT)
-
-#define I40IWQPC_WRRDRSPOK_SHIFT 20
-#define I40IWQPC_WRRDRSPOK_MASK (1UL << I40IWQPC_WRRDRSPOK_SHIFT)
-
-#define I40IWQPC_RDOK_SHIFT 21
-#define I40IWQPC_RDOK_MASK (1UL << I40IWQPC_RDOK_SHIFT)
-
-#define I40IWQPC_SNDMARKERS_SHIFT 22
-#define I40IWQPC_SNDMARKERS_MASK (1UL << I40IWQPC_SNDMARKERS_SHIFT)
-
-#define I40IWQPC_BINDEN_SHIFT 23
-#define I40IWQPC_BINDEN_MASK (1UL << I40IWQPC_BINDEN_SHIFT)
-
-#define I40IWQPC_FASTREGEN_SHIFT 24
-#define I40IWQPC_FASTREGEN_MASK (1UL << I40IWQPC_FASTREGEN_SHIFT)
-
-#define I40IWQPC_PRIVEN_SHIFT 25
-#define I40IWQPC_PRIVEN_MASK (1UL << I40IWQPC_PRIVEN_SHIFT)
-
-#define I40IWQPC_USESTATSINSTANCE_SHIFT 26
-#define I40IWQPC_USESTATSINSTANCE_MASK (1UL << I40IWQPC_USESTATSINSTANCE_SHIFT)
-
-#define I40IWQPC_IWARPMODE_SHIFT 28
-#define I40IWQPC_IWARPMODE_MASK (1UL << I40IWQPC_IWARPMODE_SHIFT)
-
-#define I40IWQPC_RCVMARKERS_SHIFT 29
-#define I40IWQPC_RCVMARKERS_MASK (1UL << I40IWQPC_RCVMARKERS_SHIFT)
-
-#define I40IWQPC_ALIGNHDRS_SHIFT 30
-#define I40IWQPC_ALIGNHDRS_MASK (1UL << I40IWQPC_ALIGNHDRS_SHIFT)
-
-#define I40IWQPC_RCVNOMPACRC_SHIFT 31
-#define I40IWQPC_RCVNOMPACRC_MASK (1UL << I40IWQPC_RCVNOMPACRC_SHIFT)
-
-#define I40IWQPC_RCVMARKOFFSET_SHIFT 33
-#define I40IWQPC_RCVMARKOFFSET_MASK (0x1ffULL << I40IWQPC_RCVMARKOFFSET_SHIFT)
-
-#define I40IWQPC_SNDMARKOFFSET_SHIFT 48
-#define I40IWQPC_SNDMARKOFFSET_MASK (0x1ffULL << I40IWQPC_SNDMARKOFFSET_SHIFT)
-
-#define I40IWQPC_QPCOMPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPC_QPCOMPCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPC_SQTPHVAL_SHIFT 0
-#define I40IWQPC_SQTPHVAL_MASK (0xffUL << I40IWQPC_SQTPHVAL_SHIFT)
-
-#define I40IWQPC_RQTPHVAL_SHIFT 8
-#define I40IWQPC_RQTPHVAL_MASK (0xffUL << I40IWQPC_RQTPHVAL_SHIFT)
-
-#define I40IWQPC_QSHANDLE_SHIFT 16
-#define I40IWQPC_QSHANDLE_MASK (0x3ffUL << I40IWQPC_QSHANDLE_SHIFT)
-
-#define I40IWQPC_EXCEPTION_LAN_QUEUE_SHIFT 32
-#define I40IWQPC_EXCEPTION_LAN_QUEUE_MASK (0xfffULL << \
- I40IWQPC_EXCEPTION_LAN_QUEUE_SHIFT)
-
-#define I40IWQPC_LOCAL_IPADDR3_SHIFT 0
-#define I40IWQPC_LOCAL_IPADDR3_MASK \
- (0xffffffffUL << I40IWQPC_LOCAL_IPADDR3_SHIFT)
-
-#define I40IWQPC_LOCAL_IPADDR2_SHIFT 32
-#define I40IWQPC_LOCAL_IPADDR2_MASK \
- (0xffffffffULL << I40IWQPC_LOCAL_IPADDR2_SHIFT)
-
-#define I40IWQPC_LOCAL_IPADDR1_SHIFT 0
-#define I40IWQPC_LOCAL_IPADDR1_MASK \
- (0xffffffffUL << I40IWQPC_LOCAL_IPADDR1_SHIFT)
-
-#define I40IWQPC_LOCAL_IPADDR0_SHIFT 32
-#define I40IWQPC_LOCAL_IPADDR0_MASK \
- (0xffffffffULL << I40IWQPC_LOCAL_IPADDR0_SHIFT)
-
-/* wqe size considering 32 bytes per wqe*/
-#define I40IW_QP_SW_MIN_WQSIZE 4 /*in WRs*/
-#define I40IW_SQ_RSVD 2
-#define I40IW_RQ_RSVD 1
-#define I40IW_MAX_QUANTAS_PER_WR 2
-#define I40IW_QP_SW_MAX_SQ_QUANTAS 2048
-#define I40IW_QP_SW_MAX_RQ_QUANTAS 16384
-#define I40IW_MAX_QP_WRS ((I40IW_QP_SW_MAX_SQ_QUANTAS / I40IW_MAX_QUANTAS_PER_WR) - 1)
-
-#define I40IWQP_OP_RDMA_WRITE 0
-#define I40IWQP_OP_RDMA_READ 1
-#define I40IWQP_OP_RDMA_SEND 3
-#define I40IWQP_OP_RDMA_SEND_INV 4
-#define I40IWQP_OP_RDMA_SEND_SOL_EVENT 5
-#define I40IWQP_OP_RDMA_SEND_SOL_EVENT_INV 6
-#define I40IWQP_OP_BIND_MW 8
-#define I40IWQP_OP_FAST_REGISTER 9
-#define I40IWQP_OP_LOCAL_INVALIDATE 10
-#define I40IWQP_OP_RDMA_READ_LOC_INV 11
-#define I40IWQP_OP_NOP 12
-
-#define I40IW_RSVD_SHIFT 41
-#define I40IW_RSVD_MASK (0x7fffULL << I40IW_RSVD_SHIFT)
-
-/* iwarp QP SQ WQE common fields */
-#define I40IWQPSQ_OPCODE_SHIFT 32
-#define I40IWQPSQ_OPCODE_MASK (0x3fULL << I40IWQPSQ_OPCODE_SHIFT)
-
-#define I40IWQPSQ_ADDFRAGCNT_SHIFT 38
-#define I40IWQPSQ_ADDFRAGCNT_MASK (0x7ULL << I40IWQPSQ_ADDFRAGCNT_SHIFT)
-
-#define I40IWQPSQ_PUSHWQE_SHIFT 56
-#define I40IWQPSQ_PUSHWQE_MASK (1ULL << I40IWQPSQ_PUSHWQE_SHIFT)
-
-#define I40IWQPSQ_STREAMMODE_SHIFT 58
-#define I40IWQPSQ_STREAMMODE_MASK (1ULL << I40IWQPSQ_STREAMMODE_SHIFT)
-
-#define I40IWQPSQ_WAITFORRCVPDU_SHIFT 59
-#define I40IWQPSQ_WAITFORRCVPDU_MASK (1ULL << I40IWQPSQ_WAITFORRCVPDU_SHIFT)
-
-#define I40IWQPSQ_READFENCE_SHIFT 60
-#define I40IWQPSQ_READFENCE_MASK (1ULL << I40IWQPSQ_READFENCE_SHIFT)
-
-#define I40IWQPSQ_LOCALFENCE_SHIFT 61
-#define I40IWQPSQ_LOCALFENCE_MASK (1ULL << I40IWQPSQ_LOCALFENCE_SHIFT)
-
-#define I40IWQPSQ_SIGCOMPL_SHIFT 62
-#define I40IWQPSQ_SIGCOMPL_MASK (1ULL << I40IWQPSQ_SIGCOMPL_SHIFT)
-
-#define I40IWQPSQ_VALID_SHIFT 63
-#define I40IWQPSQ_VALID_MASK (1ULL << I40IWQPSQ_VALID_SHIFT)
-
-#define I40IWQPSQ_FRAG_TO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPSQ_FRAG_TO_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPSQ_FRAG_LEN_SHIFT 0
-#define I40IWQPSQ_FRAG_LEN_MASK (0xffffffffUL << I40IWQPSQ_FRAG_LEN_SHIFT)
-
-#define I40IWQPSQ_FRAG_STAG_SHIFT 32
-#define I40IWQPSQ_FRAG_STAG_MASK (0xffffffffULL << I40IWQPSQ_FRAG_STAG_SHIFT)
-
-#define I40IWQPSQ_REMSTAGINV_SHIFT 0
-#define I40IWQPSQ_REMSTAGINV_MASK (0xffffffffUL << I40IWQPSQ_REMSTAGINV_SHIFT)
-
-#define I40IWQPSQ_INLINEDATAFLAG_SHIFT 57
-#define I40IWQPSQ_INLINEDATAFLAG_MASK (1ULL << I40IWQPSQ_INLINEDATAFLAG_SHIFT)
-
-#define I40IWQPSQ_INLINEDATALEN_SHIFT 48
-#define I40IWQPSQ_INLINEDATALEN_MASK \
- (0x7fULL << I40IWQPSQ_INLINEDATALEN_SHIFT)
-
-/* iwarp send with push mode */
-#define I40IWQPSQ_WQDESCIDX_SHIFT 0
-#define I40IWQPSQ_WQDESCIDX_MASK (0x3fffUL << I40IWQPSQ_WQDESCIDX_SHIFT)
-
-/* rdma write */
-#define I40IWQPSQ_REMSTAG_SHIFT 0
-#define I40IWQPSQ_REMSTAG_MASK (0xffffffffUL << I40IWQPSQ_REMSTAG_SHIFT)
-
-#define I40IWQPSQ_REMTO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPSQ_REMTO_MASK I40IW_CQPHC_QPCTX_MASK
-
-/* memory window */
-#define I40IWQPSQ_STAGRIGHTS_SHIFT 48
-#define I40IWQPSQ_STAGRIGHTS_MASK (0x1fULL << I40IWQPSQ_STAGRIGHTS_SHIFT)
-
-#define I40IWQPSQ_VABASEDTO_SHIFT 53
-#define I40IWQPSQ_VABASEDTO_MASK (1ULL << I40IWQPSQ_VABASEDTO_SHIFT)
-
-#define I40IWQPSQ_MWLEN_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPSQ_MWLEN_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPSQ_PARENTMRSTAG_SHIFT 0
-#define I40IWQPSQ_PARENTMRSTAG_MASK \
- (0xffffffffUL << I40IWQPSQ_PARENTMRSTAG_SHIFT)
-
-#define I40IWQPSQ_MWSTAG_SHIFT 32
-#define I40IWQPSQ_MWSTAG_MASK (0xffffffffULL << I40IWQPSQ_MWSTAG_SHIFT)
-
-#define I40IWQPSQ_BASEVA_TO_FBO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPSQ_BASEVA_TO_FBO_MASK I40IW_CQPHC_QPCTX_MASK
-
-/* Local Invalidate */
-#define I40IWQPSQ_LOCSTAG_SHIFT 32
-#define I40IWQPSQ_LOCSTAG_MASK (0xffffffffULL << I40IWQPSQ_LOCSTAG_SHIFT)
-
-/* Fast Register */
-#define I40IWQPSQ_STAGKEY_SHIFT 0
-#define I40IWQPSQ_STAGKEY_MASK (0xffUL << I40IWQPSQ_STAGKEY_SHIFT)
-
-#define I40IWQPSQ_STAGINDEX_SHIFT 8
-#define I40IWQPSQ_STAGINDEX_MASK (0xffffffUL << I40IWQPSQ_STAGINDEX_SHIFT)
-
-#define I40IWQPSQ_COPYHOSTPBLS_SHIFT 43
-#define I40IWQPSQ_COPYHOSTPBLS_MASK (1ULL << I40IWQPSQ_COPYHOSTPBLS_SHIFT)
-
-#define I40IWQPSQ_LPBLSIZE_SHIFT 44
-#define I40IWQPSQ_LPBLSIZE_MASK (3ULL << I40IWQPSQ_LPBLSIZE_SHIFT)
-
-#define I40IWQPSQ_HPAGESIZE_SHIFT 46
-#define I40IWQPSQ_HPAGESIZE_MASK (3ULL << I40IWQPSQ_HPAGESIZE_SHIFT)
-
-#define I40IWQPSQ_STAGLEN_SHIFT 0
-#define I40IWQPSQ_STAGLEN_MASK (0x1ffffffffffULL << I40IWQPSQ_STAGLEN_SHIFT)
-
-#define I40IWQPSQ_FIRSTPMPBLIDXLO_SHIFT 48
-#define I40IWQPSQ_FIRSTPMPBLIDXLO_MASK \
- (0xffffULL << I40IWQPSQ_FIRSTPMPBLIDXLO_SHIFT)
-
-#define I40IWQPSQ_FIRSTPMPBLIDXHI_SHIFT 0
-#define I40IWQPSQ_FIRSTPMPBLIDXHI_MASK \
- (0xfffUL << I40IWQPSQ_FIRSTPMPBLIDXHI_SHIFT)
-
-#define I40IWQPSQ_PBLADDR_SHIFT 12
-#define I40IWQPSQ_PBLADDR_MASK (0xfffffffffffffULL << I40IWQPSQ_PBLADDR_SHIFT)
-
-/* iwarp QP RQ WQE common fields */
-#define I40IWQPRQ_ADDFRAGCNT_SHIFT I40IWQPSQ_ADDFRAGCNT_SHIFT
-#define I40IWQPRQ_ADDFRAGCNT_MASK I40IWQPSQ_ADDFRAGCNT_MASK
-
-#define I40IWQPRQ_VALID_SHIFT I40IWQPSQ_VALID_SHIFT
-#define I40IWQPRQ_VALID_MASK I40IWQPSQ_VALID_MASK
-
-#define I40IWQPRQ_COMPLCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPRQ_COMPLCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPRQ_FRAG_LEN_SHIFT I40IWQPSQ_FRAG_LEN_SHIFT
-#define I40IWQPRQ_FRAG_LEN_MASK I40IWQPSQ_FRAG_LEN_MASK
-
-#define I40IWQPRQ_STAG_SHIFT I40IWQPSQ_FRAG_STAG_SHIFT
-#define I40IWQPRQ_STAG_MASK I40IWQPSQ_FRAG_STAG_MASK
-
-#define I40IWQPRQ_TO_SHIFT I40IWQPSQ_FRAG_TO_SHIFT
-#define I40IWQPRQ_TO_MASK I40IWQPSQ_FRAG_TO_MASK
-
-/* Query FPM CQP buf */
-#define I40IW_QUERY_FPM_MAX_QPS_SHIFT 0
-#define I40IW_QUERY_FPM_MAX_QPS_MASK \
- (0x7ffffUL << I40IW_QUERY_FPM_MAX_QPS_SHIFT)
-
-#define I40IW_QUERY_FPM_MAX_CQS_SHIFT 0
-#define I40IW_QUERY_FPM_MAX_CQS_MASK \
- (0x3ffffUL << I40IW_QUERY_FPM_MAX_CQS_SHIFT)
-
-#define I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_SHIFT 0
-#define I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_MASK \
- (0x3fffUL << I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_SHIFT)
-
-#define I40IW_QUERY_FPM_MAX_PE_SDS_SHIFT 32
-#define I40IW_QUERY_FPM_MAX_PE_SDS_MASK \
- (0x3fffULL << I40IW_QUERY_FPM_MAX_PE_SDS_SHIFT)
-
-#define I40IW_QUERY_FPM_MAX_QPS_SHIFT 0
-#define I40IW_QUERY_FPM_MAX_QPS_MASK \
- (0x7ffffUL << I40IW_QUERY_FPM_MAX_QPS_SHIFT)
-
-#define I40IW_QUERY_FPM_MAX_CQS_SHIFT 0
-#define I40IW_QUERY_FPM_MAX_CQS_MASK \
- (0x3ffffUL << I40IW_QUERY_FPM_MAX_CQS_SHIFT)
-
-#define I40IW_QUERY_FPM_MAX_CEQS_SHIFT 0
-#define I40IW_QUERY_FPM_MAX_CEQS_MASK \
- (0xffUL << I40IW_QUERY_FPM_MAX_CEQS_SHIFT)
-
-#define I40IW_QUERY_FPM_XFBLOCKSIZE_SHIFT 32
-#define I40IW_QUERY_FPM_XFBLOCKSIZE_MASK \
- (0xffffffffULL << I40IW_QUERY_FPM_XFBLOCKSIZE_SHIFT)
-
-#define I40IW_QUERY_FPM_Q1BLOCKSIZE_SHIFT 32
-#define I40IW_QUERY_FPM_Q1BLOCKSIZE_MASK \
- (0xffffffffULL << I40IW_QUERY_FPM_Q1BLOCKSIZE_SHIFT)
-
-#define I40IW_QUERY_FPM_HTMULTIPLIER_SHIFT 16
-#define I40IW_QUERY_FPM_HTMULTIPLIER_MASK \
- (0xfUL << I40IW_QUERY_FPM_HTMULTIPLIER_SHIFT)
-
-#define I40IW_QUERY_FPM_TIMERBUCKET_SHIFT 32
-#define I40IW_QUERY_FPM_TIMERBUCKET_MASK \
- (0xffFFULL << I40IW_QUERY_FPM_TIMERBUCKET_SHIFT)
-
-/* Static HMC pages allocated buf */
-#define I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_SHIFT 0
-#define I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_MASK \
- (0x3fUL << I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_SHIFT)
-
-#define I40IW_HW_PAGE_SIZE 4096
-#define I40IW_DONE_COUNT 1000
-#define I40IW_SLEEP_COUNT 10
-
-enum {
- I40IW_QUEUES_ALIGNMENT_MASK = (128 - 1),
- I40IW_AEQ_ALIGNMENT_MASK = (256 - 1),
- I40IW_Q2_ALIGNMENT_MASK = (256 - 1),
- I40IW_CEQ_ALIGNMENT_MASK = (256 - 1),
- I40IW_CQ0_ALIGNMENT_MASK = (256 - 1),
- I40IW_HOST_CTX_ALIGNMENT_MASK = (4 - 1),
- I40IW_SHADOWAREA_MASK = (128 - 1),
- I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK = (4 - 1),
- I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK = (4 - 1)
-};
-
-enum i40iw_alignment {
- I40IW_CQP_ALIGNMENT = 0x200,
- I40IW_AEQ_ALIGNMENT = 0x100,
- I40IW_CEQ_ALIGNMENT = 0x100,
- I40IW_CQ0_ALIGNMENT = 0x100,
- I40IW_SD_BUF_ALIGNMENT = 0x80,
- I40IW_FEATURE_BUF_ALIGNMENT = 0x8
-};
-
-#define I40IW_WQE_SIZE_64 64
-
-#define I40IW_QP_WQE_MIN_SIZE 32
-#define I40IW_QP_WQE_MAX_SIZE 128
-
-#define I40IW_UPDATE_SD_BUF_SIZE 128
-
-#define I40IW_CQE_QTYPE_RQ 0
-#define I40IW_CQE_QTYPE_SQ 1
-
-#define I40IW_RING_INIT(_ring, _size) \
- { \
- (_ring).head = 0; \
- (_ring).tail = 0; \
- (_ring).size = (_size); \
- }
-#define I40IW_RING_GETSIZE(_ring) ((_ring).size)
-#define I40IW_RING_GETCURRENT_HEAD(_ring) ((_ring).head)
-#define I40IW_RING_GETCURRENT_TAIL(_ring) ((_ring).tail)
-
-#define I40IW_RING_MOVE_HEAD(_ring, _retcode) \
- { \
- register u32 size; \
- size = (_ring).size; \
- if (!I40IW_RING_FULL_ERR(_ring)) { \
- (_ring).head = ((_ring).head + 1) % size; \
- (_retcode) = 0; \
- } else { \
- (_retcode) = I40IW_ERR_RING_FULL; \
- } \
- }
-
-#define I40IW_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
- { \
- register u32 size; \
- size = (_ring).size; \
- if ((I40IW_RING_WORK_AVAILABLE(_ring) + (_count)) < size) { \
- (_ring).head = ((_ring).head + (_count)) % size; \
- (_retcode) = 0; \
- } else { \
- (_retcode) = I40IW_ERR_RING_FULL; \
- } \
- }
-
-#define I40IW_RING_MOVE_TAIL(_ring) \
- (_ring).tail = ((_ring).tail + 1) % (_ring).size
-
-#define I40IW_RING_MOVE_HEAD_NOCHECK(_ring) \
- (_ring).head = ((_ring).head + 1) % (_ring).size
-
-#define I40IW_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \
- (_ring).tail = ((_ring).tail + (_count)) % (_ring).size
-
-#define I40IW_RING_SET_TAIL(_ring, _pos) \
- (_ring).tail = (_pos) % (_ring).size
-
-#define I40IW_RING_FULL_ERR(_ring) \
- ( \
- (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 1)) \
- )
-
-#define I40IW_ERR_RING_FULL2(_ring) \
- ( \
- (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 2)) \
- )
-
-#define I40IW_ERR_RING_FULL3(_ring) \
- ( \
- (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 3)) \
- )
-
-#define I40IW_RING_MORE_WORK(_ring) \
- ( \
- (I40IW_RING_WORK_AVAILABLE(_ring) != 0) \
- )
-
-#define I40IW_RING_WORK_AVAILABLE(_ring) \
- ( \
- (((_ring).head + (_ring).size - (_ring).tail) % (_ring).size) \
- )
-
-#define I40IW_RING_GET_WQES_AVAILABLE(_ring) \
- ( \
- ((_ring).size - I40IW_RING_WORK_AVAILABLE(_ring) - 1) \
- )
-
-#define I40IW_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \
- { \
- index = I40IW_RING_GETCURRENT_HEAD(_ring); \
- I40IW_RING_MOVE_HEAD(_ring, _retcode); \
- }
-
-/* Async Events codes */
-#define I40IW_AE_AMP_UNALLOCATED_STAG 0x0102
-#define I40IW_AE_AMP_INVALID_STAG 0x0103
-#define I40IW_AE_AMP_BAD_QP 0x0104
-#define I40IW_AE_AMP_BAD_PD 0x0105
-#define I40IW_AE_AMP_BAD_STAG_KEY 0x0106
-#define I40IW_AE_AMP_BAD_STAG_INDEX 0x0107
-#define I40IW_AE_AMP_BOUNDS_VIOLATION 0x0108
-#define I40IW_AE_AMP_RIGHTS_VIOLATION 0x0109
-#define I40IW_AE_AMP_TO_WRAP 0x010a
-#define I40IW_AE_AMP_FASTREG_SHARED 0x010b
-#define I40IW_AE_AMP_FASTREG_VALID_STAG 0x010c
-#define I40IW_AE_AMP_FASTREG_MW_STAG 0x010d
-#define I40IW_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
-#define I40IW_AE_AMP_FASTREG_PBL_TABLE_OVERFLOW 0x010f
-#define I40IW_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
-#define I40IW_AE_AMP_INVALIDATE_SHARED 0x0111
-#define I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
-#define I40IW_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
-#define I40IW_AE_AMP_MWBIND_VALID_STAG 0x0114
-#define I40IW_AE_AMP_MWBIND_OF_MR_STAG 0x0115
-#define I40IW_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
-#define I40IW_AE_AMP_MWBIND_TO_MW_STAG 0x0117
-#define I40IW_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
-#define I40IW_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
-#define I40IW_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
-#define I40IW_AE_AMP_MWBIND_BIND_DISABLED 0x011b
-#define I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
-#define I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
-#define I40IW_AE_BAD_CLOSE 0x0201
-#define I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
-#define I40IW_AE_CQ_OPERATION_ERROR 0x0203
-#define I40IW_AE_PRIV_OPERATION_DENIED 0x011c
-#define I40IW_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
-#define I40IW_AE_STAG_ZERO_INVALID 0x0206
-#define I40IW_AE_IB_RREQ_AND_Q1_FULL 0x0207
-#define I40IW_AE_WQE_UNEXPECTED_OPCODE 0x020a
-#define I40IW_AE_WQE_INVALID_PARAMETER 0x020b
-#define I40IW_AE_WQE_LSMM_TOO_LONG 0x0220
-#define I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
-#define I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
-#define I40IW_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
-#define I40IW_AE_DDP_UBE_INVALID_MO 0x0305
-#define I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
-#define I40IW_AE_DDP_UBE_INVALID_QN 0x0307
-#define I40IW_AE_DDP_NO_L_BIT 0x0308
-#define I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
-#define I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
-#define I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
-#define I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
-#define I40IW_AE_INVALID_ARP_ENTRY 0x0401
-#define I40IW_AE_INVALID_TCP_OPTION_RCVD 0x0402
-#define I40IW_AE_STALE_ARP_ENTRY 0x0403
-#define I40IW_AE_INVALID_MAC_ENTRY 0x0405
-#define I40IW_AE_LLP_CLOSE_COMPLETE 0x0501
-#define I40IW_AE_LLP_CONNECTION_RESET 0x0502
-#define I40IW_AE_LLP_FIN_RECEIVED 0x0503
-#define I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
-#define I40IW_AE_LLP_SEGMENT_TOO_LARGE 0x0506
-#define I40IW_AE_LLP_SEGMENT_TOO_SMALL 0x0507
-#define I40IW_AE_LLP_SYN_RECEIVED 0x0508
-#define I40IW_AE_LLP_TERMINATE_RECEIVED 0x0509
-#define I40IW_AE_LLP_TOO_MANY_RETRIES 0x050a
-#define I40IW_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
-#define I40IW_AE_LLP_DOUBT_REACHABILITY 0x050c
-#define I40IW_AE_LLP_RX_VLAN_MISMATCH 0x050d
-#define I40IW_AE_RESOURCE_EXHAUSTION 0x0520
-#define I40IW_AE_RESET_SENT 0x0601
-#define I40IW_AE_TERMINATE_SENT 0x0602
-#define I40IW_AE_RESET_NOT_SENT 0x0603
-#define I40IW_AE_LCE_QP_CATASTROPHIC 0x0700
-#define I40IW_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
-#define I40IW_AE_LCE_CQ_CATASTROPHIC 0x0702
-#define I40IW_AE_QP_SUSPEND_COMPLETE 0x0900
-
-#define OP_DELETE_LOCAL_MAC_IPADDR_ENTRY 1
-#define OP_CEQ_DESTROY 2
-#define OP_AEQ_DESTROY 3
-#define OP_DELETE_ARP_CACHE_ENTRY 4
-#define OP_MANAGE_APBVT_ENTRY 5
-#define OP_CEQ_CREATE 6
-#define OP_AEQ_CREATE 7
-#define OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY 8
-#define OP_ADD_LOCAL_MAC_IPADDR_ENTRY 9
-#define OP_MANAGE_QHASH_TABLE_ENTRY 10
-#define OP_QP_MODIFY 11
-#define OP_QP_UPLOAD_CONTEXT 12
-#define OP_CQ_CREATE 13
-#define OP_CQ_DESTROY 14
-#define OP_QP_CREATE 15
-#define OP_QP_DESTROY 16
-#define OP_ALLOC_STAG 17
-#define OP_MR_REG_NON_SHARED 18
-#define OP_DEALLOC_STAG 19
-#define OP_MW_ALLOC 20
-#define OP_QP_FLUSH_WQES 21
-#define OP_ADD_ARP_CACHE_ENTRY 22
-#define OP_MANAGE_PUSH_PAGE 23
-#define OP_UPDATE_PE_SDS 24
-#define OP_MANAGE_HMC_PM_FUNC_TABLE 25
-#define OP_SUSPEND 26
-#define OP_RESUME 27
-#define OP_MANAGE_VF_PBLE_BP 28
-#define OP_QUERY_FPM_VALUES 29
-#define OP_COMMIT_FPM_VALUES 30
-#define OP_REQUESTED_COMMANDS 31
-#define OP_COMPLETED_COMMANDS 32
-#define OP_GEN_AE 33
-#define OP_QUERY_RDMA_FEATURES 34
-#define OP_SIZE_CQP_STAT_ARRAY 35
-
-#endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_hmc.c b/drivers/infiniband/hw/i40iw/i40iw_hmc.c
deleted file mode 100644
index 5484cbf55f0f..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_hmc.c
+++ /dev/null
@@ -1,821 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include "i40iw_osdep.h"
-#include "i40iw_register.h"
-#include "i40iw_status.h"
-#include "i40iw_hmc.h"
-#include "i40iw_d.h"
-#include "i40iw_type.h"
-#include "i40iw_p.h"
-#include "i40iw_vf.h"
-#include "i40iw_virtchnl.h"
-
-/**
- * i40iw_find_sd_index_limit - finds segment descriptor index limit
- * @hmc_info: pointer to the HMC configuration information structure
- * @type: type of HMC resources we're searching
- * @index: starting index for the object
- * @cnt: number of objects we're trying to create
- * @sd_idx: pointer to return index of the segment descriptor in question
- * @sd_limit: pointer to return the maximum number of segment descriptors
- *
- * This function calculates the segment descriptor index and index limit
- * for the resource defined by i40iw_hmc_rsrc_type.
- */
-
-static inline void i40iw_find_sd_index_limit(struct i40iw_hmc_info *hmc_info,
- u32 type,
- u32 idx,
- u32 cnt,
- u32 *sd_idx,
- u32 *sd_limit)
-{
- u64 fpm_addr, fpm_limit;
-
- fpm_addr = hmc_info->hmc_obj[(type)].base +
- hmc_info->hmc_obj[type].size * idx;
- fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt;
- *sd_idx = (u32)(fpm_addr / I40IW_HMC_DIRECT_BP_SIZE);
- *sd_limit = (u32)((fpm_limit - 1) / I40IW_HMC_DIRECT_BP_SIZE);
- *sd_limit += 1;
-}
-
-/**
- * i40iw_find_pd_index_limit - finds page descriptor index limit
- * @hmc_info: pointer to the HMC configuration information struct
- * @type: HMC resource type we're examining
- * @idx: starting index for the object
- * @cnt: number of objects we're trying to create
- * @pd_index: pointer to return page descriptor index
- * @pd_limit: pointer to return page descriptor index limit
- *
- * Calculates the page descriptor index and index limit for the resource
- * defined by i40iw_hmc_rsrc_type.
- */
-
-static inline void i40iw_find_pd_index_limit(struct i40iw_hmc_info *hmc_info,
- u32 type,
- u32 idx,
- u32 cnt,
- u32 *pd_idx,
- u32 *pd_limit)
-{
- u64 fpm_adr, fpm_limit;
-
- fpm_adr = hmc_info->hmc_obj[type].base +
- hmc_info->hmc_obj[type].size * idx;
- fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt);
- *(pd_idx) = (u32)(fpm_adr / I40IW_HMC_PAGED_BP_SIZE);
- *(pd_limit) = (u32)((fpm_limit - 1) / I40IW_HMC_PAGED_BP_SIZE);
- *(pd_limit) += 1;
-}
-
-/**
- * i40iw_set_sd_entry - setup entry for sd programming
- * @pa: physical addr
- * @idx: sd index
- * @type: paged or direct sd
- * @entry: sd entry ptr
- */
-static inline void i40iw_set_sd_entry(u64 pa,
- u32 idx,
- enum i40iw_sd_entry_type type,
- struct update_sd_entry *entry)
-{
- entry->data = pa | (I40IW_HMC_MAX_BP_COUNT << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) |
- (((type == I40IW_SD_TYPE_PAGED) ? 0 : 1) <<
- I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT) |
- (1 << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT);
- entry->cmd = (idx | (1 << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | (1 << 15));
-}
-
-/**
- * i40iw_clr_sd_entry - setup entry for sd clear
- * @idx: sd index
- * @type: paged or direct sd
- * @entry: sd entry ptr
- */
-static inline void i40iw_clr_sd_entry(u32 idx, enum i40iw_sd_entry_type type,
- struct update_sd_entry *entry)
-{
- entry->data = (I40IW_HMC_MAX_BP_COUNT <<
- I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) |
- (((type == I40IW_SD_TYPE_PAGED) ? 0 : 1) <<
- I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT);
- entry->cmd = (idx | (1 << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | (1 << 15));
-}
-
-/**
- * i40iw_hmc_sd_one - setup 1 sd entry for cqp
- * @dev: pointer to the device structure
- * @hmc_fn_id: hmc's function id
- * @pa: physical addr
- * @sd_idx: sd index
- * @type: paged or direct sd
- * @setsd: flag to set or clear sd
- */
-enum i40iw_status_code i40iw_hmc_sd_one(struct i40iw_sc_dev *dev,
- u8 hmc_fn_id,
- u64 pa, u32 sd_idx,
- enum i40iw_sd_entry_type type,
- bool setsd)
-{
- struct i40iw_update_sds_info sdinfo;
-
- sdinfo.cnt = 1;
- sdinfo.hmc_fn_id = hmc_fn_id;
- if (setsd)
- i40iw_set_sd_entry(pa, sd_idx, type, sdinfo.entry);
- else
- i40iw_clr_sd_entry(sd_idx, type, sdinfo.entry);
-
- return dev->cqp->process_cqp_sds(dev, &sdinfo);
-}
-
-/**
- * i40iw_hmc_sd_grp - setup group od sd entries for cqp
- * @dev: pointer to the device structure
- * @hmc_info: pointer to the HMC configuration information struct
- * @sd_index: sd index
- * @sd_cnt: number of sd entries
- * @setsd: flag to set or clear sd
- */
-static enum i40iw_status_code i40iw_hmc_sd_grp(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_info *hmc_info,
- u32 sd_index,
- u32 sd_cnt,
- bool setsd)
-{
- struct i40iw_hmc_sd_entry *sd_entry;
- struct i40iw_update_sds_info sdinfo;
- u64 pa;
- u32 i;
- enum i40iw_status_code ret_code = 0;
-
- memset(&sdinfo, 0, sizeof(sdinfo));
- sdinfo.hmc_fn_id = hmc_info->hmc_fn_id;
- for (i = sd_index; i < sd_index + sd_cnt; i++) {
- sd_entry = &hmc_info->sd_table.sd_entry[i];
- if (!sd_entry ||
- (!sd_entry->valid && setsd) ||
- (sd_entry->valid && !setsd))
- continue;
- if (setsd) {
- pa = (sd_entry->entry_type == I40IW_SD_TYPE_PAGED) ?
- sd_entry->u.pd_table.pd_page_addr.pa :
- sd_entry->u.bp.addr.pa;
- i40iw_set_sd_entry(pa, i, sd_entry->entry_type,
- &sdinfo.entry[sdinfo.cnt]);
- } else {
- i40iw_clr_sd_entry(i, sd_entry->entry_type,
- &sdinfo.entry[sdinfo.cnt]);
- }
- sdinfo.cnt++;
- if (sdinfo.cnt == I40IW_MAX_SD_ENTRIES) {
- ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
- if (ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "i40iw_hmc_sd_grp: sd_programming failed err=%d\n",
- ret_code);
- return ret_code;
- }
- sdinfo.cnt = 0;
- }
- }
- if (sdinfo.cnt)
- ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
-
- return ret_code;
-}
-
-/**
- * i40iw_vfdev_from_fpm - return vf dev ptr for hmc function id
- * @dev: pointer to the device structure
- * @hmc_fn_id: hmc's function id
- */
-struct i40iw_vfdev *i40iw_vfdev_from_fpm(struct i40iw_sc_dev *dev, u8 hmc_fn_id)
-{
- struct i40iw_vfdev *vf_dev = NULL;
- u16 idx;
-
- for (idx = 0; idx < I40IW_MAX_PE_ENABLED_VF_COUNT; idx++) {
- if (dev->vf_dev[idx] &&
- ((u8)dev->vf_dev[idx]->pmf_index == hmc_fn_id)) {
- vf_dev = dev->vf_dev[idx];
- break;
- }
- }
- return vf_dev;
-}
-
-/**
- * i40iw_vf_hmcinfo_from_fpm - get ptr to hmc for func_id
- * @dev: pointer to the device structure
- * @hmc_fn_id: hmc's function id
- */
-struct i40iw_hmc_info *i40iw_vf_hmcinfo_from_fpm(struct i40iw_sc_dev *dev,
- u8 hmc_fn_id)
-{
- struct i40iw_hmc_info *hmc_info = NULL;
- u16 idx;
-
- for (idx = 0; idx < I40IW_MAX_PE_ENABLED_VF_COUNT; idx++) {
- if (dev->vf_dev[idx] &&
- ((u8)dev->vf_dev[idx]->pmf_index == hmc_fn_id)) {
- hmc_info = &dev->vf_dev[idx]->hmc_info;
- break;
- }
- }
- return hmc_info;
-}
-
-/**
- * i40iw_hmc_finish_add_sd_reg - program sd entries for objects
- * @dev: pointer to the device structure
- * @info: create obj info
- */
-static enum i40iw_status_code i40iw_hmc_finish_add_sd_reg(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_create_obj_info *info)
-{
- if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
- return I40IW_ERR_INVALID_HMC_OBJ_INDEX;
-
- if ((info->start_idx + info->count) >
- info->hmc_info->hmc_obj[info->rsrc_type].cnt)
- return I40IW_ERR_INVALID_HMC_OBJ_COUNT;
-
- if (!info->add_sd_cnt)
- return 0;
-
- return i40iw_hmc_sd_grp(dev, info->hmc_info,
- info->hmc_info->sd_indexes[0],
- info->add_sd_cnt, true);
-}
-
-/**
- * i40iw_create_iw_hmc_obj - allocate backing store for hmc objects
- * @dev: pointer to the device structure
- * @info: pointer to i40iw_hmc_iw_create_obj_info struct
- *
- * This will allocate memory for PDs and backing pages and populate
- * the sd and pd entries.
- */
-enum i40iw_status_code i40iw_sc_create_hmc_obj(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_create_obj_info *info)
-{
- struct i40iw_hmc_sd_entry *sd_entry;
- u32 sd_idx, sd_lmt;
- u32 pd_idx = 0, pd_lmt = 0;
- u32 pd_idx1 = 0, pd_lmt1 = 0;
- u32 i, j;
- bool pd_error = false;
- enum i40iw_status_code ret_code = 0;
-
- if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
- return I40IW_ERR_INVALID_HMC_OBJ_INDEX;
-
- if ((info->start_idx + info->count) >
- info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "%s: error type %u, start = %u, req cnt %u, cnt = %u\n",
- __func__, info->rsrc_type, info->start_idx, info->count,
- info->hmc_info->hmc_obj[info->rsrc_type].cnt);
- return I40IW_ERR_INVALID_HMC_OBJ_COUNT;
- }
-
- if (!dev->is_pf)
- return i40iw_vchnl_vf_add_hmc_objs(dev, info->rsrc_type, 0, info->count);
-
- i40iw_find_sd_index_limit(info->hmc_info, info->rsrc_type,
- info->start_idx, info->count,
- &sd_idx, &sd_lmt);
- if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
- sd_lmt > info->hmc_info->sd_table.sd_cnt) {
- return I40IW_ERR_INVALID_SD_INDEX;
- }
- i40iw_find_pd_index_limit(info->hmc_info, info->rsrc_type,
- info->start_idx, info->count, &pd_idx, &pd_lmt);
-
- for (j = sd_idx; j < sd_lmt; j++) {
- ret_code = i40iw_add_sd_table_entry(dev->hw, info->hmc_info,
- j,
- info->entry_type,
- I40IW_HMC_DIRECT_BP_SIZE);
- if (ret_code)
- goto exit_sd_error;
- sd_entry = &info->hmc_info->sd_table.sd_entry[j];
-
- if ((sd_entry->entry_type == I40IW_SD_TYPE_PAGED) &&
- ((dev->hmc_info == info->hmc_info) &&
- (info->rsrc_type != I40IW_HMC_IW_PBLE))) {
- pd_idx1 = max(pd_idx, (j * I40IW_HMC_MAX_BP_COUNT));
- pd_lmt1 = min(pd_lmt,
- (j + 1) * I40IW_HMC_MAX_BP_COUNT);
- for (i = pd_idx1; i < pd_lmt1; i++) {
- /* update the pd table entry */
- ret_code = i40iw_add_pd_table_entry(dev->hw, info->hmc_info,
- i, NULL);
- if (ret_code) {
- pd_error = true;
- break;
- }
- }
- if (pd_error) {
- while (i && (i > pd_idx1)) {
- i40iw_remove_pd_bp(dev->hw, info->hmc_info, (i - 1),
- info->is_pf);
- i--;
- }
- }
- }
- if (sd_entry->valid)
- continue;
-
- info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j;
- info->add_sd_cnt++;
- sd_entry->valid = true;
- }
- return i40iw_hmc_finish_add_sd_reg(dev, info);
-
-exit_sd_error:
- while (j && (j > sd_idx)) {
- sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1];
- switch (sd_entry->entry_type) {
- case I40IW_SD_TYPE_PAGED:
- pd_idx1 = max(pd_idx,
- (j - 1) * I40IW_HMC_MAX_BP_COUNT);
- pd_lmt1 = min(pd_lmt, (j * I40IW_HMC_MAX_BP_COUNT));
- for (i = pd_idx1; i < pd_lmt1; i++)
- i40iw_prep_remove_pd_page(info->hmc_info, i);
- break;
- case I40IW_SD_TYPE_DIRECT:
- i40iw_prep_remove_pd_page(info->hmc_info, (j - 1));
- break;
- default:
- ret_code = I40IW_ERR_INVALID_SD_TYPE;
- break;
- }
- j--;
- }
-
- return ret_code;
-}
-
-/**
- * i40iw_finish_del_sd_reg - delete sd entries for objects
- * @dev: pointer to the device structure
- * @info: dele obj info
- * @reset: true if called before reset
- */
-static enum i40iw_status_code i40iw_finish_del_sd_reg(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_del_obj_info *info,
- bool reset)
-{
- struct i40iw_hmc_sd_entry *sd_entry;
- enum i40iw_status_code ret_code = 0;
- u32 i, sd_idx;
- struct i40iw_dma_mem *mem;
-
- if (dev->is_pf && !reset)
- ret_code = i40iw_hmc_sd_grp(dev, info->hmc_info,
- info->hmc_info->sd_indexes[0],
- info->del_sd_cnt, false);
-
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error cqp sd sd_grp\n", __func__);
-
- for (i = 0; i < info->del_sd_cnt; i++) {
- sd_idx = info->hmc_info->sd_indexes[i];
- sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx];
- if (!sd_entry)
- continue;
- mem = (sd_entry->entry_type == I40IW_SD_TYPE_PAGED) ?
- &sd_entry->u.pd_table.pd_page_addr :
- &sd_entry->u.bp.addr;
-
- if (!mem || !mem->va)
- i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error cqp sd mem\n", __func__);
- else
- i40iw_free_dma_mem(dev->hw, mem);
- }
- return ret_code;
-}
-
-/**
- * i40iw_del_iw_hmc_obj - remove pe hmc objects
- * @dev: pointer to the device structure
- * @info: pointer to i40iw_hmc_del_obj_info struct
- * @reset: true if called before reset
- *
- * This will de-populate the SDs and PDs. It frees
- * the memory for PDS and backing storage. After this function is returned,
- * caller should deallocate memory allocated previously for
- * book-keeping information about PDs and backing storage.
- */
-enum i40iw_status_code i40iw_sc_del_hmc_obj(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_del_obj_info *info,
- bool reset)
-{
- struct i40iw_hmc_pd_table *pd_table;
- u32 sd_idx, sd_lmt;
- u32 pd_idx, pd_lmt, rel_pd_idx;
- u32 i, j;
- enum i40iw_status_code ret_code = 0;
-
- if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "%s: error start_idx[%04d] >= [type %04d].cnt[%04d]\n",
- __func__, info->start_idx, info->rsrc_type,
- info->hmc_info->hmc_obj[info->rsrc_type].cnt);
- return I40IW_ERR_INVALID_HMC_OBJ_INDEX;
- }
-
- if ((info->start_idx + info->count) >
- info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
- i40iw_debug(dev, I40IW_DEBUG_HMC,
- "%s: error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n",
- __func__, info->start_idx, info->count,
- info->rsrc_type,
- info->hmc_info->hmc_obj[info->rsrc_type].cnt);
- return I40IW_ERR_INVALID_HMC_OBJ_COUNT;
- }
- if (!dev->is_pf) {
- ret_code = i40iw_vchnl_vf_del_hmc_obj(dev, info->rsrc_type, 0,
- info->count);
- if (info->rsrc_type != I40IW_HMC_IW_PBLE)
- return ret_code;
- }
-
- i40iw_find_pd_index_limit(info->hmc_info, info->rsrc_type,
- info->start_idx, info->count, &pd_idx, &pd_lmt);
-
- for (j = pd_idx; j < pd_lmt; j++) {
- sd_idx = j / I40IW_HMC_PD_CNT_IN_SD;
-
- if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
- I40IW_SD_TYPE_PAGED)
- continue;
-
- rel_pd_idx = j % I40IW_HMC_PD_CNT_IN_SD;
- pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
- if (pd_table->pd_entry[rel_pd_idx].valid) {
- ret_code = i40iw_remove_pd_bp(dev->hw, info->hmc_info, j,
- info->is_pf);
- if (ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error\n", __func__);
- return ret_code;
- }
- }
- }
-
- i40iw_find_sd_index_limit(info->hmc_info, info->rsrc_type,
- info->start_idx, info->count, &sd_idx, &sd_lmt);
- if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
- sd_lmt > info->hmc_info->sd_table.sd_cnt) {
- i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: error invalid sd_idx\n", __func__);
- return I40IW_ERR_INVALID_SD_INDEX;
- }
-
- for (i = sd_idx; i < sd_lmt; i++) {
- if (!info->hmc_info->sd_table.sd_entry[i].valid)
- continue;
- switch (info->hmc_info->sd_table.sd_entry[i].entry_type) {
- case I40IW_SD_TYPE_DIRECT:
- ret_code = i40iw_prep_remove_sd_bp(info->hmc_info, i);
- if (!ret_code) {
- info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i;
- info->del_sd_cnt++;
- }
- break;
- case I40IW_SD_TYPE_PAGED:
- ret_code = i40iw_prep_remove_pd_page(info->hmc_info, i);
- if (!ret_code) {
- info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i;
- info->del_sd_cnt++;
- }
- break;
- default:
- break;
- }
- }
- return i40iw_finish_del_sd_reg(dev, info, reset);
-}
-
-/**
- * i40iw_add_sd_table_entry - Adds a segment descriptor to the table
- * @hw: pointer to our hw struct
- * @hmc_info: pointer to the HMC configuration information struct
- * @sd_index: segment descriptor index to manipulate
- * @type: what type of segment descriptor we're manipulating
- * @direct_mode_sz: size to alloc in direct mode
- */
-enum i40iw_status_code i40iw_add_sd_table_entry(struct i40iw_hw *hw,
- struct i40iw_hmc_info *hmc_info,
- u32 sd_index,
- enum i40iw_sd_entry_type type,
- u64 direct_mode_sz)
-{
- enum i40iw_status_code ret_code = 0;
- struct i40iw_hmc_sd_entry *sd_entry;
- bool dma_mem_alloc_done = false;
- struct i40iw_dma_mem mem;
- u64 alloc_len;
-
- sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
- if (!sd_entry->valid) {
- if (type == I40IW_SD_TYPE_PAGED)
- alloc_len = I40IW_HMC_PAGED_BP_SIZE;
- else
- alloc_len = direct_mode_sz;
-
- /* allocate a 4K pd page or 2M backing page */
- ret_code = i40iw_allocate_dma_mem(hw, &mem, alloc_len,
- I40IW_HMC_PD_BP_BUF_ALIGNMENT);
- if (ret_code)
- goto exit;
- dma_mem_alloc_done = true;
- if (type == I40IW_SD_TYPE_PAGED) {
- ret_code = i40iw_allocate_virt_mem(hw,
- &sd_entry->u.pd_table.pd_entry_virt_mem,
- sizeof(struct i40iw_hmc_pd_entry) * 512);
- if (ret_code)
- goto exit;
- sd_entry->u.pd_table.pd_entry = (struct i40iw_hmc_pd_entry *)
- sd_entry->u.pd_table.pd_entry_virt_mem.va;
-
- memcpy(&sd_entry->u.pd_table.pd_page_addr, &mem, sizeof(struct i40iw_dma_mem));
- } else {
- memcpy(&sd_entry->u.bp.addr, &mem, sizeof(struct i40iw_dma_mem));
- sd_entry->u.bp.sd_pd_index = sd_index;
- }
-
- hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
-
- I40IW_INC_SD_REFCNT(&hmc_info->sd_table);
- }
- if (sd_entry->entry_type == I40IW_SD_TYPE_DIRECT)
- I40IW_INC_BP_REFCNT(&sd_entry->u.bp);
-exit:
- if (ret_code)
- if (dma_mem_alloc_done)
- i40iw_free_dma_mem(hw, &mem);
-
- return ret_code;
-}
-
-/**
- * i40iw_add_pd_table_entry - Adds page descriptor to the specified table
- * @hw: pointer to our HW structure
- * @hmc_info: pointer to the HMC configuration information structure
- * @pd_index: which page descriptor index to manipulate
- * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
- *
- * This function:
- * 1. Initializes the pd entry
- * 2. Adds pd_entry in the pd_table
- * 3. Mark the entry valid in i40iw_hmc_pd_entry structure
- * 4. Initializes the pd_entry's ref count to 1
- * assumptions:
- * 1. The memory for pd should be pinned down, physically contiguous and
- * aligned on 4K boundary and zeroed memory.
- * 2. It should be 4K in size.
- */
-enum i40iw_status_code i40iw_add_pd_table_entry(struct i40iw_hw *hw,
- struct i40iw_hmc_info *hmc_info,
- u32 pd_index,
- struct i40iw_dma_mem *rsrc_pg)
-{
- enum i40iw_status_code ret_code = 0;
- struct i40iw_hmc_pd_table *pd_table;
- struct i40iw_hmc_pd_entry *pd_entry;
- struct i40iw_dma_mem mem;
- struct i40iw_dma_mem *page = &mem;
- u32 sd_idx, rel_pd_idx;
- u64 *pd_addr;
- u64 page_desc;
-
- if (pd_index / I40IW_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt)
- return I40IW_ERR_INVALID_PAGE_DESC_INDEX;
-
- sd_idx = (pd_index / I40IW_HMC_PD_CNT_IN_SD);
- if (hmc_info->sd_table.sd_entry[sd_idx].entry_type != I40IW_SD_TYPE_PAGED)
- return 0;
-
- rel_pd_idx = (pd_index % I40IW_HMC_PD_CNT_IN_SD);
- pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
- pd_entry = &pd_table->pd_entry[rel_pd_idx];
- if (!pd_entry->valid) {
- if (rsrc_pg) {
- pd_entry->rsrc_pg = true;
- page = rsrc_pg;
- } else {
- ret_code = i40iw_allocate_dma_mem(hw, page,
- I40IW_HMC_PAGED_BP_SIZE,
- I40IW_HMC_PD_BP_BUF_ALIGNMENT);
- if (ret_code)
- return ret_code;
- pd_entry->rsrc_pg = false;
- }
-
- memcpy(&pd_entry->bp.addr, page, sizeof(struct i40iw_dma_mem));
- pd_entry->bp.sd_pd_index = pd_index;
- pd_entry->bp.entry_type = I40IW_SD_TYPE_PAGED;
- page_desc = page->pa | 0x1;
-
- pd_addr = (u64 *)pd_table->pd_page_addr.va;
- pd_addr += rel_pd_idx;
-
- memcpy(pd_addr, &page_desc, sizeof(*pd_addr));
-
- pd_entry->sd_index = sd_idx;
- pd_entry->valid = true;
- I40IW_INC_PD_REFCNT(pd_table);
- if (hmc_info->hmc_fn_id < I40IW_FIRST_VF_FPM_ID)
- I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, rel_pd_idx);
- else if (hw->hmc.hmc_fn_id != hmc_info->hmc_fn_id)
- I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, rel_pd_idx,
- hmc_info->hmc_fn_id);
- }
- I40IW_INC_BP_REFCNT(&pd_entry->bp);
-
- return 0;
-}
-
-/**
- * i40iw_remove_pd_bp - remove a backing page from a page descriptor
- * @hw: pointer to our HW structure
- * @hmc_info: pointer to the HMC configuration information structure
- * @idx: the page index
- * @is_pf: distinguishes a VF from a PF
- *
- * This function:
- * 1. Marks the entry in pd table (for paged address mode) or in sd table
- * (for direct address mode) invalid.
- * 2. Write to register PMPDINV to invalidate the backing page in FV cache
- * 3. Decrement the ref count for the pd _entry
- * assumptions:
- * 1. Caller can deallocate the memory used by backing storage after this
- * function returns.
- */
-enum i40iw_status_code i40iw_remove_pd_bp(struct i40iw_hw *hw,
- struct i40iw_hmc_info *hmc_info,
- u32 idx,
- bool is_pf)
-{
- struct i40iw_hmc_pd_entry *pd_entry;
- struct i40iw_hmc_pd_table *pd_table;
- struct i40iw_hmc_sd_entry *sd_entry;
- u32 sd_idx, rel_pd_idx;
- struct i40iw_dma_mem *mem;
- u64 *pd_addr;
-
- sd_idx = idx / I40IW_HMC_PD_CNT_IN_SD;
- rel_pd_idx = idx % I40IW_HMC_PD_CNT_IN_SD;
- if (sd_idx >= hmc_info->sd_table.sd_cnt)
- return I40IW_ERR_INVALID_PAGE_DESC_INDEX;
-
- sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
- if (sd_entry->entry_type != I40IW_SD_TYPE_PAGED)
- return I40IW_ERR_INVALID_SD_TYPE;
-
- pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
- pd_entry = &pd_table->pd_entry[rel_pd_idx];
- I40IW_DEC_BP_REFCNT(&pd_entry->bp);
- if (pd_entry->bp.ref_cnt)
- return 0;
-
- pd_entry->valid = false;
- I40IW_DEC_PD_REFCNT(pd_table);
- pd_addr = (u64 *)pd_table->pd_page_addr.va;
- pd_addr += rel_pd_idx;
- memset(pd_addr, 0, sizeof(u64));
- if (is_pf)
- I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, idx);
- else
- I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, idx,
- hmc_info->hmc_fn_id);
-
- if (!pd_entry->rsrc_pg) {
- mem = &pd_entry->bp.addr;
- if (!mem || !mem->va)
- return I40IW_ERR_PARAM;
- i40iw_free_dma_mem(hw, mem);
- }
- if (!pd_table->ref_cnt)
- i40iw_free_virt_mem(hw, &pd_table->pd_entry_virt_mem);
-
- return 0;
-}
-
-/**
- * i40iw_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
- * @hmc_info: pointer to the HMC configuration information structure
- * @idx: the page index
- */
-enum i40iw_status_code i40iw_prep_remove_sd_bp(struct i40iw_hmc_info *hmc_info, u32 idx)
-{
- struct i40iw_hmc_sd_entry *sd_entry;
-
- sd_entry = &hmc_info->sd_table.sd_entry[idx];
- I40IW_DEC_BP_REFCNT(&sd_entry->u.bp);
- if (sd_entry->u.bp.ref_cnt)
- return I40IW_ERR_NOT_READY;
-
- I40IW_DEC_SD_REFCNT(&hmc_info->sd_table);
- sd_entry->valid = false;
-
- return 0;
-}
-
-/**
- * i40iw_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
- * @hmc_info: pointer to the HMC configuration information structure
- * @idx: segment descriptor index to find the relevant page descriptor
- */
-enum i40iw_status_code i40iw_prep_remove_pd_page(struct i40iw_hmc_info *hmc_info,
- u32 idx)
-{
- struct i40iw_hmc_sd_entry *sd_entry;
-
- sd_entry = &hmc_info->sd_table.sd_entry[idx];
-
- if (sd_entry->u.pd_table.ref_cnt)
- return I40IW_ERR_NOT_READY;
-
- sd_entry->valid = false;
- I40IW_DEC_SD_REFCNT(&hmc_info->sd_table);
-
- return 0;
-}
-
-/**
- * i40iw_pf_init_vfhmc -
- * @vf_cnt_array: array of cnt values of iwarp hmc objects
- * @vf_hmc_fn_id: hmc function id ofr vf driver
- * @dev: pointer to i40iw_dev struct
- *
- * Called by pf driver to initialize hmc_info for vf driver instance.
- */
-enum i40iw_status_code i40iw_pf_init_vfhmc(struct i40iw_sc_dev *dev,
- u8 vf_hmc_fn_id,
- u32 *vf_cnt_array)
-{
- struct i40iw_hmc_info *hmc_info;
- enum i40iw_status_code ret_code = 0;
- u32 i;
-
- if ((vf_hmc_fn_id < I40IW_FIRST_VF_FPM_ID) ||
- (vf_hmc_fn_id >= I40IW_FIRST_VF_FPM_ID +
- I40IW_MAX_PE_ENABLED_VF_COUNT)) {
- i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: invalid vf_hmc_fn_id 0x%x\n",
- __func__, vf_hmc_fn_id);
- return I40IW_ERR_INVALID_HMCFN_ID;
- }
-
- ret_code = i40iw_sc_init_iw_hmc(dev, vf_hmc_fn_id);
- if (ret_code)
- return ret_code;
-
- hmc_info = i40iw_vf_hmcinfo_from_fpm(dev, vf_hmc_fn_id);
-
- for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++)
- if (vf_cnt_array)
- hmc_info->hmc_obj[i].cnt =
- vf_cnt_array[i - I40IW_HMC_IW_QP];
- else
- hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
-
- return 0;
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_hmc.h b/drivers/infiniband/hw/i40iw/i40iw_hmc.h
deleted file mode 100644
index 4c3fdd875621..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_hmc.h
+++ /dev/null
@@ -1,241 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_HMC_H
-#define I40IW_HMC_H
-
-#include "i40iw_d.h"
-
-struct i40iw_hw;
-enum i40iw_status_code;
-
-#define I40IW_HMC_MAX_BP_COUNT 512
-#define I40IW_MAX_SD_ENTRIES 11
-#define I40IW_HW_DBG_HMC_INVALID_BP_MARK 0xCA
-
-#define I40IW_HMC_INFO_SIGNATURE 0x484D5347
-#define I40IW_HMC_PD_CNT_IN_SD 512
-#define I40IW_HMC_DIRECT_BP_SIZE 0x200000
-#define I40IW_HMC_MAX_SD_COUNT 4096
-#define I40IW_HMC_PAGED_BP_SIZE 4096
-#define I40IW_HMC_PD_BP_BUF_ALIGNMENT 4096
-#define I40IW_FIRST_VF_FPM_ID 16
-#define FPM_MULTIPLIER 1024
-
-#define I40IW_INC_SD_REFCNT(sd_table) ((sd_table)->ref_cnt++)
-#define I40IW_INC_PD_REFCNT(pd_table) ((pd_table)->ref_cnt++)
-#define I40IW_INC_BP_REFCNT(bp) ((bp)->ref_cnt++)
-
-#define I40IW_DEC_SD_REFCNT(sd_table) ((sd_table)->ref_cnt--)
-#define I40IW_DEC_PD_REFCNT(pd_table) ((pd_table)->ref_cnt--)
-#define I40IW_DEC_BP_REFCNT(bp) ((bp)->ref_cnt--)
-
-/**
- * I40IW_INVALIDATE_PF_HMC_PD - Invalidates the pd cache in the hardware
- * @hw: pointer to our hw struct
- * @sd_idx: segment descriptor index
- * @pd_idx: page descriptor index
- */
-#define I40IW_INVALIDATE_PF_HMC_PD(hw, sd_idx, pd_idx) \
- i40iw_wr32((hw), I40E_PFHMC_PDINV, \
- (((sd_idx) << I40E_PFHMC_PDINV_PMSDIDX_SHIFT) | \
- (0x1 << I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT) | \
- ((pd_idx) << I40E_PFHMC_PDINV_PMPDIDX_SHIFT)))
-
-/**
- * I40IW_INVALIDATE_VF_HMC_PD - Invalidates the pd cache in the hardware
- * @hw: pointer to our hw struct
- * @sd_idx: segment descriptor index
- * @pd_idx: page descriptor index
- * @hmc_fn_id: VF's function id
- */
-#define I40IW_INVALIDATE_VF_HMC_PD(hw, sd_idx, pd_idx, hmc_fn_id) \
- i40iw_wr32(hw, I40E_GLHMC_VFPDINV(hmc_fn_id - I40IW_FIRST_VF_FPM_ID), \
- ((sd_idx << I40E_PFHMC_PDINV_PMSDIDX_SHIFT) | \
- (pd_idx << I40E_PFHMC_PDINV_PMPDIDX_SHIFT)))
-
-struct i40iw_hmc_obj_info {
- u64 base;
- u32 max_cnt;
- u32 cnt;
- u64 size;
-};
-
-enum i40iw_sd_entry_type {
- I40IW_SD_TYPE_INVALID = 0,
- I40IW_SD_TYPE_PAGED = 1,
- I40IW_SD_TYPE_DIRECT = 2
-};
-
-struct i40iw_hmc_bp {
- enum i40iw_sd_entry_type entry_type;
- struct i40iw_dma_mem addr;
- u32 sd_pd_index;
- u32 ref_cnt;
-};
-
-struct i40iw_hmc_pd_entry {
- struct i40iw_hmc_bp bp;
- u32 sd_index;
- bool rsrc_pg;
- bool valid;
-};
-
-struct i40iw_hmc_pd_table {
- struct i40iw_dma_mem pd_page_addr;
- struct i40iw_hmc_pd_entry *pd_entry;
- struct i40iw_virt_mem pd_entry_virt_mem;
- u32 ref_cnt;
- u32 sd_index;
-};
-
-struct i40iw_hmc_sd_entry {
- enum i40iw_sd_entry_type entry_type;
- bool valid;
-
- union {
- struct i40iw_hmc_pd_table pd_table;
- struct i40iw_hmc_bp bp;
- } u;
-};
-
-struct i40iw_hmc_sd_table {
- struct i40iw_virt_mem addr;
- u32 sd_cnt;
- u32 ref_cnt;
- struct i40iw_hmc_sd_entry *sd_entry;
-};
-
-struct i40iw_hmc_info {
- u32 signature;
- u8 hmc_fn_id;
- u16 first_sd_index;
-
- struct i40iw_hmc_obj_info *hmc_obj;
- struct i40iw_virt_mem hmc_obj_virt_mem;
- struct i40iw_hmc_sd_table sd_table;
- u16 sd_indexes[I40IW_HMC_MAX_SD_COUNT];
-};
-
-struct update_sd_entry {
- u64 cmd;
- u64 data;
-};
-
-struct i40iw_update_sds_info {
- u32 cnt;
- u8 hmc_fn_id;
- struct update_sd_entry entry[I40IW_MAX_SD_ENTRIES];
-};
-
-struct i40iw_ccq_cqe_info;
-struct i40iw_hmc_fcn_info {
- void (*callback_fcn)(struct i40iw_sc_dev *, void *,
- struct i40iw_ccq_cqe_info *);
- void *cqp_callback_param;
- u32 vf_id;
- u16 iw_vf_idx;
- bool free_fcn;
-};
-
-enum i40iw_hmc_rsrc_type {
- I40IW_HMC_IW_QP = 0,
- I40IW_HMC_IW_CQ = 1,
- I40IW_HMC_IW_SRQ = 2,
- I40IW_HMC_IW_HTE = 3,
- I40IW_HMC_IW_ARP = 4,
- I40IW_HMC_IW_APBVT_ENTRY = 5,
- I40IW_HMC_IW_MR = 6,
- I40IW_HMC_IW_XF = 7,
- I40IW_HMC_IW_XFFL = 8,
- I40IW_HMC_IW_Q1 = 9,
- I40IW_HMC_IW_Q1FL = 10,
- I40IW_HMC_IW_TIMER = 11,
- I40IW_HMC_IW_FSIMC = 12,
- I40IW_HMC_IW_FSIAV = 13,
- I40IW_HMC_IW_PBLE = 14,
- I40IW_HMC_IW_MAX = 15,
-};
-
-struct i40iw_hmc_create_obj_info {
- struct i40iw_hmc_info *hmc_info;
- struct i40iw_virt_mem add_sd_virt_mem;
- u32 rsrc_type;
- u32 start_idx;
- u32 count;
- u32 add_sd_cnt;
- enum i40iw_sd_entry_type entry_type;
- bool is_pf;
-};
-
-struct i40iw_hmc_del_obj_info {
- struct i40iw_hmc_info *hmc_info;
- struct i40iw_virt_mem del_sd_virt_mem;
- u32 rsrc_type;
- u32 start_idx;
- u32 count;
- u32 del_sd_cnt;
- bool is_pf;
-};
-
-enum i40iw_status_code i40iw_copy_dma_mem(struct i40iw_hw *hw, void *dest_buf,
- struct i40iw_dma_mem *src_mem, u64 src_offset, u64 size);
-enum i40iw_status_code i40iw_sc_create_hmc_obj(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_create_obj_info *info);
-enum i40iw_status_code i40iw_sc_del_hmc_obj(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_del_obj_info *info,
- bool reset);
-enum i40iw_status_code i40iw_hmc_sd_one(struct i40iw_sc_dev *dev, u8 hmc_fn_id,
- u64 pa, u32 sd_idx, enum i40iw_sd_entry_type type,
- bool setsd);
-enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev,
- struct i40iw_update_sds_info *info);
-struct i40iw_vfdev *i40iw_vfdev_from_fpm(struct i40iw_sc_dev *dev, u8 hmc_fn_id);
-struct i40iw_hmc_info *i40iw_vf_hmcinfo_from_fpm(struct i40iw_sc_dev *dev,
- u8 hmc_fn_id);
-enum i40iw_status_code i40iw_add_sd_table_entry(struct i40iw_hw *hw,
- struct i40iw_hmc_info *hmc_info, u32 sd_index,
- enum i40iw_sd_entry_type type, u64 direct_mode_sz);
-enum i40iw_status_code i40iw_add_pd_table_entry(struct i40iw_hw *hw,
- struct i40iw_hmc_info *hmc_info, u32 pd_index,
- struct i40iw_dma_mem *rsrc_pg);
-enum i40iw_status_code i40iw_remove_pd_bp(struct i40iw_hw *hw,
- struct i40iw_hmc_info *hmc_info, u32 idx, bool is_pf);
-enum i40iw_status_code i40iw_prep_remove_sd_bp(struct i40iw_hmc_info *hmc_info, u32 idx);
-enum i40iw_status_code i40iw_prep_remove_pd_page(struct i40iw_hmc_info *hmc_info, u32 idx);
-
-#define ENTER_SHARED_FUNCTION()
-#define EXIT_SHARED_FUNCTION()
-
-#endif /* I40IW_HMC_H */
diff --git a/drivers/infiniband/hw/i40iw/i40iw_hw.c b/drivers/infiniband/hw/i40iw/i40iw_hw.c
deleted file mode 100644
index ae8b97c30665..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_hw.c
+++ /dev/null
@@ -1,852 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/if_vlan.h>
-
-#include "i40iw.h"
-
-/**
- * i40iw_initialize_hw_resources - initialize hw resource during open
- * @iwdev: iwarp device
- */
-u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev)
-{
- unsigned long num_pds;
- u32 resources_size;
- u32 max_mr;
- u32 max_qp;
- u32 max_cq;
- u32 arp_table_size;
- u32 mrdrvbits;
- void *resource_ptr;
-
- max_qp = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt;
- max_cq = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
- max_mr = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt;
- arp_table_size = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt;
- iwdev->max_cqe = 0xFFFFF;
- num_pds = I40IW_MAX_PDS;
- resources_size = sizeof(struct i40iw_arp_entry) * arp_table_size;
- resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
- resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
- resources_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
- resources_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
- resources_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
- resources_size += sizeof(struct i40iw_qp **) * max_qp;
- iwdev->mem_resources = kzalloc(resources_size, GFP_KERNEL);
-
- if (!iwdev->mem_resources)
- return -ENOMEM;
-
- iwdev->max_qp = max_qp;
- iwdev->max_mr = max_mr;
- iwdev->max_cq = max_cq;
- iwdev->max_pd = num_pds;
- iwdev->arp_table_size = arp_table_size;
- iwdev->arp_table = (struct i40iw_arp_entry *)iwdev->mem_resources;
- resource_ptr = iwdev->mem_resources + (sizeof(struct i40iw_arp_entry) * arp_table_size);
-
- iwdev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
- IB_DEVICE_MEM_WINDOW | IB_DEVICE_MEM_MGT_EXTENSIONS;
-
- iwdev->allocated_qps = resource_ptr;
- iwdev->allocated_cqs = &iwdev->allocated_qps[BITS_TO_LONGS(max_qp)];
- iwdev->allocated_mrs = &iwdev->allocated_cqs[BITS_TO_LONGS(max_cq)];
- iwdev->allocated_pds = &iwdev->allocated_mrs[BITS_TO_LONGS(max_mr)];
- iwdev->allocated_arps = &iwdev->allocated_pds[BITS_TO_LONGS(num_pds)];
- iwdev->qp_table = (struct i40iw_qp **)(&iwdev->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
- set_bit(0, iwdev->allocated_mrs);
- set_bit(0, iwdev->allocated_qps);
- set_bit(0, iwdev->allocated_cqs);
- set_bit(0, iwdev->allocated_pds);
- set_bit(0, iwdev->allocated_arps);
-
- /* Following for ILQ/IEQ */
- set_bit(1, iwdev->allocated_qps);
- set_bit(1, iwdev->allocated_cqs);
- set_bit(1, iwdev->allocated_pds);
- set_bit(2, iwdev->allocated_cqs);
- set_bit(2, iwdev->allocated_pds);
-
- spin_lock_init(&iwdev->resource_lock);
- spin_lock_init(&iwdev->qptable_lock);
- /* stag index mask has a minimum of 14 bits */
- mrdrvbits = 24 - max(get_count_order(iwdev->max_mr), 14);
- iwdev->mr_stagmask = ~(((1 << mrdrvbits) - 1) << (32 - mrdrvbits));
- return 0;
-}
-
-/**
- * i40iw_cqp_ce_handler - handle cqp completions
- * @iwdev: iwarp device
- * @arm: flag to arm after completions
- * @cq: cq for cqp completions
- */
-static void i40iw_cqp_ce_handler(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq, bool arm)
-{
- struct i40iw_cqp_request *cqp_request;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- u32 cqe_count = 0;
- struct i40iw_ccq_cqe_info info;
- int ret;
-
- do {
- memset(&info, 0, sizeof(info));
- ret = dev->ccq_ops->ccq_get_cqe_info(cq, &info);
- if (ret)
- break;
- cqp_request = (struct i40iw_cqp_request *)(unsigned long)info.scratch;
- if (info.error)
- i40iw_pr_err("opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
- info.op_code, info.maj_err_code, info.min_err_code);
- if (cqp_request) {
- cqp_request->compl_info.maj_err_code = info.maj_err_code;
- cqp_request->compl_info.min_err_code = info.min_err_code;
- cqp_request->compl_info.op_ret_val = info.op_ret_val;
- cqp_request->compl_info.error = info.error;
-
- if (cqp_request->waiting) {
- cqp_request->request_done = true;
- wake_up(&cqp_request->waitq);
- i40iw_put_cqp_request(&iwdev->cqp, cqp_request);
- } else {
- if (cqp_request->callback_fcn)
- cqp_request->callback_fcn(cqp_request, 1);
- i40iw_put_cqp_request(&iwdev->cqp, cqp_request);
- }
- }
-
- cqe_count++;
- } while (1);
-
- if (arm && cqe_count) {
- i40iw_process_bh(dev);
- dev->ccq_ops->ccq_arm(cq);
- }
-}
-
-/**
- * i40iw_iwarp_ce_handler - handle iwarp completions
- * @iwdev: iwarp device
- * @iwcp: iwarp cq receiving event
- */
-static void i40iw_iwarp_ce_handler(struct i40iw_device *iwdev,
- struct i40iw_sc_cq *iwcq)
-{
- struct i40iw_cq *i40iwcq = iwcq->back_cq;
-
- if (i40iwcq->ibcq.comp_handler)
- i40iwcq->ibcq.comp_handler(&i40iwcq->ibcq,
- i40iwcq->ibcq.cq_context);
-}
-
-/**
- * i40iw_puda_ce_handler - handle puda completion events
- * @iwdev: iwarp device
- * @cq: puda completion q for event
- */
-static void i40iw_puda_ce_handler(struct i40iw_device *iwdev,
- struct i40iw_sc_cq *cq)
-{
- struct i40iw_sc_dev *dev = (struct i40iw_sc_dev *)&iwdev->sc_dev;
- enum i40iw_status_code status;
- u32 compl_error;
-
- do {
- status = i40iw_puda_poll_completion(dev, cq, &compl_error);
- if (status == I40IW_ERR_QUEUE_EMPTY)
- break;
- if (status) {
- i40iw_pr_err("puda status = %d\n", status);
- break;
- }
- if (compl_error) {
- i40iw_pr_err("puda compl_err =0x%x\n", compl_error);
- break;
- }
- } while (1);
-
- dev->ccq_ops->ccq_arm(cq);
-}
-
-/**
- * i40iw_process_ceq - handle ceq for completions
- * @iwdev: iwarp device
- * @ceq: ceq having cq for completion
- */
-void i40iw_process_ceq(struct i40iw_device *iwdev, struct i40iw_ceq *ceq)
-{
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_sc_ceq *sc_ceq;
- struct i40iw_sc_cq *cq;
- bool arm = true;
-
- sc_ceq = &ceq->sc_ceq;
- do {
- cq = dev->ceq_ops->process_ceq(dev, sc_ceq);
- if (!cq)
- break;
-
- if (cq->cq_type == I40IW_CQ_TYPE_CQP)
- i40iw_cqp_ce_handler(iwdev, cq, arm);
- else if (cq->cq_type == I40IW_CQ_TYPE_IWARP)
- i40iw_iwarp_ce_handler(iwdev, cq);
- else if ((cq->cq_type == I40IW_CQ_TYPE_ILQ) ||
- (cq->cq_type == I40IW_CQ_TYPE_IEQ))
- i40iw_puda_ce_handler(iwdev, cq);
- } while (1);
-}
-
-/**
- * i40iw_next_iw_state - modify qp state
- * @iwqp: iwarp qp to modify
- * @state: next state for qp
- * @del_hash: del hash
- * @term: term message
- * @termlen: length of term message
- */
-void i40iw_next_iw_state(struct i40iw_qp *iwqp,
- u8 state,
- u8 del_hash,
- u8 term,
- u8 termlen)
-{
- struct i40iw_modify_qp_info info;
-
- memset(&info, 0, sizeof(info));
- info.next_iwarp_state = state;
- info.remove_hash_idx = del_hash;
- info.cq_num_valid = true;
- info.arp_cache_idx_valid = true;
- info.dont_send_term = true;
- info.dont_send_fin = true;
- info.termlen = termlen;
-
- if (term & I40IWQP_TERM_SEND_TERM_ONLY)
- info.dont_send_term = false;
- if (term & I40IWQP_TERM_SEND_FIN_ONLY)
- info.dont_send_fin = false;
- if (iwqp->sc_qp.term_flags && (state == I40IW_QP_STATE_ERROR))
- info.reset_tcp_conn = true;
- iwqp->hw_iwarp_state = state;
- i40iw_hw_modify_qp(iwqp->iwdev, iwqp, &info, 0);
-}
-
-/**
- * i40iw_process_aeq - handle aeq events
- * @iwdev: iwarp device
- */
-void i40iw_process_aeq(struct i40iw_device *iwdev)
-{
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_aeq *aeq = &iwdev->aeq;
- struct i40iw_sc_aeq *sc_aeq = &aeq->sc_aeq;
- struct i40iw_aeqe_info aeinfo;
- struct i40iw_aeqe_info *info = &aeinfo;
- int ret;
- struct i40iw_qp *iwqp = NULL;
- struct i40iw_sc_cq *cq = NULL;
- struct i40iw_cq *iwcq = NULL;
- struct i40iw_sc_qp *qp = NULL;
- struct i40iw_qp_host_ctx_info *ctx_info = NULL;
- unsigned long flags;
-
- u32 aeqcnt = 0;
-
- if (!sc_aeq->size)
- return;
-
- do {
- memset(info, 0, sizeof(*info));
- ret = dev->aeq_ops->get_next_aeqe(sc_aeq, info);
- if (ret)
- break;
-
- aeqcnt++;
- i40iw_debug(dev, I40IW_DEBUG_AEQ,
- "%s ae_id = 0x%x bool qp=%d qp_id = %d\n",
- __func__, info->ae_id, info->qp, info->qp_cq_id);
- if (info->qp) {
- spin_lock_irqsave(&iwdev->qptable_lock, flags);
- iwqp = iwdev->qp_table[info->qp_cq_id];
- if (!iwqp) {
- spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
- i40iw_debug(dev, I40IW_DEBUG_AEQ,
- "%s qp_id %d is already freed\n",
- __func__, info->qp_cq_id);
- continue;
- }
- i40iw_add_ref(&iwqp->ibqp);
- spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
- qp = &iwqp->sc_qp;
- spin_lock_irqsave(&iwqp->lock, flags);
- iwqp->hw_tcp_state = info->tcp_state;
- iwqp->hw_iwarp_state = info->iwarp_state;
- iwqp->last_aeq = info->ae_id;
- spin_unlock_irqrestore(&iwqp->lock, flags);
- ctx_info = &iwqp->ctx_info;
- ctx_info->err_rq_idx_valid = true;
- } else {
- if (info->ae_id != I40IW_AE_CQ_OPERATION_ERROR)
- continue;
- }
-
- switch (info->ae_id) {
- case I40IW_AE_LLP_FIN_RECEIVED:
- if (qp->term_flags)
- break;
- if (atomic_inc_return(&iwqp->close_timer_started) == 1) {
- iwqp->hw_tcp_state = I40IW_TCP_STATE_CLOSE_WAIT;
- if ((iwqp->hw_tcp_state == I40IW_TCP_STATE_CLOSE_WAIT) &&
- (iwqp->ibqp_state == IB_QPS_RTS)) {
- i40iw_next_iw_state(iwqp,
- I40IW_QP_STATE_CLOSING, 0, 0, 0);
- i40iw_cm_disconn(iwqp);
- }
- iwqp->cm_id->add_ref(iwqp->cm_id);
- i40iw_schedule_cm_timer(iwqp->cm_node,
- (struct i40iw_puda_buf *)iwqp,
- I40IW_TIMER_TYPE_CLOSE, 1, 0);
- }
- break;
- case I40IW_AE_LLP_CLOSE_COMPLETE:
- if (qp->term_flags)
- i40iw_terminate_done(qp, 0);
- else
- i40iw_cm_disconn(iwqp);
- break;
- case I40IW_AE_BAD_CLOSE:
- /* fall through */
- case I40IW_AE_RESET_SENT:
- i40iw_next_iw_state(iwqp, I40IW_QP_STATE_ERROR, 1, 0, 0);
- i40iw_cm_disconn(iwqp);
- break;
- case I40IW_AE_LLP_CONNECTION_RESET:
- if (atomic_read(&iwqp->close_timer_started))
- break;
- i40iw_cm_disconn(iwqp);
- break;
- case I40IW_AE_QP_SUSPEND_COMPLETE:
- i40iw_qp_suspend_resume(dev, &iwqp->sc_qp, false);
- break;
- case I40IW_AE_TERMINATE_SENT:
- i40iw_terminate_send_fin(qp);
- break;
- case I40IW_AE_LLP_TERMINATE_RECEIVED:
- i40iw_terminate_received(qp, info);
- break;
- case I40IW_AE_CQ_OPERATION_ERROR:
- i40iw_pr_err("Processing an iWARP related AE for CQ misc = 0x%04X\n",
- info->ae_id);
- cq = (struct i40iw_sc_cq *)(unsigned long)info->compl_ctx;
- iwcq = (struct i40iw_cq *)cq->back_cq;
-
- if (iwcq->ibcq.event_handler) {
- struct ib_event ibevent;
-
- ibevent.device = iwcq->ibcq.device;
- ibevent.event = IB_EVENT_CQ_ERR;
- ibevent.element.cq = &iwcq->ibcq;
- iwcq->ibcq.event_handler(&ibevent, iwcq->ibcq.cq_context);
- }
- break;
- case I40IW_AE_LLP_DOUBT_REACHABILITY:
- break;
- case I40IW_AE_PRIV_OPERATION_DENIED:
- case I40IW_AE_STAG_ZERO_INVALID:
- case I40IW_AE_IB_RREQ_AND_Q1_FULL:
- case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
- case I40IW_AE_DDP_UBE_INVALID_MO:
- case I40IW_AE_DDP_UBE_INVALID_QN:
- case I40IW_AE_DDP_NO_L_BIT:
- case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
- case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
- case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST:
- case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
- case I40IW_AE_INVALID_ARP_ENTRY:
- case I40IW_AE_INVALID_TCP_OPTION_RCVD:
- case I40IW_AE_STALE_ARP_ENTRY:
- case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
- case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
- case I40IW_AE_LLP_SYN_RECEIVED:
- case I40IW_AE_LLP_TOO_MANY_RETRIES:
- case I40IW_AE_LCE_QP_CATASTROPHIC:
- case I40IW_AE_LCE_FUNCTION_CATASTROPHIC:
- case I40IW_AE_LCE_CQ_CATASTROPHIC:
- case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG:
- case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT:
- ctx_info->err_rq_idx_valid = false;
- /* fall through */
- default:
- if (!info->sq && ctx_info->err_rq_idx_valid) {
- ctx_info->err_rq_idx = info->wqe_idx;
- ctx_info->tcp_info_valid = false;
- ctx_info->iwarp_info_valid = false;
- ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
- iwqp->host_ctx.va,
- ctx_info);
- }
- i40iw_terminate_connection(qp, info);
- break;
- }
- if (info->qp)
- i40iw_rem_ref(&iwqp->ibqp);
- } while (1);
-
- if (aeqcnt)
- dev->aeq_ops->repost_aeq_entries(dev, aeqcnt);
-}
-
-/**
- * i40iw_cqp_manage_abvpt_cmd - send cqp command manage abpvt
- * @iwdev: iwarp device
- * @accel_local_port: port for apbvt
- * @add_port: add or delete port
- */
-static enum i40iw_status_code
-i40iw_cqp_manage_abvpt_cmd(struct i40iw_device *iwdev,
- u16 accel_local_port,
- bool add_port)
-{
- struct i40iw_apbvt_info *info;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- enum i40iw_status_code status;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, add_port);
- if (!cqp_request)
- return I40IW_ERR_NO_MEMORY;
-
- cqp_info = &cqp_request->info;
- info = &cqp_info->in.u.manage_apbvt_entry.info;
-
- memset(info, 0, sizeof(*info));
- info->add = add_port;
- info->port = cpu_to_le16(accel_local_port);
-
- cqp_info->cqp_cmd = OP_MANAGE_APBVT_ENTRY;
- cqp_info->post_sq = 1;
- cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->cqp.sc_cqp;
- cqp_info->in.u.manage_apbvt_entry.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Manage APBVT entry fail");
-
- return status;
-}
-
-/**
- * i40iw_manage_apbvt - add or delete tcp port
- * @iwdev: iwarp device
- * @accel_local_port: port for apbvt
- * @add_port: add or delete port
- */
-enum i40iw_status_code i40iw_manage_apbvt(struct i40iw_device *iwdev,
- u16 accel_local_port,
- bool add_port)
-{
- struct i40iw_cm_core *cm_core = &iwdev->cm_core;
- enum i40iw_status_code status;
- unsigned long flags;
- bool in_use;
-
- /* apbvt_lock is held across CQP delete APBVT OP (non-waiting) to
- * protect against race where add APBVT CQP can race ahead of the delete
- * APBVT for same port.
- */
- if (add_port) {
- spin_lock_irqsave(&cm_core->apbvt_lock, flags);
- in_use = __test_and_set_bit(accel_local_port,
- cm_core->ports_in_use);
- spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
- if (in_use)
- return 0;
- return i40iw_cqp_manage_abvpt_cmd(iwdev, accel_local_port,
- true);
- } else {
- spin_lock_irqsave(&cm_core->apbvt_lock, flags);
- in_use = i40iw_port_in_use(cm_core, accel_local_port);
- if (in_use) {
- spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
- return 0;
- }
- __clear_bit(accel_local_port, cm_core->ports_in_use);
- status = i40iw_cqp_manage_abvpt_cmd(iwdev, accel_local_port,
- false);
- spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
- return status;
- }
-}
-
-/**
- * i40iw_manage_arp_cache - manage hw arp cache
- * @iwdev: iwarp device
- * @mac_addr: mac address ptr
- * @ip_addr: ip addr for arp cache
- * @action: add, delete or modify
- */
-void i40iw_manage_arp_cache(struct i40iw_device *iwdev,
- unsigned char *mac_addr,
- u32 *ip_addr,
- bool ipv4,
- u32 action)
-{
- struct i40iw_add_arp_cache_entry_info *info;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- int arp_index;
-
- arp_index = i40iw_arp_table(iwdev, ip_addr, ipv4, mac_addr, action);
- if (arp_index < 0)
- return;
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false);
- if (!cqp_request)
- return;
-
- cqp_info = &cqp_request->info;
- if (action == I40IW_ARP_ADD) {
- cqp_info->cqp_cmd = OP_ADD_ARP_CACHE_ENTRY;
- info = &cqp_info->in.u.add_arp_cache_entry.info;
- memset(info, 0, sizeof(*info));
- info->arp_index = cpu_to_le16((u16)arp_index);
- info->permanent = true;
- ether_addr_copy(info->mac_addr, mac_addr);
- cqp_info->in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request;
- cqp_info->in.u.add_arp_cache_entry.cqp = &iwdev->cqp.sc_cqp;
- } else {
- cqp_info->cqp_cmd = OP_DELETE_ARP_CACHE_ENTRY;
- cqp_info->in.u.del_arp_cache_entry.scratch = (uintptr_t)cqp_request;
- cqp_info->in.u.del_arp_cache_entry.cqp = &iwdev->cqp.sc_cqp;
- cqp_info->in.u.del_arp_cache_entry.arp_index = arp_index;
- }
-
- cqp_info->in.u.add_arp_cache_entry.cqp = &iwdev->cqp.sc_cqp;
- cqp_info->in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request;
- cqp_info->post_sq = 1;
- if (i40iw_handle_cqp_op(iwdev, cqp_request))
- i40iw_pr_err("CQP-OP Add/Del Arp Cache entry fail");
-}
-
-/**
- * i40iw_send_syn_cqp_callback - do syn/ack after qhash
- * @cqp_request: qhash cqp completion
- * @send_ack: flag send ack
- */
-static void i40iw_send_syn_cqp_callback(struct i40iw_cqp_request *cqp_request, u32 send_ack)
-{
- i40iw_send_syn(cqp_request->param, send_ack);
-}
-
-/**
- * i40iw_manage_qhash - add or modify qhash
- * @iwdev: iwarp device
- * @cminfo: cm info for qhash
- * @etype: type (syn or quad)
- * @mtype: type of qhash
- * @cmnode: cmnode associated with connection
- * @wait: wait for completion
- * @user_pri:user pri of the connection
- */
-enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev,
- struct i40iw_cm_info *cminfo,
- enum i40iw_quad_entry_type etype,
- enum i40iw_quad_hash_manage_type mtype,
- void *cmnode,
- bool wait)
-{
- struct i40iw_qhash_table_info *info;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_sc_vsi *vsi = &iwdev->vsi;
- enum i40iw_status_code status;
- struct i40iw_cqp *iwcqp = &iwdev->cqp;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
-
- cqp_request = i40iw_get_cqp_request(iwcqp, wait);
- if (!cqp_request)
- return I40IW_ERR_NO_MEMORY;
- cqp_info = &cqp_request->info;
- info = &cqp_info->in.u.manage_qhash_table_entry.info;
- memset(info, 0, sizeof(*info));
-
- info->vsi = &iwdev->vsi;
- info->manage = mtype;
- info->entry_type = etype;
- if (cminfo->vlan_id != 0xFFFF) {
- info->vlan_valid = true;
- info->vlan_id = cpu_to_le16(cminfo->vlan_id);
- } else {
- info->vlan_valid = false;
- }
-
- info->ipv4_valid = cminfo->ipv4;
- info->user_pri = cminfo->user_pri;
- ether_addr_copy(info->mac_addr, iwdev->netdev->dev_addr);
- info->qp_num = cpu_to_le32(vsi->ilq->qp_id);
- info->dest_port = cpu_to_le16(cminfo->loc_port);
- info->dest_ip[0] = cpu_to_le32(cminfo->loc_addr[0]);
- info->dest_ip[1] = cpu_to_le32(cminfo->loc_addr[1]);
- info->dest_ip[2] = cpu_to_le32(cminfo->loc_addr[2]);
- info->dest_ip[3] = cpu_to_le32(cminfo->loc_addr[3]);
- if (etype == I40IW_QHASH_TYPE_TCP_ESTABLISHED) {
- info->src_port = cpu_to_le16(cminfo->rem_port);
- info->src_ip[0] = cpu_to_le32(cminfo->rem_addr[0]);
- info->src_ip[1] = cpu_to_le32(cminfo->rem_addr[1]);
- info->src_ip[2] = cpu_to_le32(cminfo->rem_addr[2]);
- info->src_ip[3] = cpu_to_le32(cminfo->rem_addr[3]);
- }
- if (cmnode) {
- cqp_request->callback_fcn = i40iw_send_syn_cqp_callback;
- cqp_request->param = (void *)cmnode;
- }
-
- if (info->ipv4_valid)
- i40iw_debug(dev, I40IW_DEBUG_CM,
- "%s:%s IP=%pI4, port=%d, mac=%pM, vlan_id=%d\n",
- __func__, (!mtype) ? "DELETE" : "ADD",
- info->dest_ip,
- info->dest_port, info->mac_addr, cminfo->vlan_id);
- else
- i40iw_debug(dev, I40IW_DEBUG_CM,
- "%s:%s IP=%pI6, port=%d, mac=%pM, vlan_id=%d\n",
- __func__, (!mtype) ? "DELETE" : "ADD",
- info->dest_ip,
- info->dest_port, info->mac_addr, cminfo->vlan_id);
- cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->cqp.sc_cqp;
- cqp_info->in.u.manage_qhash_table_entry.scratch = (uintptr_t)cqp_request;
- cqp_info->cqp_cmd = OP_MANAGE_QHASH_TABLE_ENTRY;
- cqp_info->post_sq = 1;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Manage Qhash Entry fail");
- return status;
-}
-
-/**
- * i40iw_hw_flush_wqes - flush qp's wqe
- * @iwdev: iwarp device
- * @qp: hardware control qp
- * @info: info for flush
- * @wait: flag wait for completion
- */
-enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev,
- struct i40iw_sc_qp *qp,
- struct i40iw_qp_flush_info *info,
- bool wait)
-{
- enum i40iw_status_code status;
- struct i40iw_qp_flush_info *hw_info;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- struct i40iw_qp *iwqp = (struct i40iw_qp *)qp->back_qp;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait);
- if (!cqp_request)
- return I40IW_ERR_NO_MEMORY;
-
- cqp_info = &cqp_request->info;
- hw_info = &cqp_request->info.in.u.qp_flush_wqes.info;
- memcpy(hw_info, info, sizeof(*hw_info));
-
- cqp_info->cqp_cmd = OP_QP_FLUSH_WQES;
- cqp_info->post_sq = 1;
- cqp_info->in.u.qp_flush_wqes.qp = qp;
- cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status) {
- i40iw_pr_err("CQP-OP Flush WQE's fail");
- complete(&iwqp->sq_drained);
- complete(&iwqp->rq_drained);
- return status;
- }
- if (!cqp_request->compl_info.maj_err_code) {
- switch (cqp_request->compl_info.min_err_code) {
- case I40IW_CQP_COMPL_RQ_WQE_FLUSHED:
- complete(&iwqp->sq_drained);
- break;
- case I40IW_CQP_COMPL_SQ_WQE_FLUSHED:
- complete(&iwqp->rq_drained);
- break;
- case I40IW_CQP_COMPL_RQ_SQ_WQE_FLUSHED:
- break;
- default:
- complete(&iwqp->sq_drained);
- complete(&iwqp->rq_drained);
- break;
- }
- }
-
- return 0;
-}
-
-/**
- * i40iw_gen_ae - generate AE
- * @iwdev: iwarp device
- * @qp: qp associated with AE
- * @info: info for ae
- * @wait: wait for completion
- */
-void i40iw_gen_ae(struct i40iw_device *iwdev,
- struct i40iw_sc_qp *qp,
- struct i40iw_gen_ae_info *info,
- bool wait)
-{
- struct i40iw_gen_ae_info *ae_info;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait);
- if (!cqp_request)
- return;
-
- cqp_info = &cqp_request->info;
- ae_info = &cqp_request->info.in.u.gen_ae.info;
- memcpy(ae_info, info, sizeof(*ae_info));
-
- cqp_info->cqp_cmd = OP_GEN_AE;
- cqp_info->post_sq = 1;
- cqp_info->in.u.gen_ae.qp = qp;
- cqp_info->in.u.gen_ae.scratch = (uintptr_t)cqp_request;
- if (i40iw_handle_cqp_op(iwdev, cqp_request))
- i40iw_pr_err("CQP OP failed attempting to generate ae_code=0x%x\n",
- info->ae_code);
-}
-
-/**
- * i40iw_hw_manage_vf_pble_bp - manage vf pbles
- * @iwdev: iwarp device
- * @info: info for managing pble
- * @wait: flag wait for completion
- */
-enum i40iw_status_code i40iw_hw_manage_vf_pble_bp(struct i40iw_device *iwdev,
- struct i40iw_manage_vf_pble_info *info,
- bool wait)
-{
- enum i40iw_status_code status;
- struct i40iw_manage_vf_pble_info *hw_info;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
-
- if ((iwdev->init_state < CCQ_CREATED) && wait)
- wait = false;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait);
- if (!cqp_request)
- return I40IW_ERR_NO_MEMORY;
-
- cqp_info = &cqp_request->info;
- hw_info = &cqp_request->info.in.u.manage_vf_pble_bp.info;
- memcpy(hw_info, info, sizeof(*hw_info));
-
- cqp_info->cqp_cmd = OP_MANAGE_VF_PBLE_BP;
- cqp_info->post_sq = 1;
- cqp_info->in.u.manage_vf_pble_bp.cqp = &iwdev->cqp.sc_cqp;
- cqp_info->in.u.manage_vf_pble_bp.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Manage VF pble_bp fail");
- return status;
-}
-
-/**
- * i40iw_get_ib_wc - return change flush code to IB's
- * @opcode: iwarp flush code
- */
-static enum ib_wc_status i40iw_get_ib_wc(enum i40iw_flush_opcode opcode)
-{
- switch (opcode) {
- case FLUSH_PROT_ERR:
- return IB_WC_LOC_PROT_ERR;
- case FLUSH_REM_ACCESS_ERR:
- return IB_WC_REM_ACCESS_ERR;
- case FLUSH_LOC_QP_OP_ERR:
- return IB_WC_LOC_QP_OP_ERR;
- case FLUSH_REM_OP_ERR:
- return IB_WC_REM_OP_ERR;
- case FLUSH_LOC_LEN_ERR:
- return IB_WC_LOC_LEN_ERR;
- case FLUSH_GENERAL_ERR:
- return IB_WC_GENERAL_ERR;
- case FLUSH_FATAL_ERR:
- default:
- return IB_WC_FATAL_ERR;
- }
-}
-
-/**
- * i40iw_set_flush_info - set flush info
- * @pinfo: set flush info
- * @min: minor err
- * @maj: major err
- * @opcode: flush error code
- */
-static void i40iw_set_flush_info(struct i40iw_qp_flush_info *pinfo,
- u16 *min,
- u16 *maj,
- enum i40iw_flush_opcode opcode)
-{
- *min = (u16)i40iw_get_ib_wc(opcode);
- *maj = CQE_MAJOR_DRV;
- pinfo->userflushcode = true;
-}
-
-/**
- * i40iw_flush_wqes - flush wqe for qp
- * @iwdev: iwarp device
- * @iwqp: qp to flush wqes
- */
-void i40iw_flush_wqes(struct i40iw_device *iwdev, struct i40iw_qp *iwqp)
-{
- struct i40iw_qp_flush_info info;
- struct i40iw_qp_flush_info *pinfo = &info;
-
- struct i40iw_sc_qp *qp = &iwqp->sc_qp;
-
- memset(pinfo, 0, sizeof(*pinfo));
- info.sq = true;
- info.rq = true;
- if (qp->term_flags) {
- i40iw_set_flush_info(pinfo, &pinfo->sq_minor_code,
- &pinfo->sq_major_code, qp->flush_code);
- i40iw_set_flush_info(pinfo, &pinfo->rq_minor_code,
- &pinfo->rq_major_code, qp->flush_code);
- }
- (void)i40iw_hw_flush_wqes(iwdev, &iwqp->sc_qp, &info, true);
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c
deleted file mode 100644
index 9c96ece5e7f3..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_main.c
+++ /dev/null
@@ -1,2070 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/if_vlan.h>
-#include <net/addrconf.h>
-
-#include "i40iw.h"
-#include "i40iw_register.h"
-#include <net/netevent.h>
-#define CLIENT_IW_INTERFACE_VERSION_MAJOR 0
-#define CLIENT_IW_INTERFACE_VERSION_MINOR 01
-#define CLIENT_IW_INTERFACE_VERSION_BUILD 00
-
-#define DRV_VERSION_MAJOR 0
-#define DRV_VERSION_MINOR 5
-#define DRV_VERSION_BUILD 123
-#define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
- __stringify(DRV_VERSION_MINOR) "." __stringify(DRV_VERSION_BUILD)
-
-static int push_mode;
-module_param(push_mode, int, 0644);
-MODULE_PARM_DESC(push_mode, "Low latency mode: 0=disabled (default), 1=enabled)");
-
-static int debug;
-module_param(debug, int, 0644);
-MODULE_PARM_DESC(debug, "debug flags: 0=disabled (default), 0x7fffffff=all");
-
-static int resource_profile;
-module_param(resource_profile, int, 0644);
-MODULE_PARM_DESC(resource_profile,
- "Resource Profile: 0=no VF RDMA support (default), 1=Weighted VF, 2=Even Distribution");
-
-static int max_rdma_vfs = 32;
-module_param(max_rdma_vfs, int, 0644);
-MODULE_PARM_DESC(max_rdma_vfs, "Maximum VF count: 0-32 32=default");
-static int mpa_version = 2;
-module_param(mpa_version, int, 0644);
-MODULE_PARM_DESC(mpa_version, "MPA version to be used in MPA Req/Resp 1 or 2");
-
-MODULE_AUTHOR("Intel Corporation, <e1000-rdma@...ts.sourceforge.net>");
-MODULE_DESCRIPTION("Intel(R) Ethernet Connection X722 iWARP RDMA Driver");
-MODULE_LICENSE("Dual BSD/GPL");
-
-static struct i40e_client i40iw_client;
-static char i40iw_client_name[I40E_CLIENT_STR_LENGTH] = "i40iw";
-
-static LIST_HEAD(i40iw_handlers);
-static spinlock_t i40iw_handler_lock;
-
-static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,
- u32 vf_id, u8 *msg, u16 len);
-
-static struct notifier_block i40iw_inetaddr_notifier = {
- .notifier_call = i40iw_inetaddr_event
-};
-
-static struct notifier_block i40iw_inetaddr6_notifier = {
- .notifier_call = i40iw_inet6addr_event
-};
-
-static struct notifier_block i40iw_net_notifier = {
- .notifier_call = i40iw_net_event
-};
-
-static struct notifier_block i40iw_netdevice_notifier = {
- .notifier_call = i40iw_netdevice_event
-};
-
-/**
- * i40iw_find_i40e_handler - find a handler given a client info
- * @ldev: pointer to a client info
- */
-static struct i40iw_handler *i40iw_find_i40e_handler(struct i40e_info *ldev)
-{
- struct i40iw_handler *hdl;
- unsigned long flags;
-
- spin_lock_irqsave(&i40iw_handler_lock, flags);
- list_for_each_entry(hdl, &i40iw_handlers, list) {
- if (hdl->ldev.netdev == ldev->netdev) {
- spin_unlock_irqrestore(&i40iw_handler_lock, flags);
- return hdl;
- }
- }
- spin_unlock_irqrestore(&i40iw_handler_lock, flags);
- return NULL;
-}
-
-/**
- * i40iw_find_netdev - find a handler given a netdev
- * @netdev: pointer to net_device
- */
-struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev)
-{
- struct i40iw_handler *hdl;
- unsigned long flags;
-
- spin_lock_irqsave(&i40iw_handler_lock, flags);
- list_for_each_entry(hdl, &i40iw_handlers, list) {
- if (hdl->ldev.netdev == netdev) {
- spin_unlock_irqrestore(&i40iw_handler_lock, flags);
- return hdl;
- }
- }
- spin_unlock_irqrestore(&i40iw_handler_lock, flags);
- return NULL;
-}
-
-/**
- * i40iw_add_handler - add a handler to the list
- * @hdl: handler to be added to the handler list
- */
-static void i40iw_add_handler(struct i40iw_handler *hdl)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&i40iw_handler_lock, flags);
- list_add(&hdl->list, &i40iw_handlers);
- spin_unlock_irqrestore(&i40iw_handler_lock, flags);
-}
-
-/**
- * i40iw_del_handler - delete a handler from the list
- * @hdl: handler to be deleted from the handler list
- */
-static int i40iw_del_handler(struct i40iw_handler *hdl)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&i40iw_handler_lock, flags);
- list_del(&hdl->list);
- spin_unlock_irqrestore(&i40iw_handler_lock, flags);
- return 0;
-}
-
-/**
- * i40iw_enable_intr - set up device interrupts
- * @dev: hardware control device structure
- * @msix_id: id of the interrupt to be enabled
- */
-static void i40iw_enable_intr(struct i40iw_sc_dev *dev, u32 msix_id)
-{
- u32 val;
-
- val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
- I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
- (3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
- if (dev->is_pf)
- i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_id - 1), val);
- else
- i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_id - 1), val);
-}
-
-/**
- * i40iw_dpc - tasklet for aeq and ceq 0
- * @data: iwarp device
- */
-static void i40iw_dpc(unsigned long data)
-{
- struct i40iw_device *iwdev = (struct i40iw_device *)data;
-
- if (iwdev->msix_shared)
- i40iw_process_ceq(iwdev, iwdev->ceqlist);
- i40iw_process_aeq(iwdev);
- i40iw_enable_intr(&iwdev->sc_dev, iwdev->iw_msixtbl[0].idx);
-}
-
-/**
- * i40iw_ceq_dpc - dpc handler for CEQ
- * @data: data points to CEQ
- */
-static void i40iw_ceq_dpc(unsigned long data)
-{
- struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;
- struct i40iw_device *iwdev = iwceq->iwdev;
-
- i40iw_process_ceq(iwdev, iwceq);
- i40iw_enable_intr(&iwdev->sc_dev, iwceq->msix_idx);
-}
-
-/**
- * i40iw_irq_handler - interrupt handler for aeq and ceq0
- * @irq: Interrupt request number
- * @data: iwarp device
- */
-static irqreturn_t i40iw_irq_handler(int irq, void *data)
-{
- struct i40iw_device *iwdev = (struct i40iw_device *)data;
-
- tasklet_schedule(&iwdev->dpc_tasklet);
- return IRQ_HANDLED;
-}
-
-/**
- * i40iw_destroy_cqp - destroy control qp
- * @iwdev: iwarp device
- * @create_done: 1 if cqp create poll was success
- *
- * Issue destroy cqp request and
- * free the resources associated with the cqp
- */
-static void i40iw_destroy_cqp(struct i40iw_device *iwdev, bool free_hwcqp)
-{
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_cqp *cqp = &iwdev->cqp;
-
- if (free_hwcqp)
- dev->cqp_ops->cqp_destroy(dev->cqp);
-
- i40iw_cleanup_pending_cqp_op(iwdev);
-
- i40iw_free_dma_mem(dev->hw, &cqp->sq);
- kfree(cqp->scratch_array);
- iwdev->cqp.scratch_array = NULL;
-
- kfree(cqp->cqp_requests);
- cqp->cqp_requests = NULL;
-}
-
-/**
- * i40iw_disable_irqs - disable device interrupts
- * @dev: hardware control device structure
- * @msic_vec: msix vector to disable irq
- * @dev_id: parameter to pass to free_irq (used during irq setup)
- *
- * The function is called when destroying aeq/ceq
- */
-static void i40iw_disable_irq(struct i40iw_sc_dev *dev,
- struct i40iw_msix_vector *msix_vec,
- void *dev_id)
-{
- if (dev->is_pf)
- i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_vec->idx - 1), 0);
- else
- i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_vec->idx - 1), 0);
- irq_set_affinity_hint(msix_vec->irq, NULL);
- free_irq(msix_vec->irq, dev_id);
-}
-
-/**
- * i40iw_destroy_aeq - destroy aeq
- * @iwdev: iwarp device
- *
- * Issue a destroy aeq request and
- * free the resources associated with the aeq
- * The function is called during driver unload
- */
-static void i40iw_destroy_aeq(struct i40iw_device *iwdev)
-{
- enum i40iw_status_code status = I40IW_ERR_NOT_READY;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_aeq *aeq = &iwdev->aeq;
-
- if (!iwdev->msix_shared)
- i40iw_disable_irq(dev, iwdev->iw_msixtbl, (void *)iwdev);
- if (iwdev->reset)
- goto exit;
-
- if (!dev->aeq_ops->aeq_destroy(&aeq->sc_aeq, 0, 1))
- status = dev->aeq_ops->aeq_destroy_done(&aeq->sc_aeq);
- if (status)
- i40iw_pr_err("destroy aeq failed %d\n", status);
-
-exit:
- i40iw_free_dma_mem(dev->hw, &aeq->mem);
-}
-
-/**
- * i40iw_destroy_ceq - destroy ceq
- * @iwdev: iwarp device
- * @iwceq: ceq to be destroyed
- *
- * Issue a destroy ceq request and
- * free the resources associated with the ceq
- */
-static void i40iw_destroy_ceq(struct i40iw_device *iwdev,
- struct i40iw_ceq *iwceq)
-{
- enum i40iw_status_code status;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
-
- if (iwdev->reset)
- goto exit;
-
- status = dev->ceq_ops->ceq_destroy(&iwceq->sc_ceq, 0, 1);
- if (status) {
- i40iw_pr_err("ceq destroy command failed %d\n", status);
- goto exit;
- }
-
- status = dev->ceq_ops->cceq_destroy_done(&iwceq->sc_ceq);
- if (status)
- i40iw_pr_err("ceq destroy completion failed %d\n", status);
-exit:
- i40iw_free_dma_mem(dev->hw, &iwceq->mem);
-}
-
-/**
- * i40iw_dele_ceqs - destroy all ceq's
- * @iwdev: iwarp device
- *
- * Go through all of the device ceq's and for each ceq
- * disable the ceq interrupt and destroy the ceq
- */
-static void i40iw_dele_ceqs(struct i40iw_device *iwdev)
-{
- u32 i = 0;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_ceq *iwceq = iwdev->ceqlist;
- struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
-
- if (iwdev->msix_shared) {
- i40iw_disable_irq(dev, msix_vec, (void *)iwdev);
- i40iw_destroy_ceq(iwdev, iwceq);
- iwceq++;
- i++;
- }
-
- for (msix_vec++; i < iwdev->ceqs_count; i++, msix_vec++, iwceq++) {
- i40iw_disable_irq(dev, msix_vec, (void *)iwceq);
- i40iw_destroy_ceq(iwdev, iwceq);
- }
-
- iwdev->sc_dev.ceq_valid = false;
-}
-
-/**
- * i40iw_destroy_ccq - destroy control cq
- * @iwdev: iwarp device
- *
- * Issue destroy ccq request and
- * free the resources associated with the ccq
- */
-static void i40iw_destroy_ccq(struct i40iw_device *iwdev)
-{
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_ccq *ccq = &iwdev->ccq;
- enum i40iw_status_code status = 0;
-
- if (!iwdev->reset)
- status = dev->ccq_ops->ccq_destroy(dev->ccq, 0, true);
- if (status)
- i40iw_pr_err("ccq destroy failed %d\n", status);
- i40iw_free_dma_mem(dev->hw, &ccq->mem_cq);
-}
-
-/* types of hmc objects */
-static enum i40iw_hmc_rsrc_type iw_hmc_obj_types[] = {
- I40IW_HMC_IW_QP,
- I40IW_HMC_IW_CQ,
- I40IW_HMC_IW_HTE,
- I40IW_HMC_IW_ARP,
- I40IW_HMC_IW_APBVT_ENTRY,
- I40IW_HMC_IW_MR,
- I40IW_HMC_IW_XF,
- I40IW_HMC_IW_XFFL,
- I40IW_HMC_IW_Q1,
- I40IW_HMC_IW_Q1FL,
- I40IW_HMC_IW_TIMER,
-};
-
-/**
- * i40iw_close_hmc_objects_type - delete hmc objects of a given type
- * @iwdev: iwarp device
- * @obj_type: the hmc object type to be deleted
- * @is_pf: true if the function is PF otherwise false
- * @reset: true if called before reset
- */
-static void i40iw_close_hmc_objects_type(struct i40iw_sc_dev *dev,
- enum i40iw_hmc_rsrc_type obj_type,
- struct i40iw_hmc_info *hmc_info,
- bool is_pf,
- bool reset)
-{
- struct i40iw_hmc_del_obj_info info;
-
- memset(&info, 0, sizeof(info));
- info.hmc_info = hmc_info;
- info.rsrc_type = obj_type;
- info.count = hmc_info->hmc_obj[obj_type].cnt;
- info.is_pf = is_pf;
- if (dev->hmc_ops->del_hmc_object(dev, &info, reset))
- i40iw_pr_err("del obj of type %d failed\n", obj_type);
-}
-
-/**
- * i40iw_del_hmc_objects - remove all device hmc objects
- * @dev: iwarp device
- * @hmc_info: hmc_info to free
- * @is_pf: true if hmc_info belongs to PF, not vf nor allocated
- * by PF on behalf of VF
- * @reset: true if called before reset
- */
-static void i40iw_del_hmc_objects(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_info *hmc_info,
- bool is_pf,
- bool reset)
-{
- unsigned int i;
-
- for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++)
- i40iw_close_hmc_objects_type(dev, iw_hmc_obj_types[i], hmc_info, is_pf, reset);
-}
-
-/**
- * i40iw_ceq_handler - interrupt handler for ceq
- * @data: ceq pointer
- */
-static irqreturn_t i40iw_ceq_handler(int irq, void *data)
-{
- struct i40iw_ceq *iwceq = (struct i40iw_ceq *)data;
-
- if (iwceq->irq != irq)
- i40iw_pr_err("expected irq = %d received irq = %d\n", iwceq->irq, irq);
- tasklet_schedule(&iwceq->dpc_tasklet);
- return IRQ_HANDLED;
-}
-
-/**
- * i40iw_create_hmc_obj_type - create hmc object of a given type
- * @dev: hardware control device structure
- * @info: information for the hmc object to create
- */
-static enum i40iw_status_code i40iw_create_hmc_obj_type(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_create_obj_info *info)
-{
- return dev->hmc_ops->create_hmc_object(dev, info);
-}
-
-/**
- * i40iw_create_hmc_objs - create all hmc objects for the device
- * @iwdev: iwarp device
- * @is_pf: true if the function is PF otherwise false
- *
- * Create the device hmc objects and allocate hmc pages
- * Return 0 if successful, otherwise clean up and return error
- */
-static enum i40iw_status_code i40iw_create_hmc_objs(struct i40iw_device *iwdev,
- bool is_pf)
-{
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_hmc_create_obj_info info;
- enum i40iw_status_code status;
- int i;
-
- memset(&info, 0, sizeof(info));
- info.hmc_info = dev->hmc_info;
- info.is_pf = is_pf;
- info.entry_type = iwdev->sd_type;
- for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
- info.rsrc_type = iw_hmc_obj_types[i];
- info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt;
- info.add_sd_cnt = 0;
- status = i40iw_create_hmc_obj_type(dev, &info);
- if (status) {
- i40iw_pr_err("create obj type %d status = %d\n",
- iw_hmc_obj_types[i], status);
- break;
- }
- }
- if (!status)
- return (dev->cqp_misc_ops->static_hmc_pages_allocated(dev->cqp, 0,
- dev->hmc_fn_id,
- true, true));
-
- while (i) {
- i--;
- /* destroy the hmc objects of a given type */
- i40iw_close_hmc_objects_type(dev,
- iw_hmc_obj_types[i],
- dev->hmc_info,
- is_pf,
- false);
- }
- return status;
-}
-
-/**
- * i40iw_obj_aligned_mem - get aligned memory from device allocated memory
- * @iwdev: iwarp device
- * @memptr: points to the memory addresses
- * @size: size of memory needed
- * @mask: mask for the aligned memory
- *
- * Get aligned memory of the requested size and
- * update the memptr to point to the new aligned memory
- * Return 0 if successful, otherwise return no memory error
- */
-enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev,
- struct i40iw_dma_mem *memptr,
- u32 size,
- u32 mask)
-{
- unsigned long va, newva;
- unsigned long extra;
-
- va = (unsigned long)iwdev->obj_next.va;
- newva = va;
- if (mask)
- newva = ALIGN(va, (mask + 1));
- extra = newva - va;
- memptr->va = (u8 *)va + extra;
- memptr->pa = iwdev->obj_next.pa + extra;
- memptr->size = size;
- if ((memptr->va + size) > (iwdev->obj_mem.va + iwdev->obj_mem.size))
- return I40IW_ERR_NO_MEMORY;
-
- iwdev->obj_next.va = memptr->va + size;
- iwdev->obj_next.pa = memptr->pa + size;
- return 0;
-}
-
-/**
- * i40iw_create_cqp - create control qp
- * @iwdev: iwarp device
- *
- * Return 0, if the cqp and all the resources associated with it
- * are successfully created, otherwise return error
- */
-static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev)
-{
- enum i40iw_status_code status;
- u32 sqsize = I40IW_CQP_SW_SQSIZE_2048;
- struct i40iw_dma_mem mem;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_cqp_init_info cqp_init_info;
- struct i40iw_cqp *cqp = &iwdev->cqp;
- u16 maj_err, min_err;
- int i;
-
- cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
- if (!cqp->cqp_requests)
- return I40IW_ERR_NO_MEMORY;
- cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
- if (!cqp->scratch_array) {
- kfree(cqp->cqp_requests);
- return I40IW_ERR_NO_MEMORY;
- }
- dev->cqp = &cqp->sc_cqp;
- dev->cqp->dev = dev;
- memset(&cqp_init_info, 0, sizeof(cqp_init_info));
- status = i40iw_allocate_dma_mem(dev->hw, &cqp->sq,
- (sizeof(struct i40iw_cqp_sq_wqe) * sqsize),
- I40IW_CQP_ALIGNMENT);
- if (status)
- goto exit;
- status = i40iw_obj_aligned_mem(iwdev, &mem, sizeof(struct i40iw_cqp_ctx),
- I40IW_HOST_CTX_ALIGNMENT_MASK);
- if (status)
- goto exit;
- dev->cqp->host_ctx_pa = mem.pa;
- dev->cqp->host_ctx = mem.va;
- /* populate the cqp init info */
- cqp_init_info.dev = dev;
- cqp_init_info.sq_size = sqsize;
- cqp_init_info.sq = cqp->sq.va;
- cqp_init_info.sq_pa = cqp->sq.pa;
- cqp_init_info.host_ctx_pa = mem.pa;
- cqp_init_info.host_ctx = mem.va;
- cqp_init_info.hmc_profile = iwdev->resource_profile;
- cqp_init_info.enabled_vf_count = iwdev->max_rdma_vfs;
- cqp_init_info.scratch_array = cqp->scratch_array;
- status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info);
- if (status) {
- i40iw_pr_err("cqp init status %d\n", status);
- goto exit;
- }
- status = dev->cqp_ops->cqp_create(dev->cqp, &maj_err, &min_err);
- if (status) {
- i40iw_pr_err("cqp create status %d maj_err %d min_err %d\n",
- status, maj_err, min_err);
- goto exit;
- }
- spin_lock_init(&cqp->req_lock);
- INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
- INIT_LIST_HEAD(&cqp->cqp_pending_reqs);
- /* init the waitq of the cqp_requests and add them to the list */
- for (i = 0; i < sqsize; i++) {
- init_waitqueue_head(&cqp->cqp_requests[i].waitq);
- list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs);
- }
- return 0;
-exit:
- /* clean up the created resources */
- i40iw_destroy_cqp(iwdev, false);
- return status;
-}
-
-/**
- * i40iw_create_ccq - create control cq
- * @iwdev: iwarp device
- *
- * Return 0, if the ccq and the resources associated with it
- * are successfully created, otherwise return error
- */
-static enum i40iw_status_code i40iw_create_ccq(struct i40iw_device *iwdev)
-{
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_dma_mem mem;
- enum i40iw_status_code status;
- struct i40iw_ccq_init_info info;
- struct i40iw_ccq *ccq = &iwdev->ccq;
-
- memset(&info, 0, sizeof(info));
- dev->ccq = &ccq->sc_cq;
- dev->ccq->dev = dev;
- info.dev = dev;
- ccq->shadow_area.size = sizeof(struct i40iw_cq_shadow_area);
- ccq->mem_cq.size = sizeof(struct i40iw_cqe) * IW_CCQ_SIZE;
- status = i40iw_allocate_dma_mem(dev->hw, &ccq->mem_cq,
- ccq->mem_cq.size, I40IW_CQ0_ALIGNMENT);
- if (status)
- goto exit;
- status = i40iw_obj_aligned_mem(iwdev, &mem, ccq->shadow_area.size,
- I40IW_SHADOWAREA_MASK);
- if (status)
- goto exit;
- ccq->sc_cq.back_cq = (void *)ccq;
- /* populate the ccq init info */
- info.cq_base = ccq->mem_cq.va;
- info.cq_pa = ccq->mem_cq.pa;
- info.num_elem = IW_CCQ_SIZE;
- info.shadow_area = mem.va;
- info.shadow_area_pa = mem.pa;
- info.ceqe_mask = false;
- info.ceq_id_valid = true;
- info.shadow_read_threshold = 16;
- status = dev->ccq_ops->ccq_init(dev->ccq, &info);
- if (!status)
- status = dev->ccq_ops->ccq_create(dev->ccq, 0, true, true);
-exit:
- if (status)
- i40iw_free_dma_mem(dev->hw, &ccq->mem_cq);
- return status;
-}
-
-/**
- * i40iw_configure_ceq_vector - set up the msix interrupt vector for ceq
- * @iwdev: iwarp device
- * @msix_vec: interrupt vector information
- * @iwceq: ceq associated with the vector
- * @ceq_id: the id number of the iwceq
- *
- * Allocate interrupt resources and enable irq handling
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_configure_ceq_vector(struct i40iw_device *iwdev,
- struct i40iw_ceq *iwceq,
- u32 ceq_id,
- struct i40iw_msix_vector *msix_vec)
-{
- enum i40iw_status_code status;
-
- if (iwdev->msix_shared && !ceq_id) {
- tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
- status = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "AEQCEQ", iwdev);
- } else {
- tasklet_init(&iwceq->dpc_tasklet, i40iw_ceq_dpc, (unsigned long)iwceq);
- status = request_irq(msix_vec->irq, i40iw_ceq_handler, 0, "CEQ", iwceq);
- }
-
- cpumask_clear(&msix_vec->mask);
- cpumask_set_cpu(msix_vec->cpu_affinity, &msix_vec->mask);
- irq_set_affinity_hint(msix_vec->irq, &msix_vec->mask);
-
- if (status) {
- i40iw_pr_err("ceq irq config fail\n");
- return I40IW_ERR_CONFIG;
- }
- msix_vec->ceq_id = ceq_id;
-
- return 0;
-}
-
-/**
- * i40iw_create_ceq - create completion event queue
- * @iwdev: iwarp device
- * @iwceq: pointer to the ceq resources to be created
- * @ceq_id: the id number of the iwceq
- *
- * Return 0, if the ceq and the resources associated with it
- * are successfully created, otherwise return error
- */
-static enum i40iw_status_code i40iw_create_ceq(struct i40iw_device *iwdev,
- struct i40iw_ceq *iwceq,
- u32 ceq_id)
-{
- enum i40iw_status_code status;
- struct i40iw_ceq_init_info info;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- u64 scratch;
-
- memset(&info, 0, sizeof(info));
- info.ceq_id = ceq_id;
- iwceq->iwdev = iwdev;
- iwceq->mem.size = sizeof(struct i40iw_ceqe) *
- iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
- status = i40iw_allocate_dma_mem(dev->hw, &iwceq->mem, iwceq->mem.size,
- I40IW_CEQ_ALIGNMENT);
- if (status)
- goto exit;
- info.ceq_id = ceq_id;
- info.ceqe_base = iwceq->mem.va;
- info.ceqe_pa = iwceq->mem.pa;
-
- info.elem_cnt = iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
- iwceq->sc_ceq.ceq_id = ceq_id;
- info.dev = dev;
- scratch = (uintptr_t)&iwdev->cqp.sc_cqp;
- status = dev->ceq_ops->ceq_init(&iwceq->sc_ceq, &info);
- if (!status)
- status = dev->ceq_ops->cceq_create(&iwceq->sc_ceq, scratch);
-
-exit:
- if (status)
- i40iw_free_dma_mem(dev->hw, &iwceq->mem);
- return status;
-}
-
-void i40iw_request_reset(struct i40iw_device *iwdev)
-{
- struct i40e_info *ldev = iwdev->ldev;
-
- ldev->ops->request_reset(ldev, iwdev->client, 1);
-}
-
-/**
- * i40iw_setup_ceqs - manage the device ceq's and their interrupt resources
- * @iwdev: iwarp device
- * @ldev: i40e lan device
- *
- * Allocate a list for all device completion event queues
- * Create the ceq's and configure their msix interrupt vectors
- * Return 0, if at least one ceq is successfully set up, otherwise return error
- */
-static enum i40iw_status_code i40iw_setup_ceqs(struct i40iw_device *iwdev,
- struct i40e_info *ldev)
-{
- u32 i;
- u32 ceq_id;
- struct i40iw_ceq *iwceq;
- struct i40iw_msix_vector *msix_vec;
- enum i40iw_status_code status = 0;
- u32 num_ceqs;
-
- if (ldev && ldev->ops && ldev->ops->setup_qvlist) {
- status = ldev->ops->setup_qvlist(ldev, &i40iw_client,
- iwdev->iw_qvlist);
- if (status)
- goto exit;
- } else {
- status = I40IW_ERR_BAD_PTR;
- goto exit;
- }
-
- num_ceqs = min(iwdev->msix_count, iwdev->sc_dev.hmc_fpm_misc.max_ceqs);
- iwdev->ceqlist = kcalloc(num_ceqs, sizeof(*iwdev->ceqlist), GFP_KERNEL);
- if (!iwdev->ceqlist) {
- status = I40IW_ERR_NO_MEMORY;
- goto exit;
- }
- i = (iwdev->msix_shared) ? 0 : 1;
- for (ceq_id = 0; i < num_ceqs; i++, ceq_id++) {
- iwceq = &iwdev->ceqlist[ceq_id];
- status = i40iw_create_ceq(iwdev, iwceq, ceq_id);
- if (status) {
- i40iw_pr_err("create ceq status = %d\n", status);
- break;
- }
-
- msix_vec = &iwdev->iw_msixtbl[i];
- iwceq->irq = msix_vec->irq;
- iwceq->msix_idx = msix_vec->idx;
- status = i40iw_configure_ceq_vector(iwdev, iwceq, ceq_id, msix_vec);
- if (status) {
- i40iw_destroy_ceq(iwdev, iwceq);
- break;
- }
- i40iw_enable_intr(&iwdev->sc_dev, msix_vec->idx);
- iwdev->ceqs_count++;
- }
-exit:
- if (status && !iwdev->ceqs_count) {
- kfree(iwdev->ceqlist);
- iwdev->ceqlist = NULL;
- return status;
- } else {
- iwdev->sc_dev.ceq_valid = true;
- return 0;
- }
-
-}
-
-/**
- * i40iw_configure_aeq_vector - set up the msix vector for aeq
- * @iwdev: iwarp device
- *
- * Allocate interrupt resources and enable irq handling
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_configure_aeq_vector(struct i40iw_device *iwdev)
-{
- struct i40iw_msix_vector *msix_vec = iwdev->iw_msixtbl;
- u32 ret = 0;
-
- if (!iwdev->msix_shared) {
- tasklet_init(&iwdev->dpc_tasklet, i40iw_dpc, (unsigned long)iwdev);
- ret = request_irq(msix_vec->irq, i40iw_irq_handler, 0, "i40iw", iwdev);
- }
- if (ret) {
- i40iw_pr_err("aeq irq config fail\n");
- return I40IW_ERR_CONFIG;
- }
-
- return 0;
-}
-
-/**
- * i40iw_create_aeq - create async event queue
- * @iwdev: iwarp device
- *
- * Return 0, if the aeq and the resources associated with it
- * are successfully created, otherwise return error
- */
-static enum i40iw_status_code i40iw_create_aeq(struct i40iw_device *iwdev)
-{
- enum i40iw_status_code status;
- struct i40iw_aeq_init_info info;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_aeq *aeq = &iwdev->aeq;
- u64 scratch = 0;
- u32 aeq_size;
-
- aeq_size = 2 * iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt +
- iwdev->sc_dev.hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt;
- memset(&info, 0, sizeof(info));
- aeq->mem.size = sizeof(struct i40iw_sc_aeqe) * aeq_size;
- status = i40iw_allocate_dma_mem(dev->hw, &aeq->mem, aeq->mem.size,
- I40IW_AEQ_ALIGNMENT);
- if (status)
- goto exit;
-
- info.aeqe_base = aeq->mem.va;
- info.aeq_elem_pa = aeq->mem.pa;
- info.elem_cnt = aeq_size;
- info.dev = dev;
- status = dev->aeq_ops->aeq_init(&aeq->sc_aeq, &info);
- if (status)
- goto exit;
- status = dev->aeq_ops->aeq_create(&aeq->sc_aeq, scratch, 1);
- if (!status)
- status = dev->aeq_ops->aeq_create_done(&aeq->sc_aeq);
-exit:
- if (status)
- i40iw_free_dma_mem(dev->hw, &aeq->mem);
- return status;
-}
-
-/**
- * i40iw_setup_aeq - set up the device aeq
- * @iwdev: iwarp device
- *
- * Create the aeq and configure its msix interrupt vector
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_setup_aeq(struct i40iw_device *iwdev)
-{
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- enum i40iw_status_code status;
-
- status = i40iw_create_aeq(iwdev);
- if (status)
- return status;
-
- status = i40iw_configure_aeq_vector(iwdev);
- if (status) {
- i40iw_destroy_aeq(iwdev);
- return status;
- }
-
- if (!iwdev->msix_shared)
- i40iw_enable_intr(dev, iwdev->iw_msixtbl[0].idx);
- return 0;
-}
-
-/**
- * i40iw_initialize_ilq - create iwarp local queue for cm
- * @iwdev: iwarp device
- *
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_initialize_ilq(struct i40iw_device *iwdev)
-{
- struct i40iw_puda_rsrc_info info;
- enum i40iw_status_code status;
-
- memset(&info, 0, sizeof(info));
- info.type = I40IW_PUDA_RSRC_TYPE_ILQ;
- info.cq_id = 1;
- info.qp_id = 0;
- info.count = 1;
- info.pd_id = 1;
- info.sq_size = 8192;
- info.rq_size = 8192;
- info.buf_size = 1024;
- info.tx_buf_cnt = 16384;
- info.receive = i40iw_receive_ilq;
- info.xmit_complete = i40iw_free_sqbuf;
- status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
- if (status)
- i40iw_pr_err("ilq create fail\n");
- return status;
-}
-
-/**
- * i40iw_initialize_ieq - create iwarp exception queue
- * @iwdev: iwarp device
- *
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_initialize_ieq(struct i40iw_device *iwdev)
-{
- struct i40iw_puda_rsrc_info info;
- enum i40iw_status_code status;
-
- memset(&info, 0, sizeof(info));
- info.type = I40IW_PUDA_RSRC_TYPE_IEQ;
- info.cq_id = 2;
- info.qp_id = iwdev->vsi.exception_lan_queue;
- info.count = 1;
- info.pd_id = 2;
- info.sq_size = 8192;
- info.rq_size = 8192;
- info.buf_size = iwdev->vsi.mtu + VLAN_ETH_HLEN;
- info.tx_buf_cnt = 4096;
- status = i40iw_puda_create_rsrc(&iwdev->vsi, &info);
- if (status)
- i40iw_pr_err("ieq create fail\n");
- return status;
-}
-
-/**
- * i40iw_reinitialize_ieq - destroy and re-create ieq
- * @dev: iwarp device
- */
-void i40iw_reinitialize_ieq(struct i40iw_sc_dev *dev)
-{
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
-
- i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_IEQ, false);
- if (i40iw_initialize_ieq(iwdev)) {
- iwdev->reset = true;
- i40iw_request_reset(iwdev);
- }
-}
-
-/**
- * i40iw_hmc_setup - create hmc objects for the device
- * @iwdev: iwarp device
- *
- * Set up the device private memory space for the number and size of
- * the hmc objects and create the objects
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_hmc_setup(struct i40iw_device *iwdev)
-{
- enum i40iw_status_code status;
-
- iwdev->sd_type = I40IW_SD_TYPE_DIRECT;
- status = i40iw_config_fpm_values(&iwdev->sc_dev, IW_CFG_FPM_QP_COUNT);
- if (status)
- goto exit;
- status = i40iw_create_hmc_objs(iwdev, true);
- if (status)
- goto exit;
- iwdev->init_state = HMC_OBJS_CREATED;
-exit:
- return status;
-}
-
-/**
- * i40iw_del_init_mem - deallocate memory resources
- * @iwdev: iwarp device
- */
-static void i40iw_del_init_mem(struct i40iw_device *iwdev)
-{
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
-
- i40iw_free_dma_mem(&iwdev->hw, &iwdev->obj_mem);
- kfree(dev->hmc_info->sd_table.sd_entry);
- dev->hmc_info->sd_table.sd_entry = NULL;
- kfree(iwdev->mem_resources);
- iwdev->mem_resources = NULL;
- kfree(iwdev->ceqlist);
- iwdev->ceqlist = NULL;
- kfree(iwdev->iw_msixtbl);
- iwdev->iw_msixtbl = NULL;
- kfree(iwdev->hmc_info_mem);
- iwdev->hmc_info_mem = NULL;
-}
-
-/**
- * i40iw_del_macip_entry - remove a mac ip address entry from the hw table
- * @iwdev: iwarp device
- * @idx: the index of the mac ip address to delete
- */
-static void i40iw_del_macip_entry(struct i40iw_device *iwdev, u8 idx)
-{
- struct i40iw_cqp *iwcqp = &iwdev->cqp;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- enum i40iw_status_code status = 0;
-
- cqp_request = i40iw_get_cqp_request(iwcqp, true);
- if (!cqp_request) {
- i40iw_pr_err("cqp_request memory failed\n");
- return;
- }
- cqp_info = &cqp_request->info;
- cqp_info->cqp_cmd = OP_DELETE_LOCAL_MAC_IPADDR_ENTRY;
- cqp_info->post_sq = 1;
- cqp_info->in.u.del_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
- cqp_info->in.u.del_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
- cqp_info->in.u.del_local_mac_ipaddr_entry.entry_idx = idx;
- cqp_info->in.u.del_local_mac_ipaddr_entry.ignore_ref_count = 0;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Del MAC Ip entry fail");
-}
-
-/**
- * i40iw_add_mac_ipaddr_entry - add a mac ip address entry to the hw table
- * @iwdev: iwarp device
- * @mac_addr: pointer to mac address
- * @idx: the index of the mac ip address to add
- */
-static enum i40iw_status_code i40iw_add_mac_ipaddr_entry(struct i40iw_device *iwdev,
- u8 *mac_addr,
- u8 idx)
-{
- struct i40iw_local_mac_ipaddr_entry_info *info;
- struct i40iw_cqp *iwcqp = &iwdev->cqp;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- enum i40iw_status_code status = 0;
-
- cqp_request = i40iw_get_cqp_request(iwcqp, true);
- if (!cqp_request) {
- i40iw_pr_err("cqp_request memory failed\n");
- return I40IW_ERR_NO_MEMORY;
- }
-
- cqp_info = &cqp_request->info;
-
- cqp_info->post_sq = 1;
- info = &cqp_info->in.u.add_local_mac_ipaddr_entry.info;
- ether_addr_copy(info->mac_addr, mac_addr);
- info->entry_idx = idx;
- cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
- cqp_info->cqp_cmd = OP_ADD_LOCAL_MAC_IPADDR_ENTRY;
- cqp_info->in.u.add_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
- cqp_info->in.u.add_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Add MAC Ip entry fail");
- return status;
-}
-
-/**
- * i40iw_alloc_local_mac_ipaddr_entry - allocate a mac ip address entry
- * @iwdev: iwarp device
- * @mac_ip_tbl_idx: the index of the new mac ip address
- *
- * Allocate a mac ip address entry and update the mac_ip_tbl_idx
- * to hold the index of the newly created mac ip address
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_alloc_local_mac_ipaddr_entry(struct i40iw_device *iwdev,
- u16 *mac_ip_tbl_idx)
-{
- struct i40iw_cqp *iwcqp = &iwdev->cqp;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- enum i40iw_status_code status = 0;
-
- cqp_request = i40iw_get_cqp_request(iwcqp, true);
- if (!cqp_request) {
- i40iw_pr_err("cqp_request memory failed\n");
- return I40IW_ERR_NO_MEMORY;
- }
-
- /* increment refcount, because we need the cqp request ret value */
- atomic_inc(&cqp_request->refcount);
-
- cqp_info = &cqp_request->info;
- cqp_info->cqp_cmd = OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY;
- cqp_info->post_sq = 1;
- cqp_info->in.u.alloc_local_mac_ipaddr_entry.cqp = &iwcqp->sc_cqp;
- cqp_info->in.u.alloc_local_mac_ipaddr_entry.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (!status)
- *mac_ip_tbl_idx = cqp_request->compl_info.op_ret_val;
- else
- i40iw_pr_err("CQP-OP Alloc MAC Ip entry fail");
- /* decrement refcount and free the cqp request, if no longer used */
- i40iw_put_cqp_request(iwcqp, cqp_request);
- return status;
-}
-
-/**
- * i40iw_alloc_set_mac_ipaddr - set up a mac ip address table entry
- * @iwdev: iwarp device
- * @macaddr: pointer to mac address
- *
- * Allocate a mac ip address entry and add it to the hw table
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_alloc_set_mac_ipaddr(struct i40iw_device *iwdev,
- u8 *macaddr)
-{
- enum i40iw_status_code status;
-
- status = i40iw_alloc_local_mac_ipaddr_entry(iwdev, &iwdev->mac_ip_table_idx);
- if (!status) {
- status = i40iw_add_mac_ipaddr_entry(iwdev, macaddr,
- (u8)iwdev->mac_ip_table_idx);
- if (status)
- i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
- }
- return status;
-}
-
-/**
- * i40iw_add_ipv6_addr - add ipv6 address to the hw arp table
- * @iwdev: iwarp device
- */
-static void i40iw_add_ipv6_addr(struct i40iw_device *iwdev)
-{
- struct net_device *ip_dev;
- struct inet6_dev *idev;
- struct inet6_ifaddr *ifp, *tmp;
- u32 local_ipaddr6[4];
-
- rcu_read_lock();
- for_each_netdev_rcu(&init_net, ip_dev) {
- if ((((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF) &&
- (rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev)) ||
- (ip_dev == iwdev->netdev)) && (ip_dev->flags & IFF_UP)) {
- idev = __in6_dev_get(ip_dev);
- if (!idev) {
- i40iw_pr_err("ipv6 inet device not found\n");
- break;
- }
- list_for_each_entry_safe(ifp, tmp, &idev->addr_list, if_list) {
- i40iw_pr_info("IP=%pI6, vlan_id=%d, MAC=%pM\n", &ifp->addr,
- rdma_vlan_dev_vlan_id(ip_dev), ip_dev->dev_addr);
- i40iw_copy_ip_ntohl(local_ipaddr6,
- ifp->addr.in6_u.u6_addr32);
- i40iw_manage_arp_cache(iwdev,
- ip_dev->dev_addr,
- local_ipaddr6,
- false,
- I40IW_ARP_ADD);
- }
- }
- }
- rcu_read_unlock();
-}
-
-/**
- * i40iw_add_ipv4_addr - add ipv4 address to the hw arp table
- * @iwdev: iwarp device
- */
-static void i40iw_add_ipv4_addr(struct i40iw_device *iwdev)
-{
- struct net_device *dev;
- struct in_device *idev;
- u32 ip_addr;
-
- rcu_read_lock();
- for_each_netdev_rcu(&init_net, dev) {
- if ((((rdma_vlan_dev_vlan_id(dev) < 0xFFFF) &&
- (rdma_vlan_dev_real_dev(dev) == iwdev->netdev)) ||
- (dev == iwdev->netdev)) && (READ_ONCE(dev->flags) & IFF_UP)) {
- const struct in_ifaddr *ifa;
-
- idev = __in_dev_get_rcu(dev);
- if (!idev)
- continue;
- in_dev_for_each_ifa_rcu(ifa, idev) {
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_CM,
- "IP=%pI4, vlan_id=%d, MAC=%pM\n", &ifa->ifa_address,
- rdma_vlan_dev_vlan_id(dev), dev->dev_addr);
-
- ip_addr = ntohl(ifa->ifa_address);
- i40iw_manage_arp_cache(iwdev,
- dev->dev_addr,
- &ip_addr,
- true,
- I40IW_ARP_ADD);
- }
- }
- }
- rcu_read_unlock();
-}
-
-/**
- * i40iw_add_mac_ip - add mac and ip addresses
- * @iwdev: iwarp device
- *
- * Create and add a mac ip address entry to the hw table and
- * ipv4/ipv6 addresses to the arp cache
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_add_mac_ip(struct i40iw_device *iwdev)
-{
- struct net_device *netdev = iwdev->netdev;
- enum i40iw_status_code status;
-
- status = i40iw_alloc_set_mac_ipaddr(iwdev, (u8 *)netdev->dev_addr);
- if (status)
- return status;
- i40iw_add_ipv4_addr(iwdev);
- i40iw_add_ipv6_addr(iwdev);
- return 0;
-}
-
-/**
- * i40iw_wait_pe_ready - Check if firmware is ready
- * @hw: provides access to registers
- */
-static void i40iw_wait_pe_ready(struct i40iw_hw *hw)
-{
- u32 statusfw;
- u32 statuscpu0;
- u32 statuscpu1;
- u32 statuscpu2;
- u32 retrycount = 0;
-
- do {
- statusfw = i40iw_rd32(hw, I40E_GLPE_FWLDSTATUS);
- i40iw_pr_info("[%04d] fm load status[x%04X]\n", __LINE__, statusfw);
- statuscpu0 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS0);
- i40iw_pr_info("[%04d] CSR_CQP status[x%04X]\n", __LINE__, statuscpu0);
- statuscpu1 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS1);
- i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS1 status[x%04X]\n",
- __LINE__, statuscpu1);
- statuscpu2 = i40iw_rd32(hw, I40E_GLPE_CPUSTATUS2);
- i40iw_pr_info("[%04d] I40E_GLPE_CPUSTATUS2 status[x%04X]\n",
- __LINE__, statuscpu2);
- if ((statuscpu0 == 0x80) && (statuscpu1 == 0x80) && (statuscpu2 == 0x80))
- break; /* SUCCESS */
- msleep(1000);
- retrycount++;
- } while (retrycount < 14);
- i40iw_wr32(hw, 0xb4040, 0x4C104C5);
-}
-
-/**
- * i40iw_initialize_dev - initialize device
- * @iwdev: iwarp device
- * @ldev: lan device information
- *
- * Allocate memory for the hmc objects and initialize iwdev
- * Return 0 if successful, otherwise clean up the resources
- * and return error
- */
-static enum i40iw_status_code i40iw_initialize_dev(struct i40iw_device *iwdev,
- struct i40e_info *ldev)
-{
- enum i40iw_status_code status;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_device_init_info info;
- struct i40iw_vsi_init_info vsi_info;
- struct i40iw_dma_mem mem;
- struct i40iw_l2params l2params;
- u32 size;
- struct i40iw_vsi_stats_info stats_info;
- u16 last_qset = I40IW_NO_QSET;
- u16 qset;
- u32 i;
-
- memset(&l2params, 0, sizeof(l2params));
- memset(&info, 0, sizeof(info));
- size = sizeof(struct i40iw_hmc_pble_rsrc) + sizeof(struct i40iw_hmc_info) +
- (sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX);
- iwdev->hmc_info_mem = kzalloc(size, GFP_KERNEL);
- if (!iwdev->hmc_info_mem)
- return I40IW_ERR_NO_MEMORY;
-
- iwdev->pble_rsrc = (struct i40iw_hmc_pble_rsrc *)iwdev->hmc_info_mem;
- dev->hmc_info = &iwdev->hw.hmc;
- dev->hmc_info->hmc_obj = (struct i40iw_hmc_obj_info *)(iwdev->pble_rsrc + 1);
- status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_QUERY_FPM_BUF_SIZE,
- I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK);
- if (status)
- goto error;
- info.fpm_query_buf_pa = mem.pa;
- info.fpm_query_buf = mem.va;
- status = i40iw_obj_aligned_mem(iwdev, &mem, I40IW_COMMIT_FPM_BUF_SIZE,
- I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK);
- if (status)
- goto error;
- info.fpm_commit_buf_pa = mem.pa;
- info.fpm_commit_buf = mem.va;
- info.hmc_fn_id = ldev->fid;
- info.is_pf = (ldev->ftype) ? false : true;
- info.bar0 = ldev->hw_addr;
- info.hw = &iwdev->hw;
- info.debug_mask = debug;
- l2params.mtu =
- (ldev->params.mtu) ? ldev->params.mtu : I40IW_DEFAULT_MTU;
- for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++) {
- qset = ldev->params.qos.prio_qos[i].qs_handle;
- l2params.qs_handle_list[i] = qset;
- if (last_qset == I40IW_NO_QSET)
- last_qset = qset;
- else if ((qset != last_qset) && (qset != I40IW_NO_QSET))
- iwdev->dcb = true;
- }
- i40iw_pr_info("DCB is set/clear = %d\n", iwdev->dcb);
- info.vchnl_send = i40iw_virtchnl_send;
- status = i40iw_device_init(&iwdev->sc_dev, &info);
-
- if (status)
- goto error;
- memset(&vsi_info, 0, sizeof(vsi_info));
- vsi_info.dev = &iwdev->sc_dev;
- vsi_info.back_vsi = (void *)iwdev;
- vsi_info.params = &l2params;
- vsi_info.exception_lan_queue = 1;
- i40iw_sc_vsi_init(&iwdev->vsi, &vsi_info);
-
- if (dev->is_pf) {
- memset(&stats_info, 0, sizeof(stats_info));
- stats_info.fcn_id = ldev->fid;
- stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);
- if (!stats_info.pestat) {
- status = I40IW_ERR_NO_MEMORY;
- goto error;
- }
- stats_info.stats_initialize = true;
- if (stats_info.pestat)
- i40iw_vsi_stats_init(&iwdev->vsi, &stats_info);
- }
- return status;
-error:
- kfree(iwdev->hmc_info_mem);
- iwdev->hmc_info_mem = NULL;
- return status;
-}
-
-/**
- * i40iw_register_notifiers - register tcp ip notifiers
- */
-static void i40iw_register_notifiers(void)
-{
- register_inetaddr_notifier(&i40iw_inetaddr_notifier);
- register_inet6addr_notifier(&i40iw_inetaddr6_notifier);
- register_netevent_notifier(&i40iw_net_notifier);
- register_netdevice_notifier(&i40iw_netdevice_notifier);
-}
-
-/**
- * i40iw_unregister_notifiers - unregister tcp ip notifiers
- */
-
-static void i40iw_unregister_notifiers(void)
-{
- unregister_netevent_notifier(&i40iw_net_notifier);
- unregister_inetaddr_notifier(&i40iw_inetaddr_notifier);
- unregister_inet6addr_notifier(&i40iw_inetaddr6_notifier);
- unregister_netdevice_notifier(&i40iw_netdevice_notifier);
-}
-
-/**
- * i40iw_save_msix_info - copy msix vector information to iwarp device
- * @iwdev: iwarp device
- * @ldev: lan device information
- *
- * Allocate iwdev msix table and copy the ldev msix info to the table
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_save_msix_info(struct i40iw_device *iwdev,
- struct i40e_info *ldev)
-{
- struct i40e_qvlist_info *iw_qvlist;
- struct i40e_qv_info *iw_qvinfo;
- u32 ceq_idx;
- u32 i;
- u32 size;
-
- if (!ldev->msix_count) {
- i40iw_pr_err("No MSI-X vectors\n");
- return I40IW_ERR_CONFIG;
- }
-
- iwdev->msix_count = ldev->msix_count;
-
- size = sizeof(struct i40iw_msix_vector) * iwdev->msix_count;
- size += sizeof(struct i40e_qvlist_info);
- size += sizeof(struct i40e_qv_info) * iwdev->msix_count - 1;
- iwdev->iw_msixtbl = kzalloc(size, GFP_KERNEL);
-
- if (!iwdev->iw_msixtbl)
- return I40IW_ERR_NO_MEMORY;
- iwdev->iw_qvlist = (struct i40e_qvlist_info *)(&iwdev->iw_msixtbl[iwdev->msix_count]);
- iw_qvlist = iwdev->iw_qvlist;
- iw_qvinfo = iw_qvlist->qv_info;
- iw_qvlist->num_vectors = iwdev->msix_count;
- if (iwdev->msix_count <= num_online_cpus())
- iwdev->msix_shared = true;
- for (i = 0, ceq_idx = 0; i < iwdev->msix_count; i++, iw_qvinfo++) {
- iwdev->iw_msixtbl[i].idx = ldev->msix_entries[i].entry;
- iwdev->iw_msixtbl[i].irq = ldev->msix_entries[i].vector;
- iwdev->iw_msixtbl[i].cpu_affinity = ceq_idx;
- if (i == 0) {
- iw_qvinfo->aeq_idx = 0;
- if (iwdev->msix_shared)
- iw_qvinfo->ceq_idx = ceq_idx++;
- else
- iw_qvinfo->ceq_idx = I40E_QUEUE_INVALID_IDX;
- } else {
- iw_qvinfo->aeq_idx = I40E_QUEUE_INVALID_IDX;
- iw_qvinfo->ceq_idx = ceq_idx++;
- }
- iw_qvinfo->itr_idx = 3;
- iw_qvinfo->v_idx = iwdev->iw_msixtbl[i].idx;
- }
- return 0;
-}
-
-/**
- * i40iw_deinit_device - clean up the device resources
- * @iwdev: iwarp device
- *
- * Destroy the ib device interface, remove the mac ip entry and ipv4/ipv6 addresses,
- * destroy the device queues and free the pble and the hmc objects
- */
-static void i40iw_deinit_device(struct i40iw_device *iwdev)
-{
- struct i40e_info *ldev = iwdev->ldev;
-
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
-
- i40iw_pr_info("state = %d\n", iwdev->init_state);
- if (iwdev->param_wq)
- destroy_workqueue(iwdev->param_wq);
-
- switch (iwdev->init_state) {
- case RDMA_DEV_REGISTERED:
- iwdev->iw_status = 0;
- i40iw_port_ibevent(iwdev);
- i40iw_destroy_rdma_device(iwdev->iwibdev);
- /* fallthrough */
- case IP_ADDR_REGISTERED:
- if (!iwdev->reset)
- i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx);
- /* fallthrough */
- /* fallthrough */
- case PBLE_CHUNK_MEM:
- i40iw_destroy_pble_pool(dev, iwdev->pble_rsrc);
- /* fallthrough */
- case CEQ_CREATED:
- i40iw_dele_ceqs(iwdev);
- /* fallthrough */
- case AEQ_CREATED:
- i40iw_destroy_aeq(iwdev);
- /* fallthrough */
- case IEQ_CREATED:
- i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_IEQ, iwdev->reset);
- /* fallthrough */
- case ILQ_CREATED:
- i40iw_puda_dele_resources(&iwdev->vsi, I40IW_PUDA_RSRC_TYPE_ILQ, iwdev->reset);
- /* fallthrough */
- case CCQ_CREATED:
- i40iw_destroy_ccq(iwdev);
- /* fallthrough */
- case HMC_OBJS_CREATED:
- i40iw_del_hmc_objects(dev, dev->hmc_info, true, iwdev->reset);
- /* fallthrough */
- case CQP_CREATED:
- i40iw_destroy_cqp(iwdev, true);
- /* fallthrough */
- case INITIAL_STATE:
- i40iw_cleanup_cm_core(&iwdev->cm_core);
- if (iwdev->vsi.pestat) {
- i40iw_vsi_stats_free(&iwdev->vsi);
- kfree(iwdev->vsi.pestat);
- }
- i40iw_del_init_mem(iwdev);
- break;
- case INVALID_STATE:
- /* fallthrough */
- default:
- i40iw_pr_err("bad init_state = %d\n", iwdev->init_state);
- break;
- }
-
- i40iw_del_handler(i40iw_find_i40e_handler(ldev));
- kfree(iwdev->hdl);
-}
-
-/**
- * i40iw_setup_init_state - set up the initial device struct
- * @hdl: handler for iwarp device - one per instance
- * @ldev: lan device information
- * @client: iwarp client information, provided during registration
- *
- * Initialize the iwarp device and its hdl information
- * using the ldev and client information
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl,
- struct i40e_info *ldev,
- struct i40e_client *client)
-{
- struct i40iw_device *iwdev = &hdl->device;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- enum i40iw_status_code status;
-
- memcpy(&hdl->ldev, ldev, sizeof(*ldev));
-
- iwdev->mpa_version = mpa_version;
- iwdev->resource_profile = (resource_profile < I40IW_HMC_PROFILE_EQUAL) ?
- (u8)resource_profile + I40IW_HMC_PROFILE_DEFAULT :
- I40IW_HMC_PROFILE_DEFAULT;
- iwdev->max_rdma_vfs =
- (iwdev->resource_profile != I40IW_HMC_PROFILE_DEFAULT) ? max_rdma_vfs : 0;
- iwdev->max_enabled_vfs = iwdev->max_rdma_vfs;
- iwdev->netdev = ldev->netdev;
- hdl->client = client;
- if (!ldev->ftype)
- iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_DB_ADDR_OFFSET;
- else
- iwdev->db_start = pci_resource_start(ldev->pcidev, 0) + I40IW_VF_DB_ADDR_OFFSET;
-
- status = i40iw_save_msix_info(iwdev, ldev);
- if (status)
- return status;
- iwdev->hw.dev_context = (void *)ldev->pcidev;
- iwdev->hw.hw_addr = ldev->hw_addr;
- status = i40iw_allocate_dma_mem(&iwdev->hw,
- &iwdev->obj_mem, 8192, 4096);
- if (status)
- goto exit;
- iwdev->obj_next = iwdev->obj_mem;
- iwdev->push_mode = push_mode;
-
- init_waitqueue_head(&iwdev->vchnl_waitq);
- init_waitqueue_head(&dev->vf_reqs);
- init_waitqueue_head(&iwdev->close_wq);
-
- status = i40iw_initialize_dev(iwdev, ldev);
-exit:
- if (status) {
- kfree(iwdev->iw_msixtbl);
- i40iw_free_dma_mem(dev->hw, &iwdev->obj_mem);
- iwdev->iw_msixtbl = NULL;
- }
- return status;
-}
-
-/**
- * i40iw_get_used_rsrc - determine resources used internally
- * @iwdev: iwarp device
- *
- * Called after internal allocations
- */
-static void i40iw_get_used_rsrc(struct i40iw_device *iwdev)
-{
- iwdev->used_pds = find_next_zero_bit(iwdev->allocated_pds, iwdev->max_pd, 0);
- iwdev->used_qps = find_next_zero_bit(iwdev->allocated_qps, iwdev->max_qp, 0);
- iwdev->used_cqs = find_next_zero_bit(iwdev->allocated_cqs, iwdev->max_cq, 0);
- iwdev->used_mrs = find_next_zero_bit(iwdev->allocated_mrs, iwdev->max_mr, 0);
-}
-
-/**
- * i40iw_open - client interface operation open for iwarp/uda device
- * @ldev: lan device information
- * @client: iwarp client information, provided during registration
- *
- * Called by the lan driver during the processing of client register
- * Create device resources, set up queues, pble and hmc objects and
- * register the device with the ib verbs interface
- * Return 0 if successful, otherwise return error
- */
-static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
-{
- struct i40iw_device *iwdev;
- struct i40iw_sc_dev *dev;
- enum i40iw_status_code status;
- struct i40iw_handler *hdl;
-
- hdl = i40iw_find_netdev(ldev->netdev);
- if (hdl)
- return 0;
-
- hdl = kzalloc(sizeof(*hdl), GFP_KERNEL);
- if (!hdl)
- return -ENOMEM;
- iwdev = &hdl->device;
- iwdev->hdl = hdl;
- dev = &iwdev->sc_dev;
- if (i40iw_setup_cm_core(iwdev)) {
- kfree(iwdev->hdl);
- return -ENOMEM;
- }
-
- dev->back_dev = (void *)iwdev;
- iwdev->ldev = &hdl->ldev;
- iwdev->client = client;
- mutex_init(&iwdev->pbl_mutex);
- i40iw_add_handler(hdl);
-
- do {
- status = i40iw_setup_init_state(hdl, ldev, client);
- if (status)
- break;
- iwdev->init_state = INITIAL_STATE;
- if (dev->is_pf)
- i40iw_wait_pe_ready(dev->hw);
- status = i40iw_create_cqp(iwdev);
- if (status)
- break;
- iwdev->init_state = CQP_CREATED;
- status = i40iw_hmc_setup(iwdev);
- if (status)
- break;
- status = i40iw_create_ccq(iwdev);
- if (status)
- break;
- iwdev->init_state = CCQ_CREATED;
- status = i40iw_initialize_ilq(iwdev);
- if (status)
- break;
- iwdev->init_state = ILQ_CREATED;
- status = i40iw_initialize_ieq(iwdev);
- if (status)
- break;
- iwdev->init_state = IEQ_CREATED;
- status = i40iw_setup_aeq(iwdev);
- if (status)
- break;
- iwdev->init_state = AEQ_CREATED;
- status = i40iw_setup_ceqs(iwdev, ldev);
- if (status)
- break;
-
- status = i40iw_get_rdma_features(dev);
- if (status)
- dev->feature_info[I40IW_FEATURE_FW_INFO] =
- I40IW_FW_VER_DEFAULT;
-
- iwdev->init_state = CEQ_CREATED;
- status = i40iw_initialize_hw_resources(iwdev);
- if (status)
- break;
- i40iw_get_used_rsrc(iwdev);
- dev->ccq_ops->ccq_arm(dev->ccq);
- status = i40iw_hmc_init_pble(&iwdev->sc_dev, iwdev->pble_rsrc);
- if (status)
- break;
- iwdev->init_state = PBLE_CHUNK_MEM;
- iwdev->virtchnl_wq = alloc_ordered_workqueue("iwvch", WQ_MEM_RECLAIM);
- status = i40iw_add_mac_ip(iwdev);
- if (status)
- break;
- iwdev->init_state = IP_ADDR_REGISTERED;
- if (i40iw_register_rdma_device(iwdev)) {
- i40iw_pr_err("register rdma device fail\n");
- break;
- };
-
- iwdev->init_state = RDMA_DEV_REGISTERED;
- iwdev->iw_status = 1;
- i40iw_port_ibevent(iwdev);
- iwdev->param_wq = alloc_ordered_workqueue("l2params", WQ_MEM_RECLAIM);
- if(iwdev->param_wq == NULL)
- break;
- i40iw_pr_info("i40iw_open completed\n");
- return 0;
- } while (0);
-
- i40iw_pr_err("status = %d last completion = %d\n", status, iwdev->init_state);
- i40iw_deinit_device(iwdev);
- return -ERESTART;
-}
-
-/**
- * i40iw_l2params_worker - worker for l2 params change
- * @work: work pointer for l2 params
- */
-static void i40iw_l2params_worker(struct work_struct *work)
-{
- struct l2params_work *dwork =
- container_of(work, struct l2params_work, work);
- struct i40iw_device *iwdev = dwork->iwdev;
-
- i40iw_change_l2params(&iwdev->vsi, &dwork->l2params);
- atomic_dec(&iwdev->params_busy);
- kfree(work);
-}
-
-/**
- * i40iw_l2param_change - handle qs handles for qos and mss change
- * @ldev: lan device information
- * @client: client for paramater change
- * @params: new parameters from L2
- */
-static void i40iw_l2param_change(struct i40e_info *ldev, struct i40e_client *client,
- struct i40e_params *params)
-{
- struct i40iw_handler *hdl;
- struct i40iw_l2params *l2params;
- struct l2params_work *work;
- struct i40iw_device *iwdev;
- int i;
-
- hdl = i40iw_find_i40e_handler(ldev);
- if (!hdl)
- return;
-
- iwdev = &hdl->device;
-
- if (atomic_read(&iwdev->params_busy))
- return;
-
-
- work = kzalloc(sizeof(*work), GFP_KERNEL);
- if (!work)
- return;
-
- atomic_inc(&iwdev->params_busy);
-
- work->iwdev = iwdev;
- l2params = &work->l2params;
- for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++)
- l2params->qs_handle_list[i] = params->qos.prio_qos[i].qs_handle;
-
- l2params->mtu = (params->mtu) ? params->mtu : iwdev->vsi.mtu;
-
- INIT_WORK(&work->work, i40iw_l2params_worker);
- queue_work(iwdev->param_wq, &work->work);
-}
-
-/**
- * i40iw_close - client interface operation close for iwarp/uda device
- * @ldev: lan device information
- * @client: client to close
- *
- * Called by the lan driver during the processing of client unregister
- * Destroy and clean up the driver resources
- */
-static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client, bool reset)
-{
- struct i40iw_device *iwdev;
- struct i40iw_handler *hdl;
-
- hdl = i40iw_find_i40e_handler(ldev);
- if (!hdl)
- return;
-
- iwdev = &hdl->device;
- iwdev->closing = true;
-
- if (reset)
- iwdev->reset = true;
-
- i40iw_cm_teardown_connections(iwdev, NULL, NULL, true);
- destroy_workqueue(iwdev->virtchnl_wq);
- i40iw_deinit_device(iwdev);
-}
-
-/**
- * i40iw_vf_reset - process VF reset
- * @ldev: lan device information
- * @client: client interface instance
- * @vf_id: virtual function id
- *
- * Called when a VF is reset by the PF
- * Destroy and clean up the VF resources
- */
-static void i40iw_vf_reset(struct i40e_info *ldev, struct i40e_client *client, u32 vf_id)
-{
- struct i40iw_handler *hdl;
- struct i40iw_sc_dev *dev;
- struct i40iw_hmc_fcn_info hmc_fcn_info;
- struct i40iw_virt_mem vf_dev_mem;
- struct i40iw_vfdev *tmp_vfdev;
- unsigned int i;
- unsigned long flags;
- struct i40iw_device *iwdev;
-
- hdl = i40iw_find_i40e_handler(ldev);
- if (!hdl)
- return;
-
- dev = &hdl->device.sc_dev;
- iwdev = (struct i40iw_device *)dev->back_dev;
-
- for (i = 0; i < I40IW_MAX_PE_ENABLED_VF_COUNT; i++) {
- if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id != vf_id))
- continue;
- /* free all resources allocated on behalf of vf */
- tmp_vfdev = dev->vf_dev[i];
- spin_lock_irqsave(&iwdev->vsi.pestat->lock, flags);
- dev->vf_dev[i] = NULL;
- spin_unlock_irqrestore(&iwdev->vsi.pestat->lock, flags);
- i40iw_del_hmc_objects(dev, &tmp_vfdev->hmc_info, false, false);
- /* remove vf hmc function */
- memset(&hmc_fcn_info, 0, sizeof(hmc_fcn_info));
- hmc_fcn_info.vf_id = vf_id;
- hmc_fcn_info.iw_vf_idx = tmp_vfdev->iw_vf_idx;
- hmc_fcn_info.free_fcn = true;
- i40iw_cqp_manage_hmc_fcn_cmd(dev, &hmc_fcn_info);
- /* free vf_dev */
- vf_dev_mem.va = tmp_vfdev;
- vf_dev_mem.size = sizeof(struct i40iw_vfdev) +
- sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX;
- i40iw_free_virt_mem(dev->hw, &vf_dev_mem);
- break;
- }
-}
-
-/**
- * i40iw_vf_enable - enable a number of VFs
- * @ldev: lan device information
- * @client: client interface instance
- * @num_vfs: number of VFs for the PF
- *
- * Called when the number of VFs changes
- */
-static void i40iw_vf_enable(struct i40e_info *ldev,
- struct i40e_client *client,
- u32 num_vfs)
-{
- struct i40iw_handler *hdl;
-
- hdl = i40iw_find_i40e_handler(ldev);
- if (!hdl)
- return;
-
- if (num_vfs > I40IW_MAX_PE_ENABLED_VF_COUNT)
- hdl->device.max_enabled_vfs = I40IW_MAX_PE_ENABLED_VF_COUNT;
- else
- hdl->device.max_enabled_vfs = num_vfs;
-}
-
-/**
- * i40iw_vf_capable - check if VF capable
- * @ldev: lan device information
- * @client: client interface instance
- * @vf_id: virtual function id
- *
- * Return 1 if a VF slot is available or if VF is already RDMA enabled
- * Return 0 otherwise
- */
-static int i40iw_vf_capable(struct i40e_info *ldev,
- struct i40e_client *client,
- u32 vf_id)
-{
- struct i40iw_handler *hdl;
- struct i40iw_sc_dev *dev;
- unsigned int i;
-
- hdl = i40iw_find_i40e_handler(ldev);
- if (!hdl)
- return 0;
-
- dev = &hdl->device.sc_dev;
-
- for (i = 0; i < hdl->device.max_enabled_vfs; i++) {
- if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id == vf_id))
- return 1;
- }
-
- return 0;
-}
-
-/**
- * i40iw_virtchnl_receive - receive a message through the virtual channel
- * @ldev: lan device information
- * @client: client interface instance
- * @vf_id: virtual function id associated with the message
- * @msg: message buffer pointer
- * @len: length of the message
- *
- * Invoke virtual channel receive operation for the given msg
- * Return 0 if successful, otherwise return error
- */
-static int i40iw_virtchnl_receive(struct i40e_info *ldev,
- struct i40e_client *client,
- u32 vf_id,
- u8 *msg,
- u16 len)
-{
- struct i40iw_handler *hdl;
- struct i40iw_sc_dev *dev;
- struct i40iw_device *iwdev;
- int ret_code = I40IW_NOT_SUPPORTED;
-
- if (!len || !msg)
- return I40IW_ERR_PARAM;
-
- hdl = i40iw_find_i40e_handler(ldev);
- if (!hdl)
- return I40IW_ERR_PARAM;
-
- dev = &hdl->device.sc_dev;
- iwdev = dev->back_dev;
-
- if (dev->vchnl_if.vchnl_recv) {
- ret_code = dev->vchnl_if.vchnl_recv(dev, vf_id, msg, len);
- if (!dev->is_pf) {
- atomic_dec(&iwdev->vchnl_msgs);
- wake_up(&iwdev->vchnl_waitq);
- }
- }
- return ret_code;
-}
-
-/**
- * i40iw_vf_clear_to_send - wait to send virtual channel message
- * @dev: iwarp device *
- * Wait for until virtual channel is clear
- * before sending the next message
- *
- * Returns false if error
- * Returns true if clear to send
- */
-bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev)
-{
- struct i40iw_device *iwdev;
- wait_queue_entry_t wait;
-
- iwdev = dev->back_dev;
-
- if (!wq_has_sleeper(&dev->vf_reqs) &&
- (atomic_read(&iwdev->vchnl_msgs) == 0))
- return true; /* virtual channel is clear */
-
- init_wait(&wait);
- add_wait_queue_exclusive(&dev->vf_reqs, &wait);
-
- if (!wait_event_timeout(dev->vf_reqs,
- (atomic_read(&iwdev->vchnl_msgs) == 0),
- I40IW_VCHNL_EVENT_TIMEOUT))
- dev->vchnl_up = false;
-
- remove_wait_queue(&dev->vf_reqs, &wait);
-
- return dev->vchnl_up;
-}
-
-/**
- * i40iw_virtchnl_send - send a message through the virtual channel
- * @dev: iwarp device
- * @vf_id: virtual function id associated with the message
- * @msg: virtual channel message buffer pointer
- * @len: length of the message
- *
- * Invoke virtual channel send operation for the given msg
- * Return 0 if successful, otherwise return error
- */
-static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev,
- u32 vf_id,
- u8 *msg,
- u16 len)
-{
- struct i40iw_device *iwdev;
- struct i40e_info *ldev;
-
- if (!dev || !dev->back_dev)
- return I40IW_ERR_BAD_PTR;
-
- iwdev = dev->back_dev;
- ldev = iwdev->ldev;
-
- if (ldev && ldev->ops && ldev->ops->virtchnl_send)
- return ldev->ops->virtchnl_send(ldev, &i40iw_client, vf_id, msg, len);
- return I40IW_ERR_BAD_PTR;
-}
-
-/* client interface functions */
-static const struct i40e_client_ops i40e_ops = {
- .open = i40iw_open,
- .close = i40iw_close,
- .l2_param_change = i40iw_l2param_change,
- .virtchnl_receive = i40iw_virtchnl_receive,
- .vf_reset = i40iw_vf_reset,
- .vf_enable = i40iw_vf_enable,
- .vf_capable = i40iw_vf_capable
-};
-
-/**
- * i40iw_init_module - driver initialization function
- *
- * First function to call when the driver is loaded
- * Register the driver as i40e client and port mapper client
- */
-static int __init i40iw_init_module(void)
-{
- int ret;
-
- memset(&i40iw_client, 0, sizeof(i40iw_client));
- i40iw_client.version.major = CLIENT_IW_INTERFACE_VERSION_MAJOR;
- i40iw_client.version.minor = CLIENT_IW_INTERFACE_VERSION_MINOR;
- i40iw_client.version.build = CLIENT_IW_INTERFACE_VERSION_BUILD;
- i40iw_client.ops = &i40e_ops;
- memcpy(i40iw_client.name, i40iw_client_name, I40E_CLIENT_STR_LENGTH);
- i40iw_client.type = I40E_CLIENT_IWARP;
- spin_lock_init(&i40iw_handler_lock);
- ret = i40e_register_client(&i40iw_client);
- i40iw_register_notifiers();
-
- return ret;
-}
-
-/**
- * i40iw_exit_module - driver exit clean up function
- *
- * The function is called just before the driver is unloaded
- * Unregister the driver as i40e client and port mapper client
- */
-static void __exit i40iw_exit_module(void)
-{
- i40iw_unregister_notifiers();
- i40e_unregister_client(&i40iw_client);
-}
-
-module_init(i40iw_init_module);
-module_exit(i40iw_exit_module);
diff --git a/drivers/infiniband/hw/i40iw/i40iw_osdep.h b/drivers/infiniband/hw/i40iw/i40iw_osdep.h
deleted file mode 100644
index d474aad62a81..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_osdep.h
+++ /dev/null
@@ -1,217 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_OSDEP_H
-#define I40IW_OSDEP_H
-
-#include <linux/kernel.h>
-#include <linux/string.h>
-#include <linux/bitops.h>
-#include <net/tcp.h>
-#include <crypto/hash.h>
-/* get readq/writeq support for 32 bit kernels, use the low-first version */
-#include <linux/io-64-nonatomic-lo-hi.h>
-
-#define STATS_TIMER_DELAY 1000
-
-static inline void set_64bit_val(u64 *wqe_words, u32 byte_index, u64 value)
-{
- wqe_words[byte_index >> 3] = value;
-}
-
-/**
- * set_32bit_val - set 32 value to hw wqe
- * @wqe_words: wqe addr to write
- * @byte_index: index in wqe
- * @value: value to write
- **/
-static inline void set_32bit_val(u32 *wqe_words, u32 byte_index, u32 value)
-{
- wqe_words[byte_index >> 2] = value;
-}
-
-/**
- * get_64bit_val - read 64 bit value from wqe
- * @wqe_words: wqe addr
- * @byte_index: index to read from
- * @value: read value
- **/
-static inline void get_64bit_val(u64 *wqe_words, u32 byte_index, u64 *value)
-{
- *value = wqe_words[byte_index >> 3];
-}
-
-/**
- * get_32bit_val - read 32 bit value from wqe
- * @wqe_words: wqe addr
- * @byte_index: index to reaad from
- * @value: return 32 bit value
- **/
-static inline void get_32bit_val(u32 *wqe_words, u32 byte_index, u32 *value)
-{
- *value = wqe_words[byte_index >> 2];
-}
-
-struct i40iw_dma_mem {
- void *va;
- dma_addr_t pa;
- u32 size;
-} __packed;
-
-struct i40iw_virt_mem {
- void *va;
- u32 size;
-} __packed;
-
-#define i40iw_debug(h, m, s, ...) \
-do { \
- if (((m) & (h)->debug_mask)) \
- pr_info("i40iw " s, ##__VA_ARGS__); \
-} while (0)
-
-#define i40iw_flush(a) readl((a)->hw_addr + I40E_GLGEN_STAT)
-
-#define I40E_GLHMC_VFSDCMD(_i) (0x000C8000 + ((_i) * 4)) \
- /* _i=0...31 */
-#define I40E_GLHMC_VFSDCMD_MAX_INDEX 31
-#define I40E_GLHMC_VFSDCMD_PMSDIDX_SHIFT 0
-#define I40E_GLHMC_VFSDCMD_PMSDIDX_MASK (0xFFF \
- << I40E_GLHMC_VFSDCMD_PMSDIDX_SHIFT)
-#define I40E_GLHMC_VFSDCMD_PF_SHIFT 16
-#define I40E_GLHMC_VFSDCMD_PF_MASK (0xF << I40E_GLHMC_VFSDCMD_PF_SHIFT)
-#define I40E_GLHMC_VFSDCMD_VF_SHIFT 20
-#define I40E_GLHMC_VFSDCMD_VF_MASK (0x1FF << I40E_GLHMC_VFSDCMD_VF_SHIFT)
-#define I40E_GLHMC_VFSDCMD_PMF_TYPE_SHIFT 29
-#define I40E_GLHMC_VFSDCMD_PMF_TYPE_MASK (0x3 \
- << I40E_GLHMC_VFSDCMD_PMF_TYPE_SHIFT)
-#define I40E_GLHMC_VFSDCMD_PMSDWR_SHIFT 31
-#define I40E_GLHMC_VFSDCMD_PMSDWR_MASK (0x1 << I40E_GLHMC_VFSDCMD_PMSDWR_SHIFT)
-
-#define I40E_GLHMC_VFSDDATAHIGH(_i) (0x000C8200 + ((_i) * 4)) \
- /* _i=0...31 */
-#define I40E_GLHMC_VFSDDATAHIGH_MAX_INDEX 31
-#define I40E_GLHMC_VFSDDATAHIGH_PMSDDATAHIGH_SHIFT 0
-#define I40E_GLHMC_VFSDDATAHIGH_PMSDDATAHIGH_MASK (0xFFFFFFFF \
- << I40E_GLHMC_VFSDDATAHIGH_PMSDDATAHIGH_SHIFT)
-
-#define I40E_GLHMC_VFSDDATALOW(_i) (0x000C8100 + ((_i) * 4)) \
- /* _i=0...31 */
-#define I40E_GLHMC_VFSDDATALOW_MAX_INDEX 31
-#define I40E_GLHMC_VFSDDATALOW_PMSDVALID_SHIFT 0
-#define I40E_GLHMC_VFSDDATALOW_PMSDVALID_MASK (0x1 \
- << I40E_GLHMC_VFSDDATALOW_PMSDVALID_SHIFT)
-#define I40E_GLHMC_VFSDDATALOW_PMSDTYPE_SHIFT 1
-#define I40E_GLHMC_VFSDDATALOW_PMSDTYPE_MASK (0x1 \
- << I40E_GLHMC_VFSDDATALOW_PMSDTYPE_SHIFT)
-#define I40E_GLHMC_VFSDDATALOW_PMSDBPCOUNT_SHIFT 2
-#define I40E_GLHMC_VFSDDATALOW_PMSDBPCOUNT_MASK (0x3FF \
- << I40E_GLHMC_VFSDDATALOW_PMSDBPCOUNT_SHIFT)
-#define I40E_GLHMC_VFSDDATALOW_PMSDDATALOW_SHIFT 12
-#define I40E_GLHMC_VFSDDATALOW_PMSDDATALOW_MASK (0xFFFFF \
- << I40E_GLHMC_VFSDDATALOW_PMSDDATALOW_SHIFT)
-
-#define I40E_GLPE_FWLDSTATUS 0x0000D200
-#define I40E_GLPE_FWLDSTATUS_LOAD_REQUESTED_SHIFT 0
-#define I40E_GLPE_FWLDSTATUS_LOAD_REQUESTED_MASK (0x1 \
- << I40E_GLPE_FWLDSTATUS_LOAD_REQUESTED_SHIFT)
-#define I40E_GLPE_FWLDSTATUS_DONE_SHIFT 1
-#define I40E_GLPE_FWLDSTATUS_DONE_MASK (0x1 << I40E_GLPE_FWLDSTATUS_DONE_SHIFT)
-#define I40E_GLPE_FWLDSTATUS_CQP_FAIL_SHIFT 2
-#define I40E_GLPE_FWLDSTATUS_CQP_FAIL_MASK (0x1 \
- << I40E_GLPE_FWLDSTATUS_CQP_FAIL_SHIFT)
-#define I40E_GLPE_FWLDSTATUS_TEP_FAIL_SHIFT 3
-#define I40E_GLPE_FWLDSTATUS_TEP_FAIL_MASK (0x1 \
- << I40E_GLPE_FWLDSTATUS_TEP_FAIL_SHIFT)
-#define I40E_GLPE_FWLDSTATUS_OOP_FAIL_SHIFT 4
-#define I40E_GLPE_FWLDSTATUS_OOP_FAIL_MASK (0x1 \
- << I40E_GLPE_FWLDSTATUS_OOP_FAIL_SHIFT)
-
-struct i40iw_sc_dev;
-struct i40iw_sc_qp;
-struct i40iw_puda_buf;
-struct i40iw_puda_completion_info;
-struct i40iw_update_sds_info;
-struct i40iw_hmc_fcn_info;
-struct i40iw_virtchnl_work_info;
-struct i40iw_manage_vf_pble_info;
-struct i40iw_device;
-struct i40iw_hmc_info;
-struct i40iw_hw;
-
-u8 __iomem *i40iw_get_hw_addr(void *dev);
-void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);
-enum i40iw_status_code i40iw_vf_wait_vchnl_resp(struct i40iw_sc_dev *dev);
-bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev);
-enum i40iw_status_code i40iw_ieq_check_mpacrc(struct shash_desc *desc, void *addr,
- u32 length, u32 value);
-struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *buf);
-void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length, u32 seqnum);
-void i40iw_free_hash_desc(struct shash_desc *);
-enum i40iw_status_code i40iw_init_hash_desc(struct shash_desc **);
-enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info,
- struct i40iw_puda_buf *buf);
-enum i40iw_status_code i40iw_cqp_sds_cmd(struct i40iw_sc_dev *dev,
- struct i40iw_update_sds_info *info);
-enum i40iw_status_code i40iw_cqp_manage_hmc_fcn_cmd(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_fcn_info *hmcfcninfo);
-enum i40iw_status_code i40iw_cqp_query_fpm_values_cmd(struct i40iw_sc_dev *dev,
- struct i40iw_dma_mem *values_mem,
- u8 hmc_fn_id);
-enum i40iw_status_code i40iw_cqp_commit_fpm_values_cmd(struct i40iw_sc_dev *dev,
- struct i40iw_dma_mem *values_mem,
- u8 hmc_fn_id);
-enum i40iw_status_code i40iw_alloc_query_fpm_buf(struct i40iw_sc_dev *dev,
- struct i40iw_dma_mem *mem);
-enum i40iw_status_code i40iw_cqp_manage_vf_pble_bp(struct i40iw_sc_dev *dev,
- struct i40iw_manage_vf_pble_info *info);
-void i40iw_cqp_spawn_worker(struct i40iw_sc_dev *dev,
- struct i40iw_virtchnl_work_info *work_info, u32 iw_vf_idx);
-void *i40iw_remove_head(struct list_head *list);
-void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp, bool suspend);
-
-void i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len);
-void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred);
-void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp);
-void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp);
-
-enum i40iw_status_code i40iw_hw_manage_vf_pble_bp(struct i40iw_device *iwdev,
- struct i40iw_manage_vf_pble_info *info,
- bool wait);
-struct i40iw_sc_vsi;
-void i40iw_hw_stats_start_timer(struct i40iw_sc_vsi *vsi);
-void i40iw_hw_stats_stop_timer(struct i40iw_sc_vsi *vsi);
-#define i40iw_mmiowb() do { } while (0)
-void i40iw_wr32(struct i40iw_hw *hw, u32 reg, u32 value);
-u32 i40iw_rd32(struct i40iw_hw *hw, u32 reg);
-#endif /* _I40IW_OSDEP_H_ */
diff --git a/drivers/infiniband/hw/i40iw/i40iw_p.h b/drivers/infiniband/hw/i40iw/i40iw_p.h
deleted file mode 100644
index 4c429567bbb4..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_p.h
+++ /dev/null
@@ -1,129 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_P_H
-#define I40IW_P_H
-
-#define PAUSE_TIMER_VALUE 0xFFFF
-#define REFRESH_THRESHOLD 0x7FFF
-#define HIGH_THRESHOLD 0x800
-#define LOW_THRESHOLD 0x200
-#define ALL_TC2PFC 0xFF
-#define CQP_COMPL_WAIT_TIME 0x3E8
-#define CQP_TIMEOUT_THRESHOLD 5
-
-void i40iw_debug_buf(struct i40iw_sc_dev *dev, enum i40iw_debug_flag mask,
- char *desc, u64 *buf, u32 size);
-/* init operations */
-enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,
- struct i40iw_device_init_info *info);
-
-void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp);
-
-u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch);
-
-void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev);
-
-enum i40iw_status_code i40iw_sc_mr_fast_register(struct i40iw_sc_qp *qp,
- struct i40iw_fast_reg_stag_info *info,
- bool post_sq);
-
-void i40iw_insert_wqe_hdr(u64 *wqe, u64 header);
-
-/* HMC/FPM functions */
-enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev,
- u8 hmc_fn_id);
-
-enum i40iw_status_code i40iw_pf_init_vfhmc(struct i40iw_sc_dev *dev, u8 vf_hmc_fn_id,
- u32 *vf_cnt_array);
-
-/* stats functions */
-void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats);
-void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats, struct i40iw_dev_hw_stats *stats_values);
-void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,
- enum i40iw_hw_stats_index_32b index,
- u64 *value);
-void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,
- enum i40iw_hw_stats_index_64b index,
- u64 *value);
-void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 index, bool is_pf);
-
-/* vsi misc functions */
-enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info);
-void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi);
-void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info);
-
-void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params);
-void i40iw_qp_add_qos(struct i40iw_sc_qp *qp);
-void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp);
-void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp);
-
-void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info);
-
-void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info);
-
-enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp,
- struct i40iw_sc_qp *qp, u64 scratch);
-
-enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp,
- struct i40iw_sc_qp *qp, u64 scratch);
-
-enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(struct i40iw_sc_cqp *cqp,
- u64 scratch, u8 hmc_fn_id,
- bool post_sq,
- bool poll_registers);
-
-enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count);
-enum i40iw_status_code i40iw_get_rdma_features(struct i40iw_sc_dev *dev);
-
-void free_sd_mem(struct i40iw_sc_dev *dev);
-
-enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,
- struct cqp_commands_info *pcmdinfo);
-
-enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev);
-
-/* prototype for functions used for dynamic memory allocation */
-enum i40iw_status_code i40iw_allocate_dma_mem(struct i40iw_hw *hw,
- struct i40iw_dma_mem *mem, u64 size,
- u32 alignment);
-void i40iw_free_dma_mem(struct i40iw_hw *hw, struct i40iw_dma_mem *mem);
-enum i40iw_status_code i40iw_allocate_virt_mem(struct i40iw_hw *hw,
- struct i40iw_virt_mem *mem, u32 size);
-enum i40iw_status_code i40iw_free_virt_mem(struct i40iw_hw *hw,
- struct i40iw_virt_mem *mem);
-u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq);
-void i40iw_reinitialize_ieq(struct i40iw_sc_dev *dev);
-
-#endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.c b/drivers/infiniband/hw/i40iw/i40iw_pble.c
deleted file mode 100644
index 540aab5e502d..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_pble.c
+++ /dev/null
@@ -1,612 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include "i40iw_status.h"
-#include "i40iw_osdep.h"
-#include "i40iw_register.h"
-#include "i40iw_hmc.h"
-
-#include "i40iw_d.h"
-#include "i40iw_type.h"
-#include "i40iw_p.h"
-
-#include <linux/pci.h>
-#include <linux/genalloc.h>
-#include <linux/vmalloc.h>
-#include "i40iw_pble.h"
-#include "i40iw.h"
-
-struct i40iw_device;
-static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_pble_rsrc *pble_rsrc);
-static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk);
-
-/**
- * i40iw_destroy_pble_pool - destroy pool during module unload
- * @pble_rsrc: pble resources
- */
-void i40iw_destroy_pble_pool(struct i40iw_sc_dev *dev, struct i40iw_hmc_pble_rsrc *pble_rsrc)
-{
- struct list_head *clist;
- struct list_head *tlist;
- struct i40iw_chunk *chunk;
- struct i40iw_pble_pool *pinfo = &pble_rsrc->pinfo;
-
- if (pinfo->pool) {
- list_for_each_safe(clist, tlist, &pinfo->clist) {
- chunk = list_entry(clist, struct i40iw_chunk, list);
- if (chunk->type == I40IW_VMALLOC)
- i40iw_free_vmalloc_mem(dev->hw, chunk);
- kfree(chunk);
- }
- gen_pool_destroy(pinfo->pool);
- }
-}
-
-/**
- * i40iw_hmc_init_pble - Initialize pble resources during module load
- * @dev: i40iw_sc_dev struct
- * @pble_rsrc: pble resources
- */
-enum i40iw_status_code i40iw_hmc_init_pble(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_pble_rsrc *pble_rsrc)
-{
- struct i40iw_hmc_info *hmc_info;
- u32 fpm_idx = 0;
-
- hmc_info = dev->hmc_info;
- pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].base;
- /* Now start the pble' on 4k boundary */
- if (pble_rsrc->fpm_base_addr & 0xfff)
- fpm_idx = (PAGE_SIZE - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3;
-
- pble_rsrc->unallocated_pble =
- hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt - fpm_idx;
- pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3);
-
- pble_rsrc->pinfo.pool_shift = POOL_SHIFT;
- pble_rsrc->pinfo.pool = gen_pool_create(pble_rsrc->pinfo.pool_shift, -1);
- INIT_LIST_HEAD(&pble_rsrc->pinfo.clist);
- if (!pble_rsrc->pinfo.pool)
- goto error;
-
- if (add_pble_pool(dev, pble_rsrc))
- goto error;
-
- return 0;
-
- error:i40iw_destroy_pble_pool(dev, pble_rsrc);
- return I40IW_ERR_NO_MEMORY;
-}
-
-/**
- * get_sd_pd_idx - Returns sd index, pd index and rel_pd_idx from fpm address
- * @ pble_rsrc: structure containing fpm address
- * @ idx: where to return indexes
- */
-static inline void get_sd_pd_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc,
- struct sd_pd_idx *idx)
-{
- idx->sd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_DIRECT_BP_SIZE;
- idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr) / I40IW_HMC_PAGED_BP_SIZE;
- idx->rel_pd_idx = (idx->pd_idx % I40IW_HMC_PD_CNT_IN_SD);
-}
-
-/**
- * add_sd_direct - add sd direct for pble
- * @dev: hardware control device structure
- * @pble_rsrc: pble resource ptr
- * @info: page info for sd
- */
-static enum i40iw_status_code add_sd_direct(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_pble_rsrc *pble_rsrc,
- struct i40iw_add_page_info *info)
-{
- enum i40iw_status_code ret_code = 0;
- struct sd_pd_idx *idx = &info->idx;
- struct i40iw_chunk *chunk = info->chunk;
- struct i40iw_hmc_info *hmc_info = info->hmc_info;
- struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry;
- u32 offset = 0;
-
- if (!sd_entry->valid) {
- if (dev->is_pf) {
- ret_code = i40iw_add_sd_table_entry(dev->hw, hmc_info,
- info->idx.sd_idx,
- I40IW_SD_TYPE_DIRECT,
- I40IW_HMC_DIRECT_BP_SIZE);
- if (ret_code)
- return ret_code;
- chunk->type = I40IW_DMA_COHERENT;
- }
- }
- offset = idx->rel_pd_idx << I40IW_HMC_PAGED_BP_SHIFT;
- chunk->size = info->pages << I40IW_HMC_PAGED_BP_SHIFT;
- chunk->vaddr = ((u8 *)sd_entry->u.bp.addr.va + offset);
- chunk->fpm_addr = pble_rsrc->next_fpm_addr;
- i40iw_debug(dev, I40IW_DEBUG_PBLE, "chunk_size[%d] = 0x%x vaddr=%p fpm_addr = %llx\n",
- chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr);
- return 0;
-}
-
-/**
- * i40iw_free_vmalloc_mem - free vmalloc during close
- * @hw: hw struct
- * @chunk: chunk information for vmalloc
- */
-static void i40iw_free_vmalloc_mem(struct i40iw_hw *hw, struct i40iw_chunk *chunk)
-{
- struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context;
- int i;
-
- if (!chunk->pg_cnt)
- goto done;
- for (i = 0; i < chunk->pg_cnt; i++)
- dma_unmap_page(&pcidev->dev, chunk->dmaaddrs[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
-
- done:
- kfree(chunk->dmaaddrs);
- chunk->dmaaddrs = NULL;
- vfree(chunk->vaddr);
- chunk->vaddr = NULL;
- chunk->type = 0;
-}
-
-/**
- * i40iw_get_vmalloc_mem - get 2M page for sd
- * @hw: hardware address
- * @chunk: chunk to adf
- * @pg_cnt: #of 4 K pages
- */
-static enum i40iw_status_code i40iw_get_vmalloc_mem(struct i40iw_hw *hw,
- struct i40iw_chunk *chunk,
- int pg_cnt)
-{
- struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context;
- struct page *page;
- u8 *addr;
- u32 size;
- int i;
-
- chunk->dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
- if (!chunk->dmaaddrs)
- return I40IW_ERR_NO_MEMORY;
- size = PAGE_SIZE * pg_cnt;
- chunk->vaddr = vmalloc(size);
- if (!chunk->vaddr) {
- kfree(chunk->dmaaddrs);
- chunk->dmaaddrs = NULL;
- return I40IW_ERR_NO_MEMORY;
- }
- chunk->size = size;
- addr = (u8 *)chunk->vaddr;
- for (i = 0; i < pg_cnt; i++) {
- page = vmalloc_to_page((void *)addr);
- if (!page)
- break;
- chunk->dmaaddrs[i] = dma_map_page(&pcidev->dev, page, 0,
- PAGE_SIZE, DMA_BIDIRECTIONAL);
- if (dma_mapping_error(&pcidev->dev, chunk->dmaaddrs[i]))
- break;
- addr += PAGE_SIZE;
- }
-
- chunk->pg_cnt = i;
- chunk->type = I40IW_VMALLOC;
- if (i == pg_cnt)
- return 0;
-
- i40iw_free_vmalloc_mem(hw, chunk);
- return I40IW_ERR_NO_MEMORY;
-}
-
-/**
- * fpm_to_idx - given fpm address, get pble index
- * @pble_rsrc: pble resource management
- * @addr: fpm address for index
- */
-static inline u32 fpm_to_idx(struct i40iw_hmc_pble_rsrc *pble_rsrc, u64 addr)
-{
- return (addr - (pble_rsrc->fpm_base_addr)) >> 3;
-}
-
-/**
- * add_bp_pages - add backing pages for sd
- * @dev: hardware control device structure
- * @pble_rsrc: pble resource management
- * @info: page info for sd
- */
-static enum i40iw_status_code add_bp_pages(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_pble_rsrc *pble_rsrc,
- struct i40iw_add_page_info *info)
-{
- u8 *addr;
- struct i40iw_dma_mem mem;
- struct i40iw_hmc_pd_entry *pd_entry;
- struct i40iw_hmc_sd_entry *sd_entry = info->sd_entry;
- struct i40iw_hmc_info *hmc_info = info->hmc_info;
- struct i40iw_chunk *chunk = info->chunk;
- struct i40iw_manage_vf_pble_info vf_pble_info;
- enum i40iw_status_code status = 0;
- u32 rel_pd_idx = info->idx.rel_pd_idx;
- u32 pd_idx = info->idx.pd_idx;
- u32 i;
-
- status = i40iw_get_vmalloc_mem(dev->hw, chunk, info->pages);
- if (status)
- return I40IW_ERR_NO_MEMORY;
- status = i40iw_add_sd_table_entry(dev->hw, hmc_info,
- info->idx.sd_idx, I40IW_SD_TYPE_PAGED,
- I40IW_HMC_DIRECT_BP_SIZE);
- if (status)
- goto error;
- if (!dev->is_pf) {
- status = i40iw_vchnl_vf_add_hmc_objs(dev, I40IW_HMC_IW_PBLE,
- fpm_to_idx(pble_rsrc,
- pble_rsrc->next_fpm_addr),
- (info->pages << PBLE_512_SHIFT));
- if (status) {
- i40iw_pr_err("allocate PBLEs in the PF. Error %i\n", status);
- goto error;
- }
- }
- addr = chunk->vaddr;
- for (i = 0; i < info->pages; i++) {
- mem.pa = chunk->dmaaddrs[i];
- mem.size = PAGE_SIZE;
- mem.va = (void *)(addr);
- pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++];
- if (!pd_entry->valid) {
- status = i40iw_add_pd_table_entry(dev->hw, hmc_info, pd_idx++, &mem);
- if (status)
- goto error;
- addr += PAGE_SIZE;
- } else {
- i40iw_pr_err("pd entry is valid expecting to be invalid\n");
- }
- }
- if (!dev->is_pf) {
- vf_pble_info.first_pd_index = info->idx.rel_pd_idx;
- vf_pble_info.inv_pd_ent = false;
- vf_pble_info.pd_entry_cnt = PBLE_PER_PAGE;
- vf_pble_info.pd_pl_pba = sd_entry->u.pd_table.pd_page_addr.pa;
- vf_pble_info.sd_index = info->idx.sd_idx;
- status = i40iw_hw_manage_vf_pble_bp(dev->back_dev,
- &vf_pble_info, true);
- if (status) {
- i40iw_pr_err("CQP manage VF PBLE BP failed. %i\n", status);
- goto error;
- }
- }
- chunk->fpm_addr = pble_rsrc->next_fpm_addr;
- return 0;
-error:
- i40iw_free_vmalloc_mem(dev->hw, chunk);
- return status;
-}
-
-/**
- * add_pble_pool - add a sd entry for pble resoure
- * @dev: hardware control device structure
- * @pble_rsrc: pble resource management
- */
-static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_pble_rsrc *pble_rsrc)
-{
- struct i40iw_hmc_sd_entry *sd_entry;
- struct i40iw_hmc_info *hmc_info;
- struct i40iw_chunk *chunk;
- struct i40iw_add_page_info info;
- struct sd_pd_idx *idx = &info.idx;
- enum i40iw_status_code ret_code = 0;
- enum i40iw_sd_entry_type sd_entry_type;
- u64 sd_reg_val = 0;
- u32 pages;
-
- if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE)
- return I40IW_ERR_NO_MEMORY;
- if (pble_rsrc->next_fpm_addr & 0xfff) {
- i40iw_pr_err("next fpm_addr %llx\n", pble_rsrc->next_fpm_addr);
- return I40IW_ERR_INVALID_PAGE_DESC_INDEX;
- }
- chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
- if (!chunk)
- return I40IW_ERR_NO_MEMORY;
- hmc_info = dev->hmc_info;
- chunk->fpm_addr = pble_rsrc->next_fpm_addr;
- get_sd_pd_idx(pble_rsrc, idx);
- sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx];
- pages = (idx->rel_pd_idx) ? (I40IW_HMC_PD_CNT_IN_SD -
- idx->rel_pd_idx) : I40IW_HMC_PD_CNT_IN_SD;
- pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT);
- info.chunk = chunk;
- info.hmc_info = hmc_info;
- info.pages = pages;
- info.sd_entry = sd_entry;
- if (!sd_entry->valid) {
- sd_entry_type = (!idx->rel_pd_idx &&
- (pages == I40IW_HMC_PD_CNT_IN_SD) &&
- dev->is_pf) ? I40IW_SD_TYPE_DIRECT : I40IW_SD_TYPE_PAGED;
- } else {
- sd_entry_type = sd_entry->entry_type;
- }
- i40iw_debug(dev, I40IW_DEBUG_PBLE,
- "pages = %d, unallocated_pble[%u] current_fpm_addr = %llx\n",
- pages, pble_rsrc->unallocated_pble, pble_rsrc->next_fpm_addr);
- i40iw_debug(dev, I40IW_DEBUG_PBLE, "sd_entry_type = %d sd_entry valid = %d\n",
- sd_entry_type, sd_entry->valid);
-
- if (sd_entry_type == I40IW_SD_TYPE_DIRECT)
- ret_code = add_sd_direct(dev, pble_rsrc, &info);
- if (ret_code)
- sd_entry_type = I40IW_SD_TYPE_PAGED;
- else
- pble_rsrc->stats_direct_sds++;
-
- if (sd_entry_type == I40IW_SD_TYPE_PAGED) {
- ret_code = add_bp_pages(dev, pble_rsrc, &info);
- if (ret_code)
- goto error;
- else
- pble_rsrc->stats_paged_sds++;
- }
-
- if (gen_pool_add_virt(pble_rsrc->pinfo.pool, (unsigned long)chunk->vaddr,
- (phys_addr_t)chunk->fpm_addr, chunk->size, -1)) {
- i40iw_pr_err("could not allocate memory by gen_pool_addr_virt()\n");
- ret_code = I40IW_ERR_NO_MEMORY;
- goto error;
- }
- pble_rsrc->next_fpm_addr += chunk->size;
- i40iw_debug(dev, I40IW_DEBUG_PBLE, "next_fpm_addr = %llx chunk_size[%u] = 0x%x\n",
- pble_rsrc->next_fpm_addr, chunk->size, chunk->size);
- pble_rsrc->unallocated_pble -= (chunk->size >> 3);
- list_add(&chunk->list, &pble_rsrc->pinfo.clist);
- sd_reg_val = (sd_entry_type == I40IW_SD_TYPE_PAGED) ?
- sd_entry->u.pd_table.pd_page_addr.pa : sd_entry->u.bp.addr.pa;
- if (sd_entry->valid)
- return 0;
- if (dev->is_pf) {
- ret_code = i40iw_hmc_sd_one(dev, hmc_info->hmc_fn_id,
- sd_reg_val, idx->sd_idx,
- sd_entry->entry_type, true);
- if (ret_code) {
- i40iw_pr_err("cqp cmd failed for sd (pbles)\n");
- goto error;
- }
- }
-
- sd_entry->valid = true;
- return 0;
- error:
- kfree(chunk);
- return ret_code;
-}
-
-/**
- * free_lvl2 - fee level 2 pble
- * @pble_rsrc: pble resource management
- * @palloc: level 2 pble allocation
- */
-static void free_lvl2(struct i40iw_hmc_pble_rsrc *pble_rsrc,
- struct i40iw_pble_alloc *palloc)
-{
- u32 i;
- struct gen_pool *pool;
- struct i40iw_pble_level2 *lvl2 = &palloc->level2;
- struct i40iw_pble_info *root = &lvl2->root;
- struct i40iw_pble_info *leaf = lvl2->leaf;
-
- pool = pble_rsrc->pinfo.pool;
-
- for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
- if (leaf->addr)
- gen_pool_free(pool, leaf->addr, (leaf->cnt << 3));
- else
- break;
- }
-
- if (root->addr)
- gen_pool_free(pool, root->addr, (root->cnt << 3));
-
- kfree(lvl2->leaf);
- lvl2->leaf = NULL;
-}
-
-/**
- * get_lvl2_pble - get level 2 pble resource
- * @pble_rsrc: pble resource management
- * @palloc: level 2 pble allocation
- * @pool: pool pointer
- */
-static enum i40iw_status_code get_lvl2_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,
- struct i40iw_pble_alloc *palloc,
- struct gen_pool *pool)
-{
- u32 lf4k, lflast, total, i;
- u32 pblcnt = PBLE_PER_PAGE;
- u64 *addr;
- struct i40iw_pble_level2 *lvl2 = &palloc->level2;
- struct i40iw_pble_info *root = &lvl2->root;
- struct i40iw_pble_info *leaf;
-
- /* number of full 512 (4K) leafs) */
- lf4k = palloc->total_cnt >> 9;
- lflast = palloc->total_cnt % PBLE_PER_PAGE;
- total = (lflast == 0) ? lf4k : lf4k + 1;
- lvl2->leaf_cnt = total;
-
- leaf = kzalloc((sizeof(*leaf) * total), GFP_ATOMIC);
- if (!leaf)
- return I40IW_ERR_NO_MEMORY;
- lvl2->leaf = leaf;
- /* allocate pbles for the root */
- root->addr = gen_pool_alloc(pool, (total << 3));
- if (!root->addr) {
- kfree(lvl2->leaf);
- lvl2->leaf = NULL;
- return I40IW_ERR_NO_MEMORY;
- }
- root->idx = fpm_to_idx(pble_rsrc,
- (u64)gen_pool_virt_to_phys(pool, root->addr));
- root->cnt = total;
- addr = (u64 *)root->addr;
- for (i = 0; i < total; i++, leaf++) {
- pblcnt = (lflast && ((i + 1) == total)) ? lflast : PBLE_PER_PAGE;
- leaf->addr = gen_pool_alloc(pool, (pblcnt << 3));
- if (!leaf->addr)
- goto error;
- leaf->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool, leaf->addr));
-
- leaf->cnt = pblcnt;
- *addr = (u64)leaf->idx;
- addr++;
- }
- palloc->level = I40IW_LEVEL_2;
- pble_rsrc->stats_lvl2++;
- return 0;
- error:
- free_lvl2(pble_rsrc, palloc);
- return I40IW_ERR_NO_MEMORY;
-}
-
-/**
- * get_lvl1_pble - get level 1 pble resource
- * @dev: hardware control device structure
- * @pble_rsrc: pble resource management
- * @palloc: level 1 pble allocation
- */
-static enum i40iw_status_code get_lvl1_pble(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_pble_rsrc *pble_rsrc,
- struct i40iw_pble_alloc *palloc)
-{
- u64 *addr;
- struct gen_pool *pool;
- struct i40iw_pble_info *lvl1 = &palloc->level1;
-
- pool = pble_rsrc->pinfo.pool;
- addr = (u64 *)gen_pool_alloc(pool, (palloc->total_cnt << 3));
-
- if (!addr)
- return I40IW_ERR_NO_MEMORY;
-
- palloc->level = I40IW_LEVEL_1;
- lvl1->addr = (unsigned long)addr;
- lvl1->idx = fpm_to_idx(pble_rsrc, (u64)gen_pool_virt_to_phys(pool,
- (unsigned long)addr));
- lvl1->cnt = palloc->total_cnt;
- pble_rsrc->stats_lvl1++;
- return 0;
-}
-
-/**
- * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine
- * @dev: i40iw_sc_dev struct
- * @pble_rsrc: pble resources
- * @palloc: contains all inforamtion regarding pble (idx + pble addr)
- * @pool: pointer to general purpose special memory pool descriptor
- */
-static inline enum i40iw_status_code get_lvl1_lvl2_pble(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_pble_rsrc *pble_rsrc,
- struct i40iw_pble_alloc *palloc,
- struct gen_pool *pool)
-{
- enum i40iw_status_code status = 0;
-
- status = get_lvl1_pble(dev, pble_rsrc, palloc);
- if (status && (palloc->total_cnt > PBLE_PER_PAGE))
- status = get_lvl2_pble(pble_rsrc, palloc, pool);
- return status;
-}
-
-/**
- * i40iw_get_pble - allocate pbles from the pool
- * @dev: i40iw_sc_dev struct
- * @pble_rsrc: pble resources
- * @palloc: contains all inforamtion regarding pble (idx + pble addr)
- * @pble_cnt: #of pbles requested
- */
-enum i40iw_status_code i40iw_get_pble(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_pble_rsrc *pble_rsrc,
- struct i40iw_pble_alloc *palloc,
- u32 pble_cnt)
-{
- struct gen_pool *pool;
- enum i40iw_status_code status = 0;
- u32 max_sds = 0;
- int i;
-
- pool = pble_rsrc->pinfo.pool;
- palloc->total_cnt = pble_cnt;
- palloc->level = I40IW_LEVEL_0;
- /*check first to see if we can get pble's without acquiring additional sd's */
- status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);
- if (!status)
- goto exit;
- max_sds = (palloc->total_cnt >> 18) + 1;
- for (i = 0; i < max_sds; i++) {
- status = add_pble_pool(dev, pble_rsrc);
- if (status)
- break;
- status = get_lvl1_lvl2_pble(dev, pble_rsrc, palloc, pool);
- if (!status)
- break;
- }
-exit:
- if (!status)
- pble_rsrc->stats_alloc_ok++;
- else
- pble_rsrc->stats_alloc_fail++;
-
- return status;
-}
-
-/**
- * i40iw_free_pble - put pbles back into pool
- * @pble_rsrc: pble resources
- * @palloc: contains all inforamtion regarding pble resource being freed
- */
-void i40iw_free_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc,
- struct i40iw_pble_alloc *palloc)
-{
- struct gen_pool *pool;
-
- pool = pble_rsrc->pinfo.pool;
- if (palloc->level == I40IW_LEVEL_2)
- free_lvl2(pble_rsrc, palloc);
- else
- gen_pool_free(pool, palloc->level1.addr,
- (palloc->level1.cnt << 3));
- pble_rsrc->stats_alloc_freed++;
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.h b/drivers/infiniband/hw/i40iw/i40iw_pble.h
deleted file mode 100644
index 7b1851d21cc0..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_pble.h
+++ /dev/null
@@ -1,131 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_PBLE_H
-#define I40IW_PBLE_H
-
-#define POOL_SHIFT 6
-#define PBLE_PER_PAGE 512
-#define I40IW_HMC_PAGED_BP_SHIFT 12
-#define PBLE_512_SHIFT 9
-
-enum i40iw_pble_level {
- I40IW_LEVEL_0 = 0,
- I40IW_LEVEL_1 = 1,
- I40IW_LEVEL_2 = 2
-};
-
-enum i40iw_alloc_type {
- I40IW_NO_ALLOC = 0,
- I40IW_DMA_COHERENT = 1,
- I40IW_VMALLOC = 2
-};
-
-struct i40iw_pble_info {
- unsigned long addr;
- u32 idx;
- u32 cnt;
-};
-
-struct i40iw_pble_level2 {
- struct i40iw_pble_info root;
- struct i40iw_pble_info *leaf;
- u32 leaf_cnt;
-};
-
-struct i40iw_pble_alloc {
- u32 total_cnt;
- enum i40iw_pble_level level;
- union {
- struct i40iw_pble_info level1;
- struct i40iw_pble_level2 level2;
- };
-};
-
-struct sd_pd_idx {
- u32 sd_idx;
- u32 pd_idx;
- u32 rel_pd_idx;
-};
-
-struct i40iw_add_page_info {
- struct i40iw_chunk *chunk;
- struct i40iw_hmc_sd_entry *sd_entry;
- struct i40iw_hmc_info *hmc_info;
- struct sd_pd_idx idx;
- u32 pages;
-};
-
-struct i40iw_chunk {
- struct list_head list;
- u32 size;
- void *vaddr;
- u64 fpm_addr;
- u32 pg_cnt;
- dma_addr_t *dmaaddrs;
- enum i40iw_alloc_type type;
-};
-
-struct i40iw_pble_pool {
- struct gen_pool *pool;
- struct list_head clist;
- u32 total_pble_alloc;
- u32 free_pble_cnt;
- u32 pool_shift;
-};
-
-struct i40iw_hmc_pble_rsrc {
- u32 unallocated_pble;
- u64 fpm_base_addr;
- u64 next_fpm_addr;
- struct i40iw_pble_pool pinfo;
-
- u32 stats_direct_sds;
- u32 stats_paged_sds;
- u64 stats_alloc_ok;
- u64 stats_alloc_fail;
- u64 stats_alloc_freed;
- u64 stats_lvl1;
- u64 stats_lvl2;
-};
-
-void i40iw_destroy_pble_pool(struct i40iw_sc_dev *dev, struct i40iw_hmc_pble_rsrc *pble_rsrc);
-enum i40iw_status_code i40iw_hmc_init_pble(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_pble_rsrc *pble_rsrc);
-void i40iw_free_pble(struct i40iw_hmc_pble_rsrc *pble_rsrc, struct i40iw_pble_alloc *palloc);
-enum i40iw_status_code i40iw_get_pble(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_pble_rsrc *pble_rsrc,
- struct i40iw_pble_alloc *palloc,
- u32 pble_cnt);
-#endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.c b/drivers/infiniband/hw/i40iw/i40iw_puda.c
deleted file mode 100644
index d9c7ae6a7030..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_puda.c
+++ /dev/null
@@ -1,1493 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include "i40iw_osdep.h"
-#include "i40iw_register.h"
-#include "i40iw_status.h"
-#include "i40iw_hmc.h"
-
-#include "i40iw_d.h"
-#include "i40iw_type.h"
-#include "i40iw_p.h"
-#include "i40iw_puda.h"
-
-static void i40iw_ieq_receive(struct i40iw_sc_vsi *vsi,
- struct i40iw_puda_buf *buf);
-static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi *vsi, void *sqwrid);
-static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx);
-static enum i40iw_status_code i40iw_puda_replenish_rq(struct i40iw_puda_rsrc
- *rsrc, bool initial);
-/**
- * i40iw_puda_get_listbuf - get buffer from puda list
- * @list: list to use for buffers (ILQ or IEQ)
- */
-static struct i40iw_puda_buf *i40iw_puda_get_listbuf(struct list_head *list)
-{
- struct i40iw_puda_buf *buf = NULL;
-
- if (!list_empty(list)) {
- buf = (struct i40iw_puda_buf *)list->next;
- list_del((struct list_head *)&buf->list);
- }
- return buf;
-}
-
-/**
- * i40iw_puda_get_bufpool - return buffer from resource
- * @rsrc: resource to use for buffer
- */
-struct i40iw_puda_buf *i40iw_puda_get_bufpool(struct i40iw_puda_rsrc *rsrc)
-{
- struct i40iw_puda_buf *buf = NULL;
- struct list_head *list = &rsrc->bufpool;
- unsigned long flags;
-
- spin_lock_irqsave(&rsrc->bufpool_lock, flags);
- buf = i40iw_puda_get_listbuf(list);
- if (buf)
- rsrc->avail_buf_count--;
- else
- rsrc->stats_buf_alloc_fail++;
- spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
- return buf;
-}
-
-/**
- * i40iw_puda_ret_bufpool - return buffer to rsrc list
- * @rsrc: resource to use for buffer
- * @buf: buffe to return to resouce
- */
-void i40iw_puda_ret_bufpool(struct i40iw_puda_rsrc *rsrc,
- struct i40iw_puda_buf *buf)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&rsrc->bufpool_lock, flags);
- list_add(&buf->list, &rsrc->bufpool);
- spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
- rsrc->avail_buf_count++;
-}
-
-/**
- * i40iw_puda_post_recvbuf - set wqe for rcv buffer
- * @rsrc: resource ptr
- * @wqe_idx: wqe index to use
- * @buf: puda buffer for rcv q
- * @initial: flag if during init time
- */
-static void i40iw_puda_post_recvbuf(struct i40iw_puda_rsrc *rsrc, u32 wqe_idx,
- struct i40iw_puda_buf *buf, bool initial)
-{
- u64 *wqe;
- struct i40iw_sc_qp *qp = &rsrc->qp;
- u64 offset24 = 0;
-
- qp->qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf;
- wqe = qp->qp_uk.rq_base[wqe_idx].elem;
- i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA,
- "%s: wqe_idx= %d buf = %p wqe = %p\n", __func__,
- wqe_idx, buf, wqe);
- if (!initial)
- get_64bit_val(wqe, 24, &offset24);
-
- offset24 = (offset24) ? 0 : LS_64(1, I40IWQPSQ_VALID);
-
- set_64bit_val(wqe, 0, buf->mem.pa);
- set_64bit_val(wqe, 8,
- LS_64(buf->mem.size, I40IWQPSQ_FRAG_LEN));
- i40iw_insert_wqe_hdr(wqe, offset24);
-}
-
-/**
- * i40iw_puda_replenish_rq - post rcv buffers
- * @rsrc: resource to use for buffer
- * @initial: flag if during init time
- */
-static enum i40iw_status_code i40iw_puda_replenish_rq(struct i40iw_puda_rsrc *rsrc,
- bool initial)
-{
- u32 i;
- u32 invalid_cnt = rsrc->rxq_invalid_cnt;
- struct i40iw_puda_buf *buf = NULL;
-
- for (i = 0; i < invalid_cnt; i++) {
- buf = i40iw_puda_get_bufpool(rsrc);
- if (!buf)
- return I40IW_ERR_list_empty;
- i40iw_puda_post_recvbuf(rsrc, rsrc->rx_wqe_idx, buf,
- initial);
- rsrc->rx_wqe_idx =
- ((rsrc->rx_wqe_idx + 1) % rsrc->rq_size);
- rsrc->rxq_invalid_cnt--;
- }
- return 0;
-}
-
-/**
- * i40iw_puda_alloc_buf - allocate mem for buffer
- * @dev: iwarp device
- * @length: length of buffer
- */
-static struct i40iw_puda_buf *i40iw_puda_alloc_buf(struct i40iw_sc_dev *dev,
- u32 length)
-{
- struct i40iw_puda_buf *buf = NULL;
- struct i40iw_virt_mem buf_mem;
- enum i40iw_status_code ret;
-
- ret = i40iw_allocate_virt_mem(dev->hw, &buf_mem,
- sizeof(struct i40iw_puda_buf));
- if (ret) {
- i40iw_debug(dev, I40IW_DEBUG_PUDA,
- "%s: error mem for buf\n", __func__);
- return NULL;
- }
- buf = (struct i40iw_puda_buf *)buf_mem.va;
- ret = i40iw_allocate_dma_mem(dev->hw, &buf->mem, length, 1);
- if (ret) {
- i40iw_debug(dev, I40IW_DEBUG_PUDA,
- "%s: error dma mem for buf\n", __func__);
- i40iw_free_virt_mem(dev->hw, &buf_mem);
- return NULL;
- }
- buf->buf_mem.va = buf_mem.va;
- buf->buf_mem.size = buf_mem.size;
- return buf;
-}
-
-/**
- * i40iw_puda_dele_buf - delete buffer back to system
- * @dev: iwarp device
- * @buf: buffer to free
- */
-static void i40iw_puda_dele_buf(struct i40iw_sc_dev *dev,
- struct i40iw_puda_buf *buf)
-{
- i40iw_free_dma_mem(dev->hw, &buf->mem);
- i40iw_free_virt_mem(dev->hw, &buf->buf_mem);
-}
-
-/**
- * i40iw_puda_get_next_send_wqe - return next wqe for processing
- * @qp: puda qp for wqe
- * @wqe_idx: wqe index for caller
- */
-static u64 *i40iw_puda_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx)
-{
- u64 *wqe = NULL;
- enum i40iw_status_code ret_code = 0;
-
- *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- if (!*wqe_idx)
- qp->swqe_polarity = !qp->swqe_polarity;
- I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
- if (ret_code)
- return wqe;
- wqe = qp->sq_base[*wqe_idx].elem;
-
- return wqe;
-}
-
-/**
- * i40iw_puda_poll_info - poll cq for completion
- * @cq: cq for poll
- * @info: info return for successful completion
- */
-static enum i40iw_status_code i40iw_puda_poll_info(struct i40iw_sc_cq *cq,
- struct i40iw_puda_completion_info *info)
-{
- u64 qword0, qword2, qword3;
- u64 *cqe;
- u64 comp_ctx;
- bool valid_bit;
- u32 major_err, minor_err;
- bool error;
-
- cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&cq->cq_uk);
- get_64bit_val(cqe, 24, &qword3);
- valid_bit = (bool)RS_64(qword3, I40IW_CQ_VALID);
-
- if (valid_bit != cq->cq_uk.polarity)
- return I40IW_ERR_QUEUE_EMPTY;
-
- i40iw_debug_buf(cq->dev, I40IW_DEBUG_PUDA, "PUDA CQE", cqe, 32);
- error = (bool)RS_64(qword3, I40IW_CQ_ERROR);
- if (error) {
- i40iw_debug(cq->dev, I40IW_DEBUG_PUDA, "%s receive error\n", __func__);
- major_err = (u32)(RS_64(qword3, I40IW_CQ_MAJERR));
- minor_err = (u32)(RS_64(qword3, I40IW_CQ_MINERR));
- info->compl_error = major_err << 16 | minor_err;
- return I40IW_ERR_CQ_COMPL_ERROR;
- }
-
- get_64bit_val(cqe, 0, &qword0);
- get_64bit_val(cqe, 16, &qword2);
-
- info->q_type = (u8)RS_64(qword3, I40IW_CQ_SQ);
- info->qp_id = (u32)RS_64(qword2, I40IWCQ_QPID);
-
- get_64bit_val(cqe, 8, &comp_ctx);
- info->qp = (struct i40iw_qp_uk *)(unsigned long)comp_ctx;
- info->wqe_idx = (u32)RS_64(qword3, I40IW_CQ_WQEIDX);
-
- if (info->q_type == I40IW_CQE_QTYPE_RQ) {
- info->vlan_valid = (bool)RS_64(qword3, I40IW_VLAN_TAG_VALID);
- info->l4proto = (u8)RS_64(qword2, I40IW_UDA_L4PROTO);
- info->l3proto = (u8)RS_64(qword2, I40IW_UDA_L3PROTO);
- info->payload_len = (u16)RS_64(qword0, I40IW_UDA_PAYLOADLEN);
- }
-
- return 0;
-}
-
-/**
- * i40iw_puda_poll_completion - processes completion for cq
- * @dev: iwarp device
- * @cq: cq getting interrupt
- * @compl_err: return any completion err
- */
-enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev,
- struct i40iw_sc_cq *cq, u32 *compl_err)
-{
- struct i40iw_qp_uk *qp;
- struct i40iw_cq_uk *cq_uk = &cq->cq_uk;
- struct i40iw_puda_completion_info info;
- enum i40iw_status_code ret = 0;
- struct i40iw_puda_buf *buf;
- struct i40iw_puda_rsrc *rsrc;
- void *sqwrid;
- u8 cq_type = cq->cq_type;
- unsigned long flags;
-
- if ((cq_type == I40IW_CQ_TYPE_ILQ) || (cq_type == I40IW_CQ_TYPE_IEQ)) {
- rsrc = (cq_type == I40IW_CQ_TYPE_ILQ) ? cq->vsi->ilq : cq->vsi->ieq;
- } else {
- i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s qp_type error\n", __func__);
- return I40IW_ERR_BAD_PTR;
- }
- memset(&info, 0, sizeof(info));
- ret = i40iw_puda_poll_info(cq, &info);
- *compl_err = info.compl_error;
- if (ret == I40IW_ERR_QUEUE_EMPTY)
- return ret;
- if (ret)
- goto done;
-
- qp = info.qp;
- if (!qp || !rsrc) {
- ret = I40IW_ERR_BAD_PTR;
- goto done;
- }
-
- if (qp->qp_id != rsrc->qp_id) {
- ret = I40IW_ERR_BAD_PTR;
- goto done;
- }
-
- if (info.q_type == I40IW_CQE_QTYPE_RQ) {
- buf = (struct i40iw_puda_buf *)(uintptr_t)qp->rq_wrid_array[info.wqe_idx];
- /* Get all the tcpip information in the buf header */
- ret = i40iw_puda_get_tcpip_info(&info, buf);
- if (ret) {
- rsrc->stats_rcvd_pkt_err++;
- if (cq_type == I40IW_CQ_TYPE_ILQ) {
- i40iw_ilq_putback_rcvbuf(&rsrc->qp,
- info.wqe_idx);
- } else {
- i40iw_puda_ret_bufpool(rsrc, buf);
- i40iw_puda_replenish_rq(rsrc, false);
- }
- goto done;
- }
-
- rsrc->stats_pkt_rcvd++;
- rsrc->compl_rxwqe_idx = info.wqe_idx;
- i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s RQ completion\n", __func__);
- rsrc->receive(rsrc->vsi, buf);
- if (cq_type == I40IW_CQ_TYPE_ILQ)
- i40iw_ilq_putback_rcvbuf(&rsrc->qp, info.wqe_idx);
- else
- i40iw_puda_replenish_rq(rsrc, false);
-
- } else {
- i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s SQ completion\n", __func__);
- sqwrid = (void *)(uintptr_t)qp->sq_wrtrk_array[info.wqe_idx].wrid;
- I40IW_RING_SET_TAIL(qp->sq_ring, info.wqe_idx);
- rsrc->xmit_complete(rsrc->vsi, sqwrid);
- spin_lock_irqsave(&rsrc->bufpool_lock, flags);
- rsrc->tx_wqe_avail_cnt++;
- spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
- if (!list_empty(&rsrc->txpend))
- i40iw_puda_send_buf(rsrc, NULL);
- }
-
-done:
- I40IW_RING_MOVE_HEAD(cq_uk->cq_ring, ret);
- if (I40IW_RING_GETCURRENT_HEAD(cq_uk->cq_ring) == 0)
- cq_uk->polarity = !cq_uk->polarity;
- /* update cq tail in cq shadow memory also */
- I40IW_RING_MOVE_TAIL(cq_uk->cq_ring);
- set_64bit_val(cq_uk->shadow_area, 0,
- I40IW_RING_GETCURRENT_HEAD(cq_uk->cq_ring));
- return 0;
-}
-
-/**
- * i40iw_puda_send - complete send wqe for transmit
- * @qp: puda qp for send
- * @info: buffer information for transmit
- */
-enum i40iw_status_code i40iw_puda_send(struct i40iw_sc_qp *qp,
- struct i40iw_puda_send_info *info)
-{
- u64 *wqe;
- u32 iplen, l4len;
- u64 header[2];
- u32 wqe_idx;
- u8 iipt;
-
- /* number of 32 bits DWORDS in header */
- l4len = info->tcplen >> 2;
- if (info->ipv4) {
- iipt = 3;
- iplen = 5;
- } else {
- iipt = 1;
- iplen = 10;
- }
-
- wqe = i40iw_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info->scratch;
- /* Third line of WQE descriptor */
- /* maclen is in words */
- header[0] = LS_64((info->maclen >> 1), I40IW_UDA_QPSQ_MACLEN) |
- LS_64(iplen, I40IW_UDA_QPSQ_IPLEN) | LS_64(1, I40IW_UDA_QPSQ_L4T) |
- LS_64(iipt, I40IW_UDA_QPSQ_IIPT) |
- LS_64(l4len, I40IW_UDA_QPSQ_L4LEN);
- /* Forth line of WQE descriptor */
- header[1] = LS_64(I40IW_OP_TYPE_SEND, I40IW_UDA_QPSQ_OPCODE) |
- LS_64(1, I40IW_UDA_QPSQ_SIGCOMPL) |
- LS_64(info->doloopback, I40IW_UDA_QPSQ_DOLOOPBACK) |
- LS_64(qp->qp_uk.swqe_polarity, I40IW_UDA_QPSQ_VALID);
-
- set_64bit_val(wqe, 0, info->paddr);
- set_64bit_val(wqe, 8, LS_64(info->len, I40IWQPSQ_FRAG_LEN));
- set_64bit_val(wqe, 16, header[0]);
-
- i40iw_insert_wqe_hdr(wqe, header[1]);
-
- i40iw_debug_buf(qp->dev, I40IW_DEBUG_PUDA, "PUDA SEND WQE", wqe, 32);
- i40iw_qp_post_wr(&qp->qp_uk);
- return 0;
-}
-
-/**
- * i40iw_puda_send_buf - transmit puda buffer
- * @rsrc: resource to use for buffer
- * @buf: puda buffer to transmit
- */
-void i40iw_puda_send_buf(struct i40iw_puda_rsrc *rsrc, struct i40iw_puda_buf *buf)
-{
- struct i40iw_puda_send_info info;
- enum i40iw_status_code ret = 0;
- unsigned long flags;
-
- spin_lock_irqsave(&rsrc->bufpool_lock, flags);
- /* if no wqe available or not from a completion and we have
- * pending buffers, we must queue new buffer
- */
- if (!rsrc->tx_wqe_avail_cnt || (buf && !list_empty(&rsrc->txpend))) {
- list_add_tail(&buf->list, &rsrc->txpend);
- spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
- rsrc->stats_sent_pkt_q++;
- if (rsrc->type == I40IW_PUDA_RSRC_TYPE_ILQ)
- i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA,
- "%s: adding to txpend\n", __func__);
- return;
- }
- rsrc->tx_wqe_avail_cnt--;
- /* if we are coming from a completion and have pending buffers
- * then Get one from pending list
- */
- if (!buf) {
- buf = i40iw_puda_get_listbuf(&rsrc->txpend);
- if (!buf)
- goto done;
- }
-
- info.scratch = (void *)buf;
- info.paddr = buf->mem.pa;
- info.len = buf->totallen;
- info.tcplen = buf->tcphlen;
- info.maclen = buf->maclen;
- info.ipv4 = buf->ipv4;
- info.doloopback = (rsrc->type == I40IW_PUDA_RSRC_TYPE_IEQ);
-
- ret = i40iw_puda_send(&rsrc->qp, &info);
- if (ret) {
- rsrc->tx_wqe_avail_cnt++;
- rsrc->stats_sent_pkt_q++;
- list_add(&buf->list, &rsrc->txpend);
- if (rsrc->type == I40IW_PUDA_RSRC_TYPE_ILQ)
- i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA,
- "%s: adding to puda_send\n", __func__);
- } else {
- rsrc->stats_pkt_sent++;
- }
-done:
- spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
-}
-
-/**
- * i40iw_puda_qp_setctx - during init, set qp's context
- * @rsrc: qp's resource
- */
-static void i40iw_puda_qp_setctx(struct i40iw_puda_rsrc *rsrc)
-{
- struct i40iw_sc_qp *qp = &rsrc->qp;
- u64 *qp_ctx = qp->hw_host_ctx;
-
- set_64bit_val(qp_ctx, 8, qp->sq_pa);
- set_64bit_val(qp_ctx, 16, qp->rq_pa);
-
- set_64bit_val(qp_ctx, 24,
- LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) |
- LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE));
-
- set_64bit_val(qp_ctx, 48, LS_64(rsrc->buf_size, I40IW_UDA_QPC_MAXFRAMESIZE));
- set_64bit_val(qp_ctx, 56, 0);
- set_64bit_val(qp_ctx, 64, 1);
-
- set_64bit_val(qp_ctx, 136,
- LS_64(rsrc->cq_id, I40IWQPC_TXCQNUM) |
- LS_64(rsrc->cq_id, I40IWQPC_RXCQNUM));
-
- set_64bit_val(qp_ctx, 160, LS_64(1, I40IWQPC_PRIVEN));
-
- set_64bit_val(qp_ctx, 168,
- LS_64((uintptr_t)qp, I40IWQPC_QPCOMPCTX));
-
- set_64bit_val(qp_ctx, 176,
- LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) |
- LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) |
- LS_64(qp->qs_handle, I40IWQPC_QSHANDLE));
-
- i40iw_debug_buf(rsrc->dev, I40IW_DEBUG_PUDA, "PUDA QP CONTEXT",
- qp_ctx, I40IW_QP_CTX_SIZE);
-}
-
-/**
- * i40iw_puda_qp_wqe - setup wqe for qp create
- * @rsrc: resource for qp
- */
-static enum i40iw_status_code i40iw_puda_qp_wqe(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp)
-{
- struct i40iw_sc_cqp *cqp;
- u64 *wqe;
- u64 header;
- struct i40iw_ccq_cqe_info compl_info;
- enum i40iw_status_code status = 0;
-
- cqp = dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
- set_64bit_val(wqe, 40, qp->shadow_area_pa);
- header = qp->qp_uk.qp_id |
- LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) |
- LS_64(I40IW_QP_TYPE_UDA, I40IW_CQPSQ_QP_QPTYPE) |
- LS_64(1, I40IW_CQPSQ_QP_CQNUMVALID) |
- LS_64(2, I40IW_CQPSQ_QP_NEXTIWSTATE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
-
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_PUDA, "PUDA CQE", wqe, 32);
- i40iw_sc_cqp_post_sq(cqp);
- status = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
- I40IW_CQP_OP_CREATE_QP,
- &compl_info);
- return status;
-}
-
-/**
- * i40iw_puda_qp_create - create qp for resource
- * @rsrc: resource to use for buffer
- */
-static enum i40iw_status_code i40iw_puda_qp_create(struct i40iw_puda_rsrc *rsrc)
-{
- struct i40iw_sc_qp *qp = &rsrc->qp;
- struct i40iw_qp_uk *ukqp = &qp->qp_uk;
- enum i40iw_status_code ret = 0;
- u32 sq_size, rq_size, t_size;
- struct i40iw_dma_mem *mem;
-
- sq_size = rsrc->sq_size * I40IW_QP_WQE_MIN_SIZE;
- rq_size = rsrc->rq_size * I40IW_QP_WQE_MIN_SIZE;
- t_size = (sq_size + rq_size + (I40IW_SHADOW_AREA_SIZE << 3) +
- I40IW_QP_CTX_SIZE);
- /* Get page aligned memory */
- ret =
- i40iw_allocate_dma_mem(rsrc->dev->hw, &rsrc->qpmem, t_size,
- I40IW_HW_PAGE_SIZE);
- if (ret) {
- i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA, "%s: error dma mem\n", __func__);
- return ret;
- }
-
- mem = &rsrc->qpmem;
- memset(mem->va, 0, t_size);
- qp->hw_sq_size = i40iw_get_encoded_wqe_size(rsrc->sq_size, false);
- qp->hw_rq_size = i40iw_get_encoded_wqe_size(rsrc->rq_size, false);
- qp->pd = &rsrc->sc_pd;
- qp->qp_type = I40IW_QP_TYPE_UDA;
- qp->dev = rsrc->dev;
- qp->back_qp = (void *)rsrc;
- qp->sq_pa = mem->pa;
- qp->rq_pa = qp->sq_pa + sq_size;
- qp->vsi = rsrc->vsi;
- ukqp->sq_base = mem->va;
- ukqp->rq_base = &ukqp->sq_base[rsrc->sq_size];
- ukqp->shadow_area = ukqp->rq_base[rsrc->rq_size].elem;
- qp->shadow_area_pa = qp->rq_pa + rq_size;
- qp->hw_host_ctx = ukqp->shadow_area + I40IW_SHADOW_AREA_SIZE;
- qp->hw_host_ctx_pa =
- qp->shadow_area_pa + (I40IW_SHADOW_AREA_SIZE << 3);
- ukqp->qp_id = rsrc->qp_id;
- ukqp->sq_wrtrk_array = rsrc->sq_wrtrk_array;
- ukqp->rq_wrid_array = rsrc->rq_wrid_array;
-
- ukqp->qp_id = rsrc->qp_id;
- ukqp->sq_size = rsrc->sq_size;
- ukqp->rq_size = rsrc->rq_size;
-
- I40IW_RING_INIT(ukqp->sq_ring, ukqp->sq_size);
- I40IW_RING_INIT(ukqp->initial_ring, ukqp->sq_size);
- I40IW_RING_INIT(ukqp->rq_ring, ukqp->rq_size);
-
- if (qp->pd->dev->is_pf)
- ukqp->wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) +
- I40E_PFPE_WQEALLOC);
- else
- ukqp->wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) +
- I40E_VFPE_WQEALLOC1);
-
- qp->user_pri = 0;
- i40iw_qp_add_qos(qp);
- i40iw_puda_qp_setctx(rsrc);
- if (rsrc->dev->ceq_valid)
- ret = i40iw_cqp_qp_create_cmd(rsrc->dev, qp);
- else
- ret = i40iw_puda_qp_wqe(rsrc->dev, qp);
- if (ret) {
- i40iw_qp_rem_qos(qp);
- i40iw_free_dma_mem(rsrc->dev->hw, &rsrc->qpmem);
- }
- return ret;
-}
-
-/**
- * i40iw_puda_cq_wqe - setup wqe for cq create
- * @rsrc: resource for cq
- */
-static enum i40iw_status_code i40iw_puda_cq_wqe(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq)
-{
- u64 *wqe;
- struct i40iw_sc_cqp *cqp;
- u64 header;
- struct i40iw_ccq_cqe_info compl_info;
- enum i40iw_status_code status = 0;
-
- cqp = dev->cqp;
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, 0);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
- set_64bit_val(wqe, 8, RS_64_1(cq, 1));
- set_64bit_val(wqe, 16,
- LS_64(cq->shadow_read_threshold,
- I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
- set_64bit_val(wqe, 32, cq->cq_pa);
-
- set_64bit_val(wqe, 40, cq->shadow_area_pa);
-
- header = cq->cq_uk.cq_id |
- LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
- LS_64(1, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
- LS_64(1, I40IW_CQPSQ_CQ_ENCEQEMASK) |
- LS_64(1, I40IW_CQPSQ_CQ_CEQIDVALID) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
- i40iw_insert_wqe_hdr(wqe, header);
-
- i40iw_debug_buf(dev, I40IW_DEBUG_PUDA, "PUDA CQE",
- wqe, I40IW_CQP_WQE_SIZE * 8);
-
- i40iw_sc_cqp_post_sq(dev->cqp);
- status = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
- I40IW_CQP_OP_CREATE_CQ,
- &compl_info);
- return status;
-}
-
-/**
- * i40iw_puda_cq_create - create cq for resource
- * @rsrc: resource for which cq to create
- */
-static enum i40iw_status_code i40iw_puda_cq_create(struct i40iw_puda_rsrc *rsrc)
-{
- struct i40iw_sc_dev *dev = rsrc->dev;
- struct i40iw_sc_cq *cq = &rsrc->cq;
- enum i40iw_status_code ret = 0;
- u32 tsize, cqsize;
- struct i40iw_dma_mem *mem;
- struct i40iw_cq_init_info info;
- struct i40iw_cq_uk_init_info *init_info = &info.cq_uk_init_info;
-
- cq->vsi = rsrc->vsi;
- cqsize = rsrc->cq_size * (sizeof(struct i40iw_cqe));
- tsize = cqsize + sizeof(struct i40iw_cq_shadow_area);
- ret = i40iw_allocate_dma_mem(dev->hw, &rsrc->cqmem, tsize,
- I40IW_CQ0_ALIGNMENT);
- if (ret)
- return ret;
-
- mem = &rsrc->cqmem;
- memset(&info, 0, sizeof(info));
- info.dev = dev;
- info.type = (rsrc->type == I40IW_PUDA_RSRC_TYPE_ILQ) ?
- I40IW_CQ_TYPE_ILQ : I40IW_CQ_TYPE_IEQ;
- info.shadow_read_threshold = rsrc->cq_size >> 2;
- info.ceq_id_valid = true;
- info.cq_base_pa = mem->pa;
- info.shadow_area_pa = mem->pa + cqsize;
- init_info->cq_base = mem->va;
- init_info->shadow_area = (u64 *)((u8 *)mem->va + cqsize);
- init_info->cq_size = rsrc->cq_size;
- init_info->cq_id = rsrc->cq_id;
- info.ceqe_mask = true;
- info.ceq_id_valid = true;
- ret = dev->iw_priv_cq_ops->cq_init(cq, &info);
- if (ret)
- goto error;
- if (rsrc->dev->ceq_valid)
- ret = i40iw_cqp_cq_create_cmd(dev, cq);
- else
- ret = i40iw_puda_cq_wqe(dev, cq);
-error:
- if (ret)
- i40iw_free_dma_mem(dev->hw, &rsrc->cqmem);
- return ret;
-}
-
-/**
- * i40iw_puda_free_qp - free qp for resource
- * @rsrc: resource for which qp to free
- */
-static void i40iw_puda_free_qp(struct i40iw_puda_rsrc *rsrc)
-{
- enum i40iw_status_code ret;
- struct i40iw_ccq_cqe_info compl_info;
- struct i40iw_sc_dev *dev = rsrc->dev;
-
- if (rsrc->dev->ceq_valid) {
- i40iw_cqp_qp_destroy_cmd(dev, &rsrc->qp);
- return;
- }
-
- ret = dev->iw_priv_qp_ops->qp_destroy(&rsrc->qp,
- 0, false, true, true);
- if (ret)
- i40iw_debug(dev, I40IW_DEBUG_PUDA,
- "%s error puda qp destroy wqe\n",
- __func__);
-
- if (!ret) {
- ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
- I40IW_CQP_OP_DESTROY_QP,
- &compl_info);
- if (ret)
- i40iw_debug(dev, I40IW_DEBUG_PUDA,
- "%s error puda qp destroy failed\n",
- __func__);
- }
-}
-
-/**
- * i40iw_puda_free_cq - free cq for resource
- * @rsrc: resource for which cq to free
- */
-static void i40iw_puda_free_cq(struct i40iw_puda_rsrc *rsrc)
-{
- enum i40iw_status_code ret;
- struct i40iw_ccq_cqe_info compl_info;
- struct i40iw_sc_dev *dev = rsrc->dev;
-
- if (rsrc->dev->ceq_valid) {
- i40iw_cqp_cq_destroy_cmd(dev, &rsrc->cq);
- return;
- }
- ret = dev->iw_priv_cq_ops->cq_destroy(&rsrc->cq, 0, true);
-
- if (ret)
- i40iw_debug(dev, I40IW_DEBUG_PUDA,
- "%s error ieq cq destroy\n",
- __func__);
-
- if (!ret) {
- ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
- I40IW_CQP_OP_DESTROY_CQ,
- &compl_info);
- if (ret)
- i40iw_debug(dev, I40IW_DEBUG_PUDA,
- "%s error ieq qp destroy done\n",
- __func__);
- }
-}
-
-/**
- * i40iw_puda_dele_resources - delete all resources during close
- * @dev: iwarp device
- * @type: type of resource to dele
- * @reset: true if reset chip
- */
-void i40iw_puda_dele_resources(struct i40iw_sc_vsi *vsi,
- enum puda_resource_type type,
- bool reset)
-{
- struct i40iw_sc_dev *dev = vsi->dev;
- struct i40iw_puda_rsrc *rsrc;
- struct i40iw_puda_buf *buf = NULL;
- struct i40iw_puda_buf *nextbuf = NULL;
- struct i40iw_virt_mem *vmem;
-
- switch (type) {
- case I40IW_PUDA_RSRC_TYPE_ILQ:
- rsrc = vsi->ilq;
- vmem = &vsi->ilq_mem;
- break;
- case I40IW_PUDA_RSRC_TYPE_IEQ:
- rsrc = vsi->ieq;
- vmem = &vsi->ieq_mem;
- break;
- default:
- i40iw_debug(dev, I40IW_DEBUG_PUDA, "%s: error resource type = 0x%x\n",
- __func__, type);
- return;
- }
-
- switch (rsrc->completion) {
- case PUDA_HASH_CRC_COMPLETE:
- i40iw_free_hash_desc(rsrc->hash_desc);
- /* fall through */
- case PUDA_QP_CREATED:
- if (!reset)
- i40iw_puda_free_qp(rsrc);
-
- i40iw_free_dma_mem(dev->hw, &rsrc->qpmem);
- /* fallthrough */
- case PUDA_CQ_CREATED:
- if (!reset)
- i40iw_puda_free_cq(rsrc);
-
- i40iw_free_dma_mem(dev->hw, &rsrc->cqmem);
- break;
- default:
- i40iw_debug(rsrc->dev, I40IW_DEBUG_PUDA, "%s error no resources\n", __func__);
- break;
- }
- /* Free all allocated puda buffers for both tx and rx */
- buf = rsrc->alloclist;
- while (buf) {
- nextbuf = buf->next;
- i40iw_puda_dele_buf(dev, buf);
- buf = nextbuf;
- rsrc->alloc_buf_count--;
- }
- i40iw_free_virt_mem(dev->hw, vmem);
-}
-
-/**
- * i40iw_puda_allocbufs - allocate buffers for resource
- * @rsrc: resource for buffer allocation
- * @count: number of buffers to create
- */
-static enum i40iw_status_code i40iw_puda_allocbufs(struct i40iw_puda_rsrc *rsrc,
- u32 count)
-{
- u32 i;
- struct i40iw_puda_buf *buf;
- struct i40iw_puda_buf *nextbuf;
-
- for (i = 0; i < count; i++) {
- buf = i40iw_puda_alloc_buf(rsrc->dev, rsrc->buf_size);
- if (!buf) {
- rsrc->stats_buf_alloc_fail++;
- return I40IW_ERR_NO_MEMORY;
- }
- i40iw_puda_ret_bufpool(rsrc, buf);
- rsrc->alloc_buf_count++;
- if (!rsrc->alloclist) {
- rsrc->alloclist = buf;
- } else {
- nextbuf = rsrc->alloclist;
- rsrc->alloclist = buf;
- buf->next = nextbuf;
- }
- }
- rsrc->avail_buf_count = rsrc->alloc_buf_count;
- return 0;
-}
-
-/**
- * i40iw_puda_create_rsrc - create resouce (ilq or ieq)
- * @dev: iwarp device
- * @info: resource information
- */
-enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi,
- struct i40iw_puda_rsrc_info *info)
-{
- struct i40iw_sc_dev *dev = vsi->dev;
- enum i40iw_status_code ret = 0;
- struct i40iw_puda_rsrc *rsrc;
- u32 pudasize;
- u32 sqwridsize, rqwridsize;
- struct i40iw_virt_mem *vmem;
-
- info->count = 1;
- pudasize = sizeof(struct i40iw_puda_rsrc);
- sqwridsize = info->sq_size * sizeof(struct i40iw_sq_uk_wr_trk_info);
- rqwridsize = info->rq_size * 8;
- switch (info->type) {
- case I40IW_PUDA_RSRC_TYPE_ILQ:
- vmem = &vsi->ilq_mem;
- break;
- case I40IW_PUDA_RSRC_TYPE_IEQ:
- vmem = &vsi->ieq_mem;
- break;
- default:
- return I40IW_NOT_SUPPORTED;
- }
- ret =
- i40iw_allocate_virt_mem(dev->hw, vmem,
- pudasize + sqwridsize + rqwridsize);
- if (ret)
- return ret;
- rsrc = (struct i40iw_puda_rsrc *)vmem->va;
- spin_lock_init(&rsrc->bufpool_lock);
- if (info->type == I40IW_PUDA_RSRC_TYPE_ILQ) {
- vsi->ilq = (struct i40iw_puda_rsrc *)vmem->va;
- vsi->ilq_count = info->count;
- rsrc->receive = info->receive;
- rsrc->xmit_complete = info->xmit_complete;
- } else {
- vmem = &vsi->ieq_mem;
- vsi->ieq_count = info->count;
- vsi->ieq = (struct i40iw_puda_rsrc *)vmem->va;
- rsrc->receive = i40iw_ieq_receive;
- rsrc->xmit_complete = i40iw_ieq_tx_compl;
- }
-
- rsrc->type = info->type;
- rsrc->sq_wrtrk_array = (struct i40iw_sq_uk_wr_trk_info *)((u8 *)vmem->va + pudasize);
- rsrc->rq_wrid_array = (u64 *)((u8 *)vmem->va + pudasize + sqwridsize);
- /* Initialize all ieq lists */
- INIT_LIST_HEAD(&rsrc->bufpool);
- INIT_LIST_HEAD(&rsrc->txpend);
-
- rsrc->tx_wqe_avail_cnt = info->sq_size - 1;
- dev->iw_pd_ops->pd_init(dev, &rsrc->sc_pd, info->pd_id, -1);
- rsrc->qp_id = info->qp_id;
- rsrc->cq_id = info->cq_id;
- rsrc->sq_size = info->sq_size;
- rsrc->rq_size = info->rq_size;
- rsrc->cq_size = info->rq_size + info->sq_size;
- rsrc->buf_size = info->buf_size;
- rsrc->dev = dev;
- rsrc->vsi = vsi;
-
- ret = i40iw_puda_cq_create(rsrc);
- if (!ret) {
- rsrc->completion = PUDA_CQ_CREATED;
- ret = i40iw_puda_qp_create(rsrc);
- }
- if (ret) {
- i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error qp_create\n",
- __func__);
- goto error;
- }
- rsrc->completion = PUDA_QP_CREATED;
-
- ret = i40iw_puda_allocbufs(rsrc, info->tx_buf_cnt + info->rq_size);
- if (ret) {
- i40iw_debug(dev, I40IW_DEBUG_PUDA, "[%s] error alloc_buf\n",
- __func__);
- goto error;
- }
-
- rsrc->rxq_invalid_cnt = info->rq_size;
- ret = i40iw_puda_replenish_rq(rsrc, true);
- if (ret)
- goto error;
-
- if (info->type == I40IW_PUDA_RSRC_TYPE_IEQ) {
- if (!i40iw_init_hash_desc(&rsrc->hash_desc)) {
- rsrc->check_crc = true;
- rsrc->completion = PUDA_HASH_CRC_COMPLETE;
- ret = 0;
- }
- }
-
- dev->ccq_ops->ccq_arm(&rsrc->cq);
- return ret;
- error:
- i40iw_puda_dele_resources(vsi, info->type, false);
-
- return ret;
-}
-
-/**
- * i40iw_ilq_putback_rcvbuf - ilq buffer to put back on rq
- * @qp: ilq's qp resource
- * @wqe_idx: wqe index of completed rcvbuf
- */
-static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx)
-{
- u64 *wqe;
- u64 offset24;
-
- wqe = qp->qp_uk.rq_base[wqe_idx].elem;
- get_64bit_val(wqe, 24, &offset24);
- offset24 = (offset24) ? 0 : LS_64(1, I40IWQPSQ_VALID);
- set_64bit_val(wqe, 24, offset24);
-}
-
-/**
- * i40iw_ieq_get_fpdu - given length return fpdu length
- * @length: length if fpdu
- */
-static u16 i40iw_ieq_get_fpdu_length(u16 length)
-{
- u16 fpdu_len;
-
- fpdu_len = length + I40IW_IEQ_MPA_FRAMING;
- fpdu_len = (fpdu_len + 3) & 0xfffffffc;
- return fpdu_len;
-}
-
-/**
- * i40iw_ieq_copy_to_txbuf - copydata from rcv buf to tx buf
- * @buf: rcv buffer with partial
- * @txbuf: tx buffer for sendign back
- * @buf_offset: rcv buffer offset to copy from
- * @txbuf_offset: at offset in tx buf to copy
- * @length: length of data to copy
- */
-static void i40iw_ieq_copy_to_txbuf(struct i40iw_puda_buf *buf,
- struct i40iw_puda_buf *txbuf,
- u16 buf_offset, u32 txbuf_offset,
- u32 length)
-{
- void *mem1 = (u8 *)buf->mem.va + buf_offset;
- void *mem2 = (u8 *)txbuf->mem.va + txbuf_offset;
-
- memcpy(mem2, mem1, length);
-}
-
-/**
- * i40iw_ieq_setup_tx_buf - setup tx buffer for partial handling
- * @buf: reeive buffer with partial
- * @txbuf: buffer to prepare
- */
-static void i40iw_ieq_setup_tx_buf(struct i40iw_puda_buf *buf,
- struct i40iw_puda_buf *txbuf)
-{
- txbuf->maclen = buf->maclen;
- txbuf->tcphlen = buf->tcphlen;
- txbuf->ipv4 = buf->ipv4;
- txbuf->hdrlen = buf->hdrlen;
- i40iw_ieq_copy_to_txbuf(buf, txbuf, 0, 0, buf->hdrlen);
-}
-
-/**
- * i40iw_ieq_check_first_buf - check if rcv buffer's seq is in range
- * @buf: receive exception buffer
- * @fps: first partial sequence number
- */
-static void i40iw_ieq_check_first_buf(struct i40iw_puda_buf *buf, u32 fps)
-{
- u32 offset;
-
- if (buf->seqnum < fps) {
- offset = fps - buf->seqnum;
- if (offset > buf->datalen)
- return;
- buf->data += offset;
- buf->datalen -= (u16)offset;
- buf->seqnum = fps;
- }
-}
-
-/**
- * i40iw_ieq_compl_pfpdu - write txbuf with full fpdu
- * @ieq: ieq resource
- * @rxlist: ieq's received buffer list
- * @pbufl: temporary list for buffers for fpddu
- * @txbuf: tx buffer for fpdu
- * @fpdu_len: total length of fpdu
- */
-static void i40iw_ieq_compl_pfpdu(struct i40iw_puda_rsrc *ieq,
- struct list_head *rxlist,
- struct list_head *pbufl,
- struct i40iw_puda_buf *txbuf,
- u16 fpdu_len)
-{
- struct i40iw_puda_buf *buf;
- u32 nextseqnum;
- u16 txoffset, bufoffset;
-
- buf = i40iw_puda_get_listbuf(pbufl);
- if (!buf)
- return;
- nextseqnum = buf->seqnum + fpdu_len;
- txbuf->totallen = buf->hdrlen + fpdu_len;
- txbuf->data = (u8 *)txbuf->mem.va + buf->hdrlen;
- i40iw_ieq_setup_tx_buf(buf, txbuf);
-
- txoffset = buf->hdrlen;
- bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
-
- do {
- if (buf->datalen >= fpdu_len) {
- /* copied full fpdu */
- i40iw_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset, fpdu_len);
- buf->datalen -= fpdu_len;
- buf->data += fpdu_len;
- buf->seqnum = nextseqnum;
- break;
- }
- /* copy partial fpdu */
- i40iw_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset, buf->datalen);
- txoffset += buf->datalen;
- fpdu_len -= buf->datalen;
- i40iw_puda_ret_bufpool(ieq, buf);
- buf = i40iw_puda_get_listbuf(pbufl);
- if (!buf)
- return;
- bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
- } while (1);
-
- /* last buffer on the list*/
- if (buf->datalen)
- list_add(&buf->list, rxlist);
- else
- i40iw_puda_ret_bufpool(ieq, buf);
-}
-
-/**
- * i40iw_ieq_create_pbufl - create buffer list for single fpdu
- * @rxlist: resource list for receive ieq buffes
- * @pbufl: temp. list for buffers for fpddu
- * @buf: first receive buffer
- * @fpdu_len: total length of fpdu
- */
-static enum i40iw_status_code i40iw_ieq_create_pbufl(
- struct i40iw_pfpdu *pfpdu,
- struct list_head *rxlist,
- struct list_head *pbufl,
- struct i40iw_puda_buf *buf,
- u16 fpdu_len)
-{
- enum i40iw_status_code status = 0;
- struct i40iw_puda_buf *nextbuf;
- u32 nextseqnum;
- u16 plen = fpdu_len - buf->datalen;
- bool done = false;
-
- nextseqnum = buf->seqnum + buf->datalen;
- do {
- nextbuf = i40iw_puda_get_listbuf(rxlist);
- if (!nextbuf) {
- status = I40IW_ERR_list_empty;
- break;
- }
- list_add_tail(&nextbuf->list, pbufl);
- if (nextbuf->seqnum != nextseqnum) {
- pfpdu->bad_seq_num++;
- status = I40IW_ERR_SEQ_NUM;
- break;
- }
- if (nextbuf->datalen >= plen) {
- done = true;
- } else {
- plen -= nextbuf->datalen;
- nextseqnum = nextbuf->seqnum + nextbuf->datalen;
- }
-
- } while (!done);
-
- return status;
-}
-
-/**
- * i40iw_ieq_handle_partial - process partial fpdu buffer
- * @ieq: ieq resource
- * @pfpdu: partial management per user qp
- * @buf: receive buffer
- * @fpdu_len: fpdu len in the buffer
- */
-static enum i40iw_status_code i40iw_ieq_handle_partial(struct i40iw_puda_rsrc *ieq,
- struct i40iw_pfpdu *pfpdu,
- struct i40iw_puda_buf *buf,
- u16 fpdu_len)
-{
- enum i40iw_status_code status = 0;
- u8 *crcptr;
- u32 mpacrc;
- u32 seqnum = buf->seqnum;
- struct list_head pbufl; /* partial buffer list */
- struct i40iw_puda_buf *txbuf = NULL;
- struct list_head *rxlist = &pfpdu->rxlist;
-
- INIT_LIST_HEAD(&pbufl);
- list_add(&buf->list, &pbufl);
-
- status = i40iw_ieq_create_pbufl(pfpdu, rxlist, &pbufl, buf, fpdu_len);
- if (status)
- goto error;
-
- txbuf = i40iw_puda_get_bufpool(ieq);
- if (!txbuf) {
- pfpdu->no_tx_bufs++;
- status = I40IW_ERR_NO_TXBUFS;
- goto error;
- }
-
- i40iw_ieq_compl_pfpdu(ieq, rxlist, &pbufl, txbuf, fpdu_len);
- i40iw_ieq_update_tcpip_info(txbuf, fpdu_len, seqnum);
- crcptr = txbuf->data + fpdu_len - 4;
- mpacrc = *(u32 *)crcptr;
- if (ieq->check_crc) {
- status = i40iw_ieq_check_mpacrc(ieq->hash_desc, txbuf->data,
- (fpdu_len - 4), mpacrc);
- if (status) {
- i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ,
- "%s: error bad crc\n", __func__);
- goto error;
- }
- }
-
- i40iw_debug_buf(ieq->dev, I40IW_DEBUG_IEQ, "IEQ TX BUFFER",
- txbuf->mem.va, txbuf->totallen);
- i40iw_puda_send_buf(ieq, txbuf);
- pfpdu->rcv_nxt = seqnum + fpdu_len;
- return status;
- error:
- while (!list_empty(&pbufl)) {
- buf = (struct i40iw_puda_buf *)(pbufl.prev);
- list_del(&buf->list);
- list_add(&buf->list, rxlist);
- }
- if (txbuf)
- i40iw_puda_ret_bufpool(ieq, txbuf);
- return status;
-}
-
-/**
- * i40iw_ieq_process_buf - process buffer rcvd for ieq
- * @ieq: ieq resource
- * @pfpdu: partial management per user qp
- * @buf: receive buffer
- */
-static enum i40iw_status_code i40iw_ieq_process_buf(struct i40iw_puda_rsrc *ieq,
- struct i40iw_pfpdu *pfpdu,
- struct i40iw_puda_buf *buf)
-{
- u16 fpdu_len = 0;
- u16 datalen = buf->datalen;
- u8 *datap = buf->data;
- u8 *crcptr;
- u16 ioffset = 0;
- u32 mpacrc;
- u32 seqnum = buf->seqnum;
- u16 length = 0;
- u16 full = 0;
- bool partial = false;
- struct i40iw_puda_buf *txbuf;
- struct list_head *rxlist = &pfpdu->rxlist;
- enum i40iw_status_code ret = 0;
- enum i40iw_status_code status = 0;
-
- ioffset = (u16)(buf->data - (u8 *)buf->mem.va);
- while (datalen) {
- fpdu_len = i40iw_ieq_get_fpdu_length(ntohs(*(__be16 *)datap));
- if (fpdu_len > pfpdu->max_fpdu_data) {
- i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ,
- "%s: error bad fpdu_len\n", __func__);
- status = I40IW_ERR_MPA_CRC;
- list_add(&buf->list, rxlist);
- return status;
- }
-
- if (datalen < fpdu_len) {
- partial = true;
- break;
- }
- crcptr = datap + fpdu_len - 4;
- mpacrc = *(u32 *)crcptr;
- if (ieq->check_crc)
- ret = i40iw_ieq_check_mpacrc(ieq->hash_desc,
- datap, fpdu_len - 4, mpacrc);
- if (ret) {
- status = I40IW_ERR_MPA_CRC;
- list_add(&buf->list, rxlist);
- return status;
- }
- full++;
- pfpdu->fpdu_processed++;
- datap += fpdu_len;
- length += fpdu_len;
- datalen -= fpdu_len;
- }
- if (full) {
- /* copy full pdu's in the txbuf and send them out */
- txbuf = i40iw_puda_get_bufpool(ieq);
- if (!txbuf) {
- pfpdu->no_tx_bufs++;
- status = I40IW_ERR_NO_TXBUFS;
- list_add(&buf->list, rxlist);
- return status;
- }
- /* modify txbuf's buffer header */
- i40iw_ieq_setup_tx_buf(buf, txbuf);
- /* copy full fpdu's to new buffer */
- i40iw_ieq_copy_to_txbuf(buf, txbuf, ioffset, buf->hdrlen,
- length);
- txbuf->totallen = buf->hdrlen + length;
-
- i40iw_ieq_update_tcpip_info(txbuf, length, buf->seqnum);
- i40iw_puda_send_buf(ieq, txbuf);
-
- if (!datalen) {
- pfpdu->rcv_nxt = buf->seqnum + length;
- i40iw_puda_ret_bufpool(ieq, buf);
- return status;
- }
- buf->data = datap;
- buf->seqnum = seqnum + length;
- buf->datalen = datalen;
- pfpdu->rcv_nxt = buf->seqnum;
- }
- if (partial)
- status = i40iw_ieq_handle_partial(ieq, pfpdu, buf, fpdu_len);
-
- return status;
-}
-
-/**
- * i40iw_ieq_process_fpdus - process fpdu's buffers on its list
- * @qp: qp for which partial fpdus
- * @ieq: ieq resource
- */
-static void i40iw_ieq_process_fpdus(struct i40iw_sc_qp *qp,
- struct i40iw_puda_rsrc *ieq)
-{
- struct i40iw_pfpdu *pfpdu = &qp->pfpdu;
- struct list_head *rxlist = &pfpdu->rxlist;
- struct i40iw_puda_buf *buf;
- enum i40iw_status_code status;
-
- do {
- if (list_empty(rxlist))
- break;
- buf = i40iw_puda_get_listbuf(rxlist);
- if (!buf) {
- i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ,
- "%s: error no buf\n", __func__);
- break;
- }
- if (buf->seqnum != pfpdu->rcv_nxt) {
- /* This could be out of order or missing packet */
- pfpdu->out_of_order++;
- list_add(&buf->list, rxlist);
- break;
- }
- /* keep processing buffers from the head of the list */
- status = i40iw_ieq_process_buf(ieq, pfpdu, buf);
- if (status == I40IW_ERR_MPA_CRC) {
- pfpdu->mpa_crc_err = true;
- while (!list_empty(rxlist)) {
- buf = i40iw_puda_get_listbuf(rxlist);
- i40iw_puda_ret_bufpool(ieq, buf);
- pfpdu->crc_err++;
- }
- /* create CQP for AE */
- i40iw_ieq_mpa_crc_ae(ieq->dev, qp);
- }
- } while (!status);
-}
-
-/**
- * i40iw_ieq_handle_exception - handle qp's exception
- * @ieq: ieq resource
- * @qp: qp receiving excpetion
- * @buf: receive buffer
- */
-static void i40iw_ieq_handle_exception(struct i40iw_puda_rsrc *ieq,
- struct i40iw_sc_qp *qp,
- struct i40iw_puda_buf *buf)
-{
- struct i40iw_puda_buf *tmpbuf = NULL;
- struct i40iw_pfpdu *pfpdu = &qp->pfpdu;
- u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx;
- u32 rcv_wnd = hw_host_ctx[23];
- /* first partial seq # in q2 */
- u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET);
- struct list_head *rxlist = &pfpdu->rxlist;
- struct list_head *plist;
-
- pfpdu->total_ieq_bufs++;
-
- if (pfpdu->mpa_crc_err) {
- pfpdu->crc_err++;
- goto error;
- }
- if (pfpdu->mode && (fps != pfpdu->fps)) {
- /* clean up qp as it is new partial sequence */
- i40iw_ieq_cleanup_qp(ieq, qp);
- i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ,
- "%s: restarting new partial\n", __func__);
- pfpdu->mode = false;
- }
-
- if (!pfpdu->mode) {
- i40iw_debug_buf(ieq->dev, I40IW_DEBUG_IEQ, "Q2 BUFFER", (u64 *)qp->q2_buf, 128);
- /* First_Partial_Sequence_Number check */
- pfpdu->rcv_nxt = fps;
- pfpdu->fps = fps;
- pfpdu->mode = true;
- pfpdu->max_fpdu_data = (buf->ipv4) ? (ieq->vsi->mtu - I40IW_MTU_TO_MSS_IPV4) :
- (ieq->vsi->mtu - I40IW_MTU_TO_MSS_IPV6);
- pfpdu->pmode_count++;
- INIT_LIST_HEAD(rxlist);
- i40iw_ieq_check_first_buf(buf, fps);
- }
-
- if (!(rcv_wnd >= (buf->seqnum - pfpdu->rcv_nxt))) {
- pfpdu->bad_seq_num++;
- goto error;
- }
-
- if (!list_empty(rxlist)) {
- tmpbuf = (struct i40iw_puda_buf *)rxlist->next;
- while ((struct list_head *)tmpbuf != rxlist) {
- if ((int)(buf->seqnum - tmpbuf->seqnum) < 0)
- break;
- plist = &tmpbuf->list;
- tmpbuf = (struct i40iw_puda_buf *)plist->next;
- }
- /* Insert buf before tmpbuf */
- list_add_tail(&buf->list, &tmpbuf->list);
- } else {
- list_add_tail(&buf->list, rxlist);
- }
- i40iw_ieq_process_fpdus(qp, ieq);
- return;
- error:
- i40iw_puda_ret_bufpool(ieq, buf);
-}
-
-/**
- * i40iw_ieq_receive - received exception buffer
- * @dev: iwarp device
- * @buf: exception buffer received
- */
-static void i40iw_ieq_receive(struct i40iw_sc_vsi *vsi,
- struct i40iw_puda_buf *buf)
-{
- struct i40iw_puda_rsrc *ieq = vsi->ieq;
- struct i40iw_sc_qp *qp = NULL;
- u32 wqe_idx = ieq->compl_rxwqe_idx;
-
- qp = i40iw_ieq_get_qp(vsi->dev, buf);
- if (!qp) {
- ieq->stats_bad_qp_id++;
- i40iw_puda_ret_bufpool(ieq, buf);
- } else {
- i40iw_ieq_handle_exception(ieq, qp, buf);
- }
- /*
- * ieq->rx_wqe_idx is used by i40iw_puda_replenish_rq()
- * on which wqe_idx to start replenish rq
- */
- if (!ieq->rxq_invalid_cnt)
- ieq->rx_wqe_idx = wqe_idx;
- ieq->rxq_invalid_cnt++;
-}
-
-/**
- * i40iw_ieq_tx_compl - put back after sending completed exception buffer
- * @vsi: pointer to the vsi structure
- * @sqwrid: pointer to puda buffer
- */
-static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi *vsi, void *sqwrid)
-{
- struct i40iw_puda_rsrc *ieq = vsi->ieq;
- struct i40iw_puda_buf *buf = (struct i40iw_puda_buf *)sqwrid;
-
- i40iw_puda_ret_bufpool(ieq, buf);
-}
-
-/**
- * i40iw_ieq_cleanup_qp - qp is being destroyed
- * @ieq: ieq resource
- * @qp: all pending fpdu buffers
- */
-void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp)
-{
- struct i40iw_puda_buf *buf;
- struct i40iw_pfpdu *pfpdu = &qp->pfpdu;
- struct list_head *rxlist = &pfpdu->rxlist;
-
- if (!pfpdu->mode)
- return;
- while (!list_empty(rxlist)) {
- buf = i40iw_puda_get_listbuf(rxlist);
- i40iw_puda_ret_bufpool(ieq, buf);
- }
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.h b/drivers/infiniband/hw/i40iw/i40iw_puda.h
deleted file mode 100644
index 53a7d58c84b5..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_puda.h
+++ /dev/null
@@ -1,188 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_PUDA_H
-#define I40IW_PUDA_H
-
-#define I40IW_IEQ_MPA_FRAMING 6
-
-struct i40iw_sc_dev;
-struct i40iw_sc_qp;
-struct i40iw_sc_cq;
-
-enum puda_resource_type {
- I40IW_PUDA_RSRC_TYPE_ILQ = 1,
- I40IW_PUDA_RSRC_TYPE_IEQ
-};
-
-enum puda_rsrc_complete {
- PUDA_CQ_CREATED = 1,
- PUDA_QP_CREATED,
- PUDA_TX_COMPLETE,
- PUDA_RX_COMPLETE,
- PUDA_HASH_CRC_COMPLETE
-};
-
-struct i40iw_puda_completion_info {
- struct i40iw_qp_uk *qp;
- u8 q_type;
- u8 vlan_valid;
- u8 l3proto;
- u8 l4proto;
- u16 payload_len;
- u32 compl_error; /* No_err=0, else major and minor err code */
- u32 qp_id;
- u32 wqe_idx;
-};
-
-struct i40iw_puda_send_info {
- u64 paddr; /* Physical address */
- u32 len;
- u8 tcplen;
- u8 maclen;
- bool ipv4;
- bool doloopback;
- void *scratch;
-};
-
-struct i40iw_puda_buf {
- struct list_head list; /* MUST be first entry */
- struct i40iw_dma_mem mem; /* DMA memory for the buffer */
- struct i40iw_puda_buf *next; /* for alloclist in rsrc struct */
- struct i40iw_virt_mem buf_mem; /* Buffer memory for this buffer */
- void *scratch;
- u8 *iph;
- u8 *tcph;
- u8 *data;
- u16 datalen;
- u16 vlan_id;
- u8 tcphlen; /* tcp length in bytes */
- u8 maclen; /* mac length in bytes */
- u32 totallen; /* machlen+iphlen+tcphlen+datalen */
- atomic_t refcount;
- u8 hdrlen;
- bool ipv4;
- u32 seqnum;
-};
-
-struct i40iw_puda_rsrc_info {
- enum puda_resource_type type; /* ILQ or IEQ */
- u32 count;
- u16 pd_id;
- u32 cq_id;
- u32 qp_id;
- u32 sq_size;
- u32 rq_size;
- u16 buf_size;
- u16 mss;
- u32 tx_buf_cnt; /* total bufs allocated will be rq_size + tx_buf_cnt */
- void (*receive)(struct i40iw_sc_vsi *, struct i40iw_puda_buf *);
- void (*xmit_complete)(struct i40iw_sc_vsi *, void *);
-};
-
-struct i40iw_puda_rsrc {
- struct i40iw_sc_cq cq;
- struct i40iw_sc_qp qp;
- struct i40iw_sc_pd sc_pd;
- struct i40iw_sc_dev *dev;
- struct i40iw_sc_vsi *vsi;
- struct i40iw_dma_mem cqmem;
- struct i40iw_dma_mem qpmem;
- struct i40iw_virt_mem ilq_mem;
- enum puda_rsrc_complete completion;
- enum puda_resource_type type;
- u16 buf_size; /*buffer must be max datalen + tcpip hdr + mac */
- u16 mss;
- u32 cq_id;
- u32 qp_id;
- u32 sq_size;
- u32 rq_size;
- u32 cq_size;
- struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
- u64 *rq_wrid_array;
- u32 compl_rxwqe_idx;
- u32 rx_wqe_idx;
- u32 rxq_invalid_cnt;
- u32 tx_wqe_avail_cnt;
- bool check_crc;
- struct shash_desc *hash_desc;
- struct list_head txpend;
- struct list_head bufpool; /* free buffers pool list for recv and xmit */
- u32 alloc_buf_count;
- u32 avail_buf_count; /* snapshot of currently available buffers */
- spinlock_t bufpool_lock;
- struct i40iw_puda_buf *alloclist;
- void (*receive)(struct i40iw_sc_vsi *, struct i40iw_puda_buf *);
- void (*xmit_complete)(struct i40iw_sc_vsi *, void *);
- /* puda stats */
- u64 stats_buf_alloc_fail;
- u64 stats_pkt_rcvd;
- u64 stats_pkt_sent;
- u64 stats_rcvd_pkt_err;
- u64 stats_sent_pkt_q;
- u64 stats_bad_qp_id;
-};
-
-struct i40iw_puda_buf *i40iw_puda_get_bufpool(struct i40iw_puda_rsrc *rsrc);
-void i40iw_puda_ret_bufpool(struct i40iw_puda_rsrc *rsrc,
- struct i40iw_puda_buf *buf);
-void i40iw_puda_send_buf(struct i40iw_puda_rsrc *rsrc,
- struct i40iw_puda_buf *buf);
-enum i40iw_status_code i40iw_puda_send(struct i40iw_sc_qp *qp,
- struct i40iw_puda_send_info *info);
-enum i40iw_status_code i40iw_puda_create_rsrc(struct i40iw_sc_vsi *vsi,
- struct i40iw_puda_rsrc_info *info);
-void i40iw_puda_dele_resources(struct i40iw_sc_vsi *vsi,
- enum puda_resource_type type,
- bool reset);
-enum i40iw_status_code i40iw_puda_poll_completion(struct i40iw_sc_dev *dev,
- struct i40iw_sc_cq *cq, u32 *compl_err);
-
-struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev,
- struct i40iw_puda_buf *buf);
-enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info,
- struct i40iw_puda_buf *buf);
-enum i40iw_status_code i40iw_ieq_check_mpacrc(struct shash_desc *desc,
- void *addr, u32 length, u32 value);
-enum i40iw_status_code i40iw_init_hash_desc(struct shash_desc **desc);
-void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);
-void i40iw_free_hash_desc(struct shash_desc *desc);
-void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length,
- u32 seqnum);
-enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);
-enum i40iw_status_code i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq);
-void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);
-void i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq);
-void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp);
-#endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_register.h b/drivers/infiniband/hw/i40iw/i40iw_register.h
deleted file mode 100644
index 57768184e251..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_register.h
+++ /dev/null
@@ -1,1030 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_REGISTER_H
-#define I40IW_REGISTER_H
-
-#define I40E_GLGEN_STAT 0x000B612C /* Reset: POR */
-
-#define I40E_PFHMC_PDINV 0x000C0300 /* Reset: PFR */
-#define I40E_PFHMC_PDINV_PMSDIDX_SHIFT 0
-#define I40E_PFHMC_PDINV_PMSDIDX_MASK (0xFFF << I40E_PFHMC_PDINV_PMSDIDX_SHIFT)
-#define I40E_PFHMC_PDINV_PMPDIDX_SHIFT 16
-#define I40E_PFHMC_PDINV_PMPDIDX_MASK (0x1FF << I40E_PFHMC_PDINV_PMPDIDX_SHIFT)
-#define I40E_PFHMC_SDCMD_PMSDWR_SHIFT 31
-#define I40E_PFHMC_SDCMD_PMSDWR_MASK (0x1 << I40E_PFHMC_SDCMD_PMSDWR_SHIFT)
-#define I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT 0
-#define I40E_PFHMC_SDDATALOW_PMSDVALID_MASK (0x1 << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)
-#define I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT 1
-#define I40E_PFHMC_SDDATALOW_PMSDTYPE_MASK (0x1 << I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT)
-#define I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT 2
-#define I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_MASK (0x3FF << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT)
-
-#define I40E_PFINT_DYN_CTLN(_INTPF) (0x00034800 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: PFR */
-#define I40E_PFINT_DYN_CTLN_INTENA_SHIFT 0
-#define I40E_PFINT_DYN_CTLN_INTENA_MASK (0x1 << I40E_PFINT_DYN_CTLN_INTENA_SHIFT)
-#define I40E_PFINT_DYN_CTLN_CLEARPBA_SHIFT 1
-#define I40E_PFINT_DYN_CTLN_CLEARPBA_MASK (0x1 << I40E_PFINT_DYN_CTLN_CLEARPBA_SHIFT)
-#define I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT 3
-#define I40E_PFINT_DYN_CTLN_ITR_INDX_MASK (0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)
-
-#define I40E_VFINT_DYN_CTLN1(_INTVF) (0x00003800 + ((_INTVF) * 4)) /* _i=0...15 */ /* Reset: VFR */
-#define I40E_GLHMC_VFPDINV(_i) (0x000C8300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */
-
-#define I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT 15
-#define I40E_PFHMC_PDINV_PMSDPARTSEL_MASK (0x1 << I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT)
-#define I40E_GLPCI_LBARCTRL 0x000BE484 /* Reset: POR */
-#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT 4
-#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_MASK (0x3 << I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT)
-#define I40E_GLPCI_DREVID 0x0009C480 /* Reset: PCIR */
-#define I40E_GLPCI_DREVID_DEFAULT_REVID_SHIFT 0
-#define I40E_GLPCI_DREVID_DEFAULT_REVID_MASK 0xFF
-
-#define I40E_PFPE_AEQALLOC 0x00131180 /* Reset: PFR */
-#define I40E_PFPE_AEQALLOC_AECOUNT_SHIFT 0
-#define I40E_PFPE_AEQALLOC_AECOUNT_MASK (0xFFFFFFFF << I40E_PFPE_AEQALLOC_AECOUNT_SHIFT)
-#define I40E_PFPE_CCQPHIGH 0x00008200 /* Reset: PFR */
-#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0
-#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT)
-#define I40E_PFPE_CCQPLOW 0x00008180 /* Reset: PFR */
-#define I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT 0
-#define I40E_PFPE_CCQPLOW_PECCQPLOW_MASK (0xFFFFFFFF << I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT)
-#define I40E_PFPE_CCQPSTATUS 0x00008100 /* Reset: PFR */
-#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0
-#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_MASK (0x1 << I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT)
-#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4
-#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_MASK (0x7 << I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT)
-#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16
-#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_MASK (0x3F << I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT)
-#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31
-#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_MASK (0x1 << I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT)
-#define I40E_PFPE_CQACK 0x00131100 /* Reset: PFR */
-#define I40E_PFPE_CQACK_PECQID_SHIFT 0
-#define I40E_PFPE_CQACK_PECQID_MASK (0x1FFFF << I40E_PFPE_CQACK_PECQID_SHIFT)
-#define I40E_PFPE_CQARM 0x00131080 /* Reset: PFR */
-#define I40E_PFPE_CQARM_PECQID_SHIFT 0
-#define I40E_PFPE_CQARM_PECQID_MASK (0x1FFFF << I40E_PFPE_CQARM_PECQID_SHIFT)
-#define I40E_PFPE_CQPDB 0x00008000 /* Reset: PFR */
-#define I40E_PFPE_CQPDB_WQHEAD_SHIFT 0
-#define I40E_PFPE_CQPDB_WQHEAD_MASK (0x7FF << I40E_PFPE_CQPDB_WQHEAD_SHIFT)
-#define I40E_PFPE_CQPERRCODES 0x00008880 /* Reset: PFR */
-#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0
-#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_MASK (0xFFFF << I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT)
-#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16
-#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT)
-#define I40E_PFPE_CQPTAIL 0x00008080 /* Reset: PFR */
-#define I40E_PFPE_CQPTAIL_WQTAIL_SHIFT 0
-#define I40E_PFPE_CQPTAIL_WQTAIL_MASK (0x7FF << I40E_PFPE_CQPTAIL_WQTAIL_SHIFT)
-#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31
-#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_MASK (0x1 << I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT)
-#define I40E_PFPE_FLMQ1ALLOCERR 0x00008980 /* Reset: PFR */
-#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0
-#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT)
-#define I40E_PFPE_FLMXMITALLOCERR 0x00008900 /* Reset: PFR */
-#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT 0
-#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT)
-#define I40E_PFPE_IPCONFIG0 0x00008280 /* Reset: PFR */
-#define I40E_PFPE_IPCONFIG0_PEIPID_SHIFT 0
-#define I40E_PFPE_IPCONFIG0_PEIPID_MASK (0xFFFF << I40E_PFPE_IPCONFIG0_PEIPID_SHIFT)
-#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16
-#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_MASK (0x1 << I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT)
-#define I40E_PFPE_MRTEIDXMASK 0x00008600 /* Reset: PFR */
-#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0
-#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK (0x1F << I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT)
-#define I40E_PFPE_RCVUNEXPECTEDERROR 0x00008680 /* Reset: PFR */
-#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0
-#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT)
-#define I40E_PFPE_TCPNOWTIMER 0x00008580 /* Reset: PFR */
-#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0
-#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_MASK (0xFFFFFFFF << I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT)
-
-#define I40E_PFPE_WQEALLOC 0x00138C00 /* Reset: PFR */
-#define I40E_PFPE_WQEALLOC_PEQPID_SHIFT 0
-#define I40E_PFPE_WQEALLOC_PEQPID_MASK (0x3FFFF << I40E_PFPE_WQEALLOC_PEQPID_SHIFT)
-#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20
-#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_MASK (0xFFF << I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT)
-
-#define I40E_VFPE_AEQALLOC(_VF) (0x00130C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_AEQALLOC_MAX_INDEX 127
-#define I40E_VFPE_AEQALLOC_AECOUNT_SHIFT 0
-#define I40E_VFPE_AEQALLOC_AECOUNT_MASK (0xFFFFFFFF << I40E_VFPE_AEQALLOC_AECOUNT_SHIFT)
-#define I40E_VFPE_CCQPHIGH(_VF) (0x00001000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CCQPHIGH_MAX_INDEX 127
-#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0
-#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT)
-#define I40E_VFPE_CCQPLOW(_VF) (0x00000C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CCQPLOW_MAX_INDEX 127
-#define I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT 0
-#define I40E_VFPE_CCQPLOW_PECCQPLOW_MASK (0xFFFFFFFF << I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT)
-#define I40E_VFPE_CCQPSTATUS(_VF) (0x00000800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CCQPSTATUS_MAX_INDEX 127
-#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0
-#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_MASK (0x1 << I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT)
-#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4
-#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_MASK (0x7 << I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT)
-#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16
-#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_MASK (0x3F << I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT)
-#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31
-#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_MASK (0x1 << I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT)
-#define I40E_VFPE_CQACK(_VF) (0x00130800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CQACK_MAX_INDEX 127
-#define I40E_VFPE_CQACK_PECQID_SHIFT 0
-#define I40E_VFPE_CQACK_PECQID_MASK (0x1FFFF << I40E_VFPE_CQACK_PECQID_SHIFT)
-#define I40E_VFPE_CQARM(_VF) (0x00130400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CQARM_MAX_INDEX 127
-#define I40E_VFPE_CQARM_PECQID_SHIFT 0
-#define I40E_VFPE_CQARM_PECQID_MASK (0x1FFFF << I40E_VFPE_CQARM_PECQID_SHIFT)
-#define I40E_VFPE_CQPDB(_VF) (0x00000000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CQPDB_MAX_INDEX 127
-#define I40E_VFPE_CQPDB_WQHEAD_SHIFT 0
-#define I40E_VFPE_CQPDB_WQHEAD_MASK (0x7FF << I40E_VFPE_CQPDB_WQHEAD_SHIFT)
-#define I40E_VFPE_CQPERRCODES(_VF) (0x00001800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CQPERRCODES_MAX_INDEX 127
-#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0
-#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT)
-#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16
-#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT)
-#define I40E_VFPE_CQPTAIL(_VF) (0x00000400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CQPTAIL_MAX_INDEX 127
-#define I40E_VFPE_CQPTAIL_WQTAIL_SHIFT 0
-#define I40E_VFPE_CQPTAIL_WQTAIL_MASK (0x7FF << I40E_VFPE_CQPTAIL_WQTAIL_SHIFT)
-#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31
-#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_MASK (0x1 << I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT)
-#define I40E_VFPE_IPCONFIG0(_VF) (0x00001400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_IPCONFIG0_MAX_INDEX 127
-#define I40E_VFPE_IPCONFIG0_PEIPID_SHIFT 0
-#define I40E_VFPE_IPCONFIG0_PEIPID_MASK (0xFFFF << I40E_VFPE_IPCONFIG0_PEIPID_SHIFT)
-#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16
-#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_MASK (0x1 << I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT)
-#define I40E_VFPE_MRTEIDXMASK(_VF) (0x00003000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_MRTEIDXMASK_MAX_INDEX 127
-#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0
-#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK (0x1F << I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT)
-#define I40E_VFPE_RCVUNEXPECTEDERROR(_VF) (0x00003400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_RCVUNEXPECTEDERROR_MAX_INDEX 127
-#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0
-#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT)
-#define I40E_VFPE_TCPNOWTIMER(_VF) (0x00002C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_TCPNOWTIMER_MAX_INDEX 127
-#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0
-#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_MASK (0xFFFFFFFF << I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT)
-#define I40E_VFPE_WQEALLOC(_VF) (0x00138000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_WQEALLOC_MAX_INDEX 127
-#define I40E_VFPE_WQEALLOC_PEQPID_SHIFT 0
-#define I40E_VFPE_WQEALLOC_PEQPID_MASK (0x3FFFF << I40E_VFPE_WQEALLOC_PEQPID_SHIFT)
-#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20
-#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_MASK (0xFFF << I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT)
-
-#define I40E_GLPE_CPUSTATUS0 0x0000D040 /* Reset: PE_CORER */
-#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT 0
-#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT)
-#define I40E_GLPE_CPUSTATUS1 0x0000D044 /* Reset: PE_CORER */
-#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT 0
-#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT)
-#define I40E_GLPE_CPUSTATUS2 0x0000D048 /* Reset: PE_CORER */
-#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT 0
-#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT)
-#define I40E_GLPE_CPUTRIG0 0x0000D060 /* Reset: PE_CORER */
-#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT 0
-#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_MASK (0xFFFF << I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT)
-#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT 17
-#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_MASK (0x1 << I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT)
-#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT 18
-#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_MASK (0x1 << I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT)
-#define I40E_GLPE_DUAL40_RUPM 0x0000DA04 /* Reset: PE_CORER */
-#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT 0
-#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_MASK (0x1 << I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT)
-#define I40E_GLPE_PFAEQEDROPCNT(_i) (0x00131440 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */
-#define I40E_GLPE_PFAEQEDROPCNT_MAX_INDEX 15
-#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0
-#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT)
-#define I40E_GLPE_PFCEQEDROPCNT(_i) (0x001313C0 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */
-#define I40E_GLPE_PFCEQEDROPCNT_MAX_INDEX 15
-#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0
-#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT)
-#define I40E_GLPE_PFCQEDROPCNT(_i) (0x00131340 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */
-#define I40E_GLPE_PFCQEDROPCNT_MAX_INDEX 15
-#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT 0
-#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT)
-#define I40E_GLPE_RUPM_CQPPOOL 0x0000DACC /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT 0
-#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_MASK (0xFF << I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT)
-#define I40E_GLPE_RUPM_FLRPOOL 0x0000DAC4 /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT 0
-#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_MASK (0xFF << I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL 0x0000DA00 /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT 0
-#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_MASK (0xFF << I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT 26
-#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT 27
-#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT 28
-#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT 29
-#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT 30
-#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT 31
-#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_MASK (0x1 << I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT)
-#define I40E_GLPE_RUPM_PTXPOOL 0x0000DAC8 /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT 0
-#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_MASK (0xFF << I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT)
-#define I40E_GLPE_RUPM_PUSHPOOL 0x0000DAC0 /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT 0
-#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_MASK (0xFF << I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT)
-#define I40E_GLPE_RUPM_TXHOST_EN 0x0000DA08 /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT 0
-#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_MASK (0x1 << I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT)
-#define I40E_GLPE_VFAEQEDROPCNT(_i) (0x00132540 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */
-#define I40E_GLPE_VFAEQEDROPCNT_MAX_INDEX 31
-#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0
-#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT)
-#define I40E_GLPE_VFCEQEDROPCNT(_i) (0x00132440 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */
-#define I40E_GLPE_VFCEQEDROPCNT_MAX_INDEX 31
-#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0
-#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT)
-#define I40E_GLPE_VFCQEDROPCNT(_i) (0x00132340 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */
-#define I40E_GLPE_VFCQEDROPCNT_MAX_INDEX 31
-#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT 0
-#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT)
-#define I40E_GLPE_VFFLMOBJCTRL(_i) (0x0000D400 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPE_VFFLMOBJCTRL_MAX_INDEX 31
-#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT 0
-#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_MASK (0x7 << I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT)
-#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT 8
-#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_MASK (0x7 << I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT)
-#define I40E_GLPE_VFFLMQ1ALLOCERR(_i) (0x0000C700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPE_VFFLMQ1ALLOCERR_MAX_INDEX 31
-#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0
-#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT)
-#define I40E_GLPE_VFFLMXMITALLOCERR(_i) (0x0000C600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPE_VFFLMXMITALLOCERR_MAX_INDEX 31
-#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT 0
-#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT)
-#define I40E_GLPE_VFUDACTRL(_i) (0x0000C000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPE_VFUDACTRL_MAX_INDEX 31
-#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT 0
-#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT)
-#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT 1
-#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT)
-#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT 2
-#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT)
-#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT 3
-#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT)
-#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT 4
-#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_MASK (0x1 << I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT)
-#define I40E_GLPE_VFUDAUCFBQPN(_i) (0x0000C100 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPE_VFUDAUCFBQPN_MAX_INDEX 31
-#define I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT 0
-#define I40E_GLPE_VFUDAUCFBQPN_QPN_MASK (0x3FFFF << I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT)
-#define I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT 31
-#define I40E_GLPE_VFUDAUCFBQPN_VALID_MASK (0x1 << I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT)
-
-#define I40E_GLPES_PFIP4RXDISCARD(_i) (0x00010600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXDISCARD_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0
-#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT)
-#define I40E_GLPES_PFIP4RXFRAGSHI(_i) (0x00010804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXFRAGSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0
-#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT)
-#define I40E_GLPES_PFIP4RXFRAGSLO(_i) (0x00010800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXFRAGSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0
-#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT)
-#define I40E_GLPES_PFIP4RXMCOCTSHI(_i) (0x00010A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXMCOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP4RXMCOCTSLO(_i) (0x00010A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXMCOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP4RXMCPKTSHI(_i) (0x00010C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXMCPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP4RXMCPKTSLO(_i) (0x00010C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXMCPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP4RXOCTSHI(_i) (0x00010204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP4RXOCTSLO(_i) (0x00010200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP4RXPKTSHI(_i) (0x00010404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP4RXPKTSLO(_i) (0x00010400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP4RXTRUNC(_i) (0x00010700 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXTRUNC_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0
-#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT)
-#define I40E_GLPES_PFIP4TXFRAGSHI(_i) (0x00011E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXFRAGSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0
-#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT)
-#define I40E_GLPES_PFIP4TXFRAGSLO(_i) (0x00011E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXFRAGSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0
-#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT)
-#define I40E_GLPES_PFIP4TXMCOCTSHI(_i) (0x00012004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXMCOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP4TXMCOCTSLO(_i) (0x00012000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXMCOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP4TXMCPKTSHI(_i) (0x00012204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXMCPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP4TXMCPKTSLO(_i) (0x00012200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXMCPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP4TXNOROUTE(_i) (0x00012E00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXNOROUTE_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0
-#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT)
-#define I40E_GLPES_PFIP4TXOCTSHI(_i) (0x00011A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP4TXOCTSLO(_i) (0x00011A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP4TXPKTSHI(_i) (0x00011C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP4TXPKTSLO(_i) (0x00011C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXDISCARD(_i) (0x00011200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXDISCARD_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0
-#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT)
-#define I40E_GLPES_PFIP6RXFRAGSHI(_i) (0x00011404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXFRAGSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0
-#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT)
-#define I40E_GLPES_PFIP6RXFRAGSLO(_i) (0x00011400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXFRAGSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0
-#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXMCOCTSHI(_i) (0x00011604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXMCOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP6RXMCOCTSLO(_i) (0x00011600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXMCOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXMCPKTSHI(_i) (0x00011804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXMCPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP6RXMCPKTSLO(_i) (0x00011800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXMCPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXOCTSHI(_i) (0x00010E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP6RXOCTSLO(_i) (0x00010E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXPKTSHI(_i) (0x00011004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP6RXPKTSLO(_i) (0x00011000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXTRUNC(_i) (0x00011300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXTRUNC_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0
-#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT)
-#define I40E_GLPES_PFIP6TXFRAGSHI(_i) (0x00012804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXFRAGSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0
-#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT)
-#define I40E_GLPES_PFIP6TXFRAGSLO(_i) (0x00012800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXFRAGSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0
-#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT)
-#define I40E_GLPES_PFIP6TXMCOCTSHI(_i) (0x00012A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXMCOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP6TXMCOCTSLO(_i) (0x00012A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXMCOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP6TXMCPKTSHI(_i) (0x00012C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXMCPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP6TXMCPKTSLO(_i) (0x00012C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXMCPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP6TXNOROUTE(_i) (0x00012F00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXNOROUTE_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0
-#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT)
-#define I40E_GLPES_PFIP6TXOCTSHI(_i) (0x00012404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP6TXOCTSLO(_i) (0x00012400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP6TXPKTSHI(_i) (0x00012604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP6TXPKTSLO(_i) (0x00012600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT)
-#define I40E_GLPES_PFRDMARXRDSHI(_i) (0x00013E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXRDSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0
-#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT)
-#define I40E_GLPES_PFRDMARXRDSLO(_i) (0x00013E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXRDSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0
-#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT)
-#define I40E_GLPES_PFRDMARXSNDSHI(_i) (0x00014004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXSNDSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0
-#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT)
-#define I40E_GLPES_PFRDMARXSNDSLO(_i) (0x00014000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXSNDSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0
-#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT)
-#define I40E_GLPES_PFRDMARXWRSHI(_i) (0x00013C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXWRSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0
-#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT)
-#define I40E_GLPES_PFRDMARXWRSLO(_i) (0x00013C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXWRSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0
-#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT)
-#define I40E_GLPES_PFRDMATXRDSHI(_i) (0x00014404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXRDSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0
-#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT)
-#define I40E_GLPES_PFRDMATXRDSLO(_i) (0x00014400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXRDSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0
-#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT)
-#define I40E_GLPES_PFRDMATXSNDSHI(_i) (0x00014604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXSNDSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0
-#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT)
-#define I40E_GLPES_PFRDMATXSNDSLO(_i) (0x00014600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXSNDSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0
-#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT)
-#define I40E_GLPES_PFRDMATXWRSHI(_i) (0x00014204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXWRSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0
-#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT)
-#define I40E_GLPES_PFRDMATXWRSLO(_i) (0x00014200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXWRSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0
-#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT)
-#define I40E_GLPES_PFRDMAVBNDHI(_i) (0x00014804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMAVBNDHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0
-#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT)
-#define I40E_GLPES_PFRDMAVBNDLO(_i) (0x00014800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMAVBNDLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0
-#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT)
-#define I40E_GLPES_PFRDMAVINVHI(_i) (0x00014A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMAVINVHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT 0
-#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT)
-#define I40E_GLPES_PFRDMAVINVLO(_i) (0x00014A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMAVINVLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT 0
-#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT)
-#define I40E_GLPES_PFRXVLANERR(_i) (0x00010000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRXVLANERR_MAX_INDEX 15
-#define I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT 0
-#define I40E_GLPES_PFRXVLANERR_RXVLANERR_MASK (0xFFFFFF << I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT)
-#define I40E_GLPES_PFTCPRTXSEG(_i) (0x00013600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPRTXSEG_MAX_INDEX 15
-#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT 0
-#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT)
-#define I40E_GLPES_PFTCPRXOPTERR(_i) (0x00013200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPRXOPTERR_MAX_INDEX 15
-#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0
-#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_MASK (0xFFFFFF << I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT)
-#define I40E_GLPES_PFTCPRXPROTOERR(_i) (0x00013300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPRXPROTOERR_MAX_INDEX 15
-#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0
-#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_MASK (0xFFFFFF << I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT)
-#define I40E_GLPES_PFTCPRXSEGSHI(_i) (0x00013004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPRXSEGSHI_MAX_INDEX 15
-#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0
-#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_MASK (0xFFFF << I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT)
-#define I40E_GLPES_PFTCPRXSEGSLO(_i) (0x00013000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPRXSEGSLO_MAX_INDEX 15
-#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0
-#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT)
-#define I40E_GLPES_PFTCPTXSEGHI(_i) (0x00013404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPTXSEGHI_MAX_INDEX 15
-#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0
-#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_MASK (0xFFFF << I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT)
-#define I40E_GLPES_PFTCPTXSEGLO(_i) (0x00013400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPTXSEGLO_MAX_INDEX 15
-#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0
-#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT)
-#define I40E_GLPES_PFUDPRXPKTSHI(_i) (0x00013804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFUDPRXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT)
-#define I40E_GLPES_PFUDPRXPKTSLO(_i) (0x00013800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFUDPRXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT)
-#define I40E_GLPES_PFUDPTXPKTSHI(_i) (0x00013A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFUDPTXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT)
-#define I40E_GLPES_PFUDPTXPKTSLO(_i) (0x00013A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFUDPTXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT)
-#define I40E_GLPES_RDMARXMULTFPDUSHI 0x0001E014 /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT 0
-#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_MASK (0xFFFFFF << I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT)
-#define I40E_GLPES_RDMARXMULTFPDUSLO 0x0001E010 /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT 0
-#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT)
-#define I40E_GLPES_RDMARXOOODDPHI 0x0001E01C /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT 0
-#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_MASK (0xFFFFFF << I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT)
-#define I40E_GLPES_RDMARXOOODDPLO 0x0001E018 /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT 0
-#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT)
-#define I40E_GLPES_RDMARXOOONOMARK 0x0001E004 /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT 0
-#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT)
-#define I40E_GLPES_RDMARXUNALIGN 0x0001E000 /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT 0
-#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT)
-#define I40E_GLPES_TCPRXFOURHOLEHI 0x0001E044 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT 0
-#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT)
-#define I40E_GLPES_TCPRXFOURHOLELO 0x0001E040 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT 0
-#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT)
-#define I40E_GLPES_TCPRXONEHOLEHI 0x0001E02C /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT 0
-#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT)
-#define I40E_GLPES_TCPRXONEHOLELO 0x0001E028 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT 0
-#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT)
-#define I40E_GLPES_TCPRXPUREACKHI 0x0001E024 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT 0
-#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT)
-#define I40E_GLPES_TCPRXPUREACKSLO 0x0001E020 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT 0
-#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT)
-#define I40E_GLPES_TCPRXTHREEHOLEHI 0x0001E03C /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT 0
-#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT)
-#define I40E_GLPES_TCPRXTHREEHOLELO 0x0001E038 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT 0
-#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT)
-#define I40E_GLPES_TCPRXTWOHOLEHI 0x0001E034 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT 0
-#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT)
-#define I40E_GLPES_TCPRXTWOHOLELO 0x0001E030 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT 0
-#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT)
-#define I40E_GLPES_TCPTXRETRANSFASTHI 0x0001E04C /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT 0
-#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT)
-#define I40E_GLPES_TCPTXRETRANSFASTLO 0x0001E048 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT 0
-#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT)
-#define I40E_GLPES_TCPTXTOUTSFASTHI 0x0001E054 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT 0
-#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT)
-#define I40E_GLPES_TCPTXTOUTSFASTLO 0x0001E050 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT 0
-#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT)
-#define I40E_GLPES_TCPTXTOUTSHI 0x0001E05C /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT 0
-#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT)
-#define I40E_GLPES_TCPTXTOUTSLO 0x0001E058 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT 0
-#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXDISCARD(_i) (0x00018600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXDISCARD_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0
-#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT)
-#define I40E_GLPES_VFIP4RXFRAGSHI(_i) (0x00018804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXFRAGSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0
-#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT)
-#define I40E_GLPES_VFIP4RXFRAGSLO(_i) (0x00018800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXFRAGSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0
-#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXMCOCTSHI(_i) (0x00018A04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXMCOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP4RXMCOCTSLO(_i) (0x00018A00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXMCOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXMCPKTSHI(_i) (0x00018C04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXMCPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP4RXMCPKTSLO(_i) (0x00018C00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXMCPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXOCTSHI(_i) (0x00018204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP4RXOCTSLO(_i) (0x00018200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXPKTSHI(_i) (0x00018404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP4RXPKTSLO(_i) (0x00018400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXTRUNC(_i) (0x00018700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXTRUNC_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0
-#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT)
-#define I40E_GLPES_VFIP4TXFRAGSHI(_i) (0x00019E04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXFRAGSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0
-#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT)
-#define I40E_GLPES_VFIP4TXFRAGSLO(_i) (0x00019E00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXFRAGSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0
-#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT)
-#define I40E_GLPES_VFIP4TXMCOCTSHI(_i) (0x0001A004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXMCOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP4TXMCOCTSLO(_i) (0x0001A000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXMCOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP4TXMCPKTSHI(_i) (0x0001A204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXMCPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP4TXMCPKTSLO(_i) (0x0001A200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXMCPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP4TXNOROUTE(_i) (0x0001AE00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXNOROUTE_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0
-#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT)
-#define I40E_GLPES_VFIP4TXOCTSHI(_i) (0x00019A04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP4TXOCTSLO(_i) (0x00019A00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP4TXPKTSHI(_i) (0x00019C04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP4TXPKTSLO(_i) (0x00019C00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXDISCARD(_i) (0x00019200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXDISCARD_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0
-#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT)
-#define I40E_GLPES_VFIP6RXFRAGSHI(_i) (0x00019404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXFRAGSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0
-#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT)
-#define I40E_GLPES_VFIP6RXFRAGSLO(_i) (0x00019400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXFRAGSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0
-#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXMCOCTSHI(_i) (0x00019604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXMCOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP6RXMCOCTSLO(_i) (0x00019600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXMCOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXMCPKTSHI(_i) (0x00019804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXMCPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP6RXMCPKTSLO(_i) (0x00019800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXMCPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXOCTSHI(_i) (0x00018E04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP6RXOCTSLO(_i) (0x00018E00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXPKTSHI(_i) (0x00019004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP6RXPKTSLO(_i) (0x00019000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXTRUNC(_i) (0x00019300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXTRUNC_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0
-#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT)
-#define I40E_GLPES_VFIP6TXFRAGSHI(_i) (0x0001A804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXFRAGSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0
-#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT)
-#define I40E_GLPES_VFIP6TXFRAGSLO(_i) (0x0001A800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXFRAGSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0
-#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT)
-#define I40E_GLPES_VFIP6TXMCOCTSHI(_i) (0x0001AA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXMCOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP6TXMCOCTSLO(_i) (0x0001AA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXMCOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP6TXMCPKTSHI(_i) (0x0001AC04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXMCPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP6TXMCPKTSLO(_i) (0x0001AC00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXMCPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP6TXNOROUTE(_i) (0x0001AF00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXNOROUTE_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0
-#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT)
-#define I40E_GLPES_VFIP6TXOCTSHI(_i) (0x0001A404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP6TXOCTSLO(_i) (0x0001A400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP6TXPKTSHI(_i) (0x0001A604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP6TXPKTSLO(_i) (0x0001A600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT)
-#define I40E_GLPES_VFRDMARXRDSHI(_i) (0x0001BE04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXRDSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0
-#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT)
-#define I40E_GLPES_VFRDMARXRDSLO(_i) (0x0001BE00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXRDSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0
-#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT)
-#define I40E_GLPES_VFRDMARXSNDSHI(_i) (0x0001C004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXSNDSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0
-#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT)
-#define I40E_GLPES_VFRDMARXSNDSLO(_i) (0x0001C000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXSNDSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0
-#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT)
-#define I40E_GLPES_VFRDMARXWRSHI(_i) (0x0001BC04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXWRSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0
-#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT)
-#define I40E_GLPES_VFRDMARXWRSLO(_i) (0x0001BC00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXWRSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0
-#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT)
-#define I40E_GLPES_VFRDMATXRDSHI(_i) (0x0001C404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXRDSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0
-#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT)
-#define I40E_GLPES_VFRDMATXRDSLO(_i) (0x0001C400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXRDSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0
-#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT)
-#define I40E_GLPES_VFRDMATXSNDSHI(_i) (0x0001C604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXSNDSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0
-#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT)
-#define I40E_GLPES_VFRDMATXSNDSLO(_i) (0x0001C600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXSNDSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0
-#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT)
-#define I40E_GLPES_VFRDMATXWRSHI(_i) (0x0001C204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXWRSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0
-#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT)
-#define I40E_GLPES_VFRDMATXWRSLO(_i) (0x0001C200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXWRSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0
-#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT)
-#define I40E_GLPES_VFRDMAVBNDHI(_i) (0x0001C804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMAVBNDHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0
-#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT)
-#define I40E_GLPES_VFRDMAVBNDLO(_i) (0x0001C800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMAVBNDLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0
-#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT)
-#define I40E_GLPES_VFRDMAVINVHI(_i) (0x0001CA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMAVINVHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT 0
-#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT)
-#define I40E_GLPES_VFRDMAVINVLO(_i) (0x0001CA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMAVINVLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT 0
-#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT)
-#define I40E_GLPES_VFRXVLANERR(_i) (0x00018000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRXVLANERR_MAX_INDEX 31
-#define I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT 0
-#define I40E_GLPES_VFRXVLANERR_RXVLANERR_MASK (0xFFFFFF << I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT)
-#define I40E_GLPES_VFTCPRTXSEG(_i) (0x0001B600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPRTXSEG_MAX_INDEX 31
-#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT 0
-#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT)
-#define I40E_GLPES_VFTCPRXOPTERR(_i) (0x0001B200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPRXOPTERR_MAX_INDEX 31
-#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0
-#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_MASK (0xFFFFFF << I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT)
-#define I40E_GLPES_VFTCPRXPROTOERR(_i) (0x0001B300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPRXPROTOERR_MAX_INDEX 31
-#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0
-#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_MASK (0xFFFFFF << I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT)
-#define I40E_GLPES_VFTCPRXSEGSHI(_i) (0x0001B004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPRXSEGSHI_MAX_INDEX 31
-#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0
-#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_MASK (0xFFFF << I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT)
-#define I40E_GLPES_VFTCPRXSEGSLO(_i) (0x0001B000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPRXSEGSLO_MAX_INDEX 31
-#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0
-#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT)
-#define I40E_GLPES_VFTCPTXSEGHI(_i) (0x0001B404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPTXSEGHI_MAX_INDEX 31
-#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0
-#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_MASK (0xFFFF << I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT)
-#define I40E_GLPES_VFTCPTXSEGLO(_i) (0x0001B400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPTXSEGLO_MAX_INDEX 31
-#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0
-#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT)
-#define I40E_GLPES_VFUDPRXPKTSHI(_i) (0x0001B804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFUDPRXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT)
-#define I40E_GLPES_VFUDPRXPKTSLO(_i) (0x0001B800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFUDPRXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT)
-#define I40E_GLPES_VFUDPTXPKTSHI(_i) (0x0001BA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFUDPTXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT)
-#define I40E_GLPES_VFUDPTXPKTSLO(_i) (0x0001BA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFUDPTXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT)
-
-#define I40E_VFPE_AEQALLOC1 0x0000A400 /* Reset: VFR */
-#define I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT 0
-#define I40E_VFPE_AEQALLOC1_AECOUNT_MASK (0xFFFFFFFF << I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT)
-#define I40E_VFPE_CCQPHIGH1 0x00009800 /* Reset: VFR */
-#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT 0
-#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT)
-#define I40E_VFPE_CCQPLOW1 0x0000AC00 /* Reset: VFR */
-#define I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT 0
-#define I40E_VFPE_CCQPLOW1_PECCQPLOW_MASK (0xFFFFFFFF << I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT)
-#define I40E_VFPE_CCQPSTATUS1 0x0000B800 /* Reset: VFR */
-#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT 0
-#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_MASK (0x1 << I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT)
-#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT 4
-#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_MASK (0x7 << I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT)
-#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT 16
-#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_MASK (0x3F << I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT)
-#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT 31
-#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_MASK (0x1 << I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT)
-#define I40E_VFPE_CQACK1 0x0000B000 /* Reset: VFR */
-#define I40E_VFPE_CQACK1_PECQID_SHIFT 0
-#define I40E_VFPE_CQACK1_PECQID_MASK (0x1FFFF << I40E_VFPE_CQACK1_PECQID_SHIFT)
-#define I40E_VFPE_CQARM1 0x0000B400 /* Reset: VFR */
-#define I40E_VFPE_CQARM1_PECQID_SHIFT 0
-#define I40E_VFPE_CQARM1_PECQID_MASK (0x1FFFF << I40E_VFPE_CQARM1_PECQID_SHIFT)
-#define I40E_VFPE_CQPDB1 0x0000BC00 /* Reset: VFR */
-#define I40E_VFPE_CQPDB1_WQHEAD_SHIFT 0
-#define I40E_VFPE_CQPDB1_WQHEAD_MASK (0x7FF << I40E_VFPE_CQPDB1_WQHEAD_SHIFT)
-#define I40E_VFPE_CQPERRCODES1 0x00009C00 /* Reset: VFR */
-#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT 0
-#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT)
-#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT 16
-#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT)
-#define I40E_VFPE_CQPTAIL1 0x0000A000 /* Reset: VFR */
-#define I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT 0
-#define I40E_VFPE_CQPTAIL1_WQTAIL_MASK (0x7FF << I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT)
-#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT 31
-#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_MASK (0x1 << I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT)
-#define I40E_VFPE_IPCONFIG01 0x00008C00 /* Reset: VFR */
-#define I40E_VFPE_IPCONFIG01_PEIPID_SHIFT 0
-#define I40E_VFPE_IPCONFIG01_PEIPID_MASK (0xFFFF << I40E_VFPE_IPCONFIG01_PEIPID_SHIFT)
-#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT 16
-#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_MASK (0x1 << I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT)
-#define I40E_VFPE_MRTEIDXMASK1 0x00009000 /* Reset: VFR */
-#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT 0
-#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_MASK (0x1F << I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT)
-#define I40E_VFPE_RCVUNEXPECTEDERROR1 0x00009400 /* Reset: VFR */
-#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT 0
-#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT)
-#define I40E_VFPE_TCPNOWTIMER1 0x0000A800 /* Reset: VFR */
-#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT 0
-#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_MASK (0xFFFFFFFF << I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT)
-#define I40E_VFPE_WQEALLOC1 0x0000C000 /* Reset: VFR */
-#define I40E_VFPE_WQEALLOC1_PEQPID_SHIFT 0
-#define I40E_VFPE_WQEALLOC1_PEQPID_MASK (0x3FFFF << I40E_VFPE_WQEALLOC1_PEQPID_SHIFT)
-#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT 20
-#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_MASK (0xFFF << I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT)
-#endif /* I40IW_REGISTER_H */
diff --git a/drivers/infiniband/hw/i40iw/i40iw_status.h b/drivers/infiniband/hw/i40iw/i40iw_status.h
deleted file mode 100644
index d1c5855bd8c3..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_status.h
+++ /dev/null
@@ -1,102 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_STATUS_H
-#define I40IW_STATUS_H
-
-/* Error Codes */
-enum i40iw_status_code {
- I40IW_SUCCESS = 0,
- I40IW_ERR_NVM = -1,
- I40IW_ERR_NVM_CHECKSUM = -2,
- I40IW_ERR_CONFIG = -4,
- I40IW_ERR_PARAM = -5,
- I40IW_ERR_DEVICE_NOT_SUPPORTED = -6,
- I40IW_ERR_RESET_FAILED = -7,
- I40IW_ERR_SWFW_SYNC = -8,
- I40IW_ERR_NO_MEMORY = -9,
- I40IW_ERR_BAD_PTR = -10,
- I40IW_ERR_INVALID_PD_ID = -11,
- I40IW_ERR_INVALID_QP_ID = -12,
- I40IW_ERR_INVALID_CQ_ID = -13,
- I40IW_ERR_INVALID_CEQ_ID = -14,
- I40IW_ERR_INVALID_AEQ_ID = -15,
- I40IW_ERR_INVALID_SIZE = -16,
- I40IW_ERR_INVALID_ARP_INDEX = -17,
- I40IW_ERR_INVALID_FPM_FUNC_ID = -18,
- I40IW_ERR_QP_INVALID_MSG_SIZE = -19,
- I40IW_ERR_QP_TOOMANY_WRS_POSTED = -20,
- I40IW_ERR_INVALID_FRAG_COUNT = -21,
- I40IW_ERR_QUEUE_EMPTY = -22,
- I40IW_ERR_INVALID_ALIGNMENT = -23,
- I40IW_ERR_FLUSHED_QUEUE = -24,
- I40IW_ERR_INVALID_PUSH_PAGE_INDEX = -25,
- I40IW_ERR_INVALID_INLINE_DATA_SIZE = -26,
- I40IW_ERR_TIMEOUT = -27,
- I40IW_ERR_OPCODE_MISMATCH = -28,
- I40IW_ERR_CQP_COMPL_ERROR = -29,
- I40IW_ERR_INVALID_VF_ID = -30,
- I40IW_ERR_INVALID_HMCFN_ID = -31,
- I40IW_ERR_BACKING_PAGE_ERROR = -32,
- I40IW_ERR_NO_PBLCHUNKS_AVAILABLE = -33,
- I40IW_ERR_INVALID_PBLE_INDEX = -34,
- I40IW_ERR_INVALID_SD_INDEX = -35,
- I40IW_ERR_INVALID_PAGE_DESC_INDEX = -36,
- I40IW_ERR_INVALID_SD_TYPE = -37,
- I40IW_ERR_MEMCPY_FAILED = -38,
- I40IW_ERR_INVALID_HMC_OBJ_INDEX = -39,
- I40IW_ERR_INVALID_HMC_OBJ_COUNT = -40,
- I40IW_ERR_INVALID_SRQ_ARM_LIMIT = -41,
- I40IW_ERR_SRQ_ENABLED = -42,
- I40IW_ERR_BUF_TOO_SHORT = -43,
- I40IW_ERR_BAD_IWARP_CQE = -44,
- I40IW_ERR_NVM_BLANK_MODE = -45,
- I40IW_ERR_NOT_IMPLEMENTED = -46,
- I40IW_ERR_PE_DOORBELL_NOT_ENABLED = -47,
- I40IW_ERR_NOT_READY = -48,
- I40IW_NOT_SUPPORTED = -49,
- I40IW_ERR_FIRMWARE_API_VERSION = -50,
- I40IW_ERR_RING_FULL = -51,
- I40IW_ERR_MPA_CRC = -61,
- I40IW_ERR_NO_TXBUFS = -62,
- I40IW_ERR_SEQ_NUM = -63,
- I40IW_ERR_list_empty = -64,
- I40IW_ERR_INVALID_MAC_ADDR = -65,
- I40IW_ERR_BAD_STAG = -66,
- I40IW_ERR_CQ_COMPL_ERROR = -67,
- I40IW_ERR_QUEUE_DESTROYED = -68,
- I40IW_ERR_INVALID_FEAT_CNT = -69
-
-};
-#endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_type.h b/drivers/infiniband/hw/i40iw/i40iw_type.h
deleted file mode 100644
index 54c323c40d96..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_type.h
+++ /dev/null
@@ -1,1375 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_TYPE_H
-#define I40IW_TYPE_H
-#include "i40iw_user.h"
-#include "i40iw_hmc.h"
-#include "i40iw_vf.h"
-#include "i40iw_virtchnl.h"
-
-struct i40iw_cqp_sq_wqe {
- u64 buf[I40IW_CQP_WQE_SIZE];
-};
-
-struct i40iw_sc_aeqe {
- u64 buf[I40IW_AEQE_SIZE];
-};
-
-struct i40iw_ceqe {
- u64 buf[I40IW_CEQE_SIZE];
-};
-
-struct i40iw_cqp_ctx {
- u64 buf[I40IW_CQP_CTX_SIZE];
-};
-
-struct i40iw_cq_shadow_area {
- u64 buf[I40IW_SHADOW_AREA_SIZE];
-};
-
-struct i40iw_sc_dev;
-struct i40iw_hmc_info;
-struct i40iw_vsi_pestat;
-
-struct i40iw_cqp_ops;
-struct i40iw_ccq_ops;
-struct i40iw_ceq_ops;
-struct i40iw_aeq_ops;
-struct i40iw_mr_ops;
-struct i40iw_cqp_misc_ops;
-struct i40iw_pd_ops;
-struct i40iw_priv_qp_ops;
-struct i40iw_priv_cq_ops;
-struct i40iw_hmc_ops;
-
-enum i40iw_page_size {
- I40IW_PAGE_SIZE_4K,
- I40IW_PAGE_SIZE_2M
-};
-
-enum i40iw_resource_indicator_type {
- I40IW_RSRC_INDICATOR_TYPE_ADAPTER = 0,
- I40IW_RSRC_INDICATOR_TYPE_CQ,
- I40IW_RSRC_INDICATOR_TYPE_QP,
- I40IW_RSRC_INDICATOR_TYPE_SRQ
-};
-
-enum i40iw_hdrct_flags {
- DDP_LEN_FLAG = 0x80,
- DDP_HDR_FLAG = 0x40,
- RDMA_HDR_FLAG = 0x20
-};
-
-enum i40iw_term_layers {
- LAYER_RDMA = 0,
- LAYER_DDP = 1,
- LAYER_MPA = 2
-};
-
-enum i40iw_term_error_types {
- RDMAP_REMOTE_PROT = 1,
- RDMAP_REMOTE_OP = 2,
- DDP_CATASTROPHIC = 0,
- DDP_TAGGED_BUFFER = 1,
- DDP_UNTAGGED_BUFFER = 2,
- DDP_LLP = 3
-};
-
-enum i40iw_term_rdma_errors {
- RDMAP_INV_STAG = 0x00,
- RDMAP_INV_BOUNDS = 0x01,
- RDMAP_ACCESS = 0x02,
- RDMAP_UNASSOC_STAG = 0x03,
- RDMAP_TO_WRAP = 0x04,
- RDMAP_INV_RDMAP_VER = 0x05,
- RDMAP_UNEXPECTED_OP = 0x06,
- RDMAP_CATASTROPHIC_LOCAL = 0x07,
- RDMAP_CATASTROPHIC_GLOBAL = 0x08,
- RDMAP_CANT_INV_STAG = 0x09,
- RDMAP_UNSPECIFIED = 0xff
-};
-
-enum i40iw_term_ddp_errors {
- DDP_CATASTROPHIC_LOCAL = 0x00,
- DDP_TAGGED_INV_STAG = 0x00,
- DDP_TAGGED_BOUNDS = 0x01,
- DDP_TAGGED_UNASSOC_STAG = 0x02,
- DDP_TAGGED_TO_WRAP = 0x03,
- DDP_TAGGED_INV_DDP_VER = 0x04,
- DDP_UNTAGGED_INV_QN = 0x01,
- DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02,
- DDP_UNTAGGED_INV_MSN_RANGE = 0x03,
- DDP_UNTAGGED_INV_MO = 0x04,
- DDP_UNTAGGED_INV_TOO_LONG = 0x05,
- DDP_UNTAGGED_INV_DDP_VER = 0x06
-};
-
-enum i40iw_term_mpa_errors {
- MPA_CLOSED = 0x01,
- MPA_CRC = 0x02,
- MPA_MARKER = 0x03,
- MPA_REQ_RSP = 0x04,
-};
-
-enum i40iw_flush_opcode {
- FLUSH_INVALID = 0,
- FLUSH_PROT_ERR,
- FLUSH_REM_ACCESS_ERR,
- FLUSH_LOC_QP_OP_ERR,
- FLUSH_REM_OP_ERR,
- FLUSH_LOC_LEN_ERR,
- FLUSH_GENERAL_ERR,
- FLUSH_FATAL_ERR
-};
-
-enum i40iw_term_eventtypes {
- TERM_EVENT_QP_FATAL,
- TERM_EVENT_QP_ACCESS_ERR
-};
-
-struct i40iw_terminate_hdr {
- u8 layer_etype;
- u8 error_code;
- u8 hdrct;
- u8 rsvd;
-};
-
-enum i40iw_debug_flag {
- I40IW_DEBUG_NONE = 0x00000000,
- I40IW_DEBUG_ERR = 0x00000001,
- I40IW_DEBUG_INIT = 0x00000002,
- I40IW_DEBUG_DEV = 0x00000004,
- I40IW_DEBUG_CM = 0x00000008,
- I40IW_DEBUG_VERBS = 0x00000010,
- I40IW_DEBUG_PUDA = 0x00000020,
- I40IW_DEBUG_ILQ = 0x00000040,
- I40IW_DEBUG_IEQ = 0x00000080,
- I40IW_DEBUG_QP = 0x00000100,
- I40IW_DEBUG_CQ = 0x00000200,
- I40IW_DEBUG_MR = 0x00000400,
- I40IW_DEBUG_PBLE = 0x00000800,
- I40IW_DEBUG_WQE = 0x00001000,
- I40IW_DEBUG_AEQ = 0x00002000,
- I40IW_DEBUG_CQP = 0x00004000,
- I40IW_DEBUG_HMC = 0x00008000,
- I40IW_DEBUG_USER = 0x00010000,
- I40IW_DEBUG_VIRT = 0x00020000,
- I40IW_DEBUG_DCB = 0x00040000,
- I40IW_DEBUG_CQE = 0x00800000,
- I40IW_DEBUG_ALL = 0xFFFFFFFF
-};
-
-enum i40iw_hw_stats_index_32b {
- I40IW_HW_STAT_INDEX_IP4RXDISCARD = 0,
- I40IW_HW_STAT_INDEX_IP4RXTRUNC,
- I40IW_HW_STAT_INDEX_IP4TXNOROUTE,
- I40IW_HW_STAT_INDEX_IP6RXDISCARD,
- I40IW_HW_STAT_INDEX_IP6RXTRUNC,
- I40IW_HW_STAT_INDEX_IP6TXNOROUTE,
- I40IW_HW_STAT_INDEX_TCPRTXSEG,
- I40IW_HW_STAT_INDEX_TCPRXOPTERR,
- I40IW_HW_STAT_INDEX_TCPRXPROTOERR,
- I40IW_HW_STAT_INDEX_MAX_32
-};
-
-enum i40iw_hw_stats_index_64b {
- I40IW_HW_STAT_INDEX_IP4RXOCTS = 0,
- I40IW_HW_STAT_INDEX_IP4RXPKTS,
- I40IW_HW_STAT_INDEX_IP4RXFRAGS,
- I40IW_HW_STAT_INDEX_IP4RXMCPKTS,
- I40IW_HW_STAT_INDEX_IP4TXOCTS,
- I40IW_HW_STAT_INDEX_IP4TXPKTS,
- I40IW_HW_STAT_INDEX_IP4TXFRAGS,
- I40IW_HW_STAT_INDEX_IP4TXMCPKTS,
- I40IW_HW_STAT_INDEX_IP6RXOCTS,
- I40IW_HW_STAT_INDEX_IP6RXPKTS,
- I40IW_HW_STAT_INDEX_IP6RXFRAGS,
- I40IW_HW_STAT_INDEX_IP6RXMCPKTS,
- I40IW_HW_STAT_INDEX_IP6TXOCTS,
- I40IW_HW_STAT_INDEX_IP6TXPKTS,
- I40IW_HW_STAT_INDEX_IP6TXFRAGS,
- I40IW_HW_STAT_INDEX_IP6TXMCPKTS,
- I40IW_HW_STAT_INDEX_TCPRXSEGS,
- I40IW_HW_STAT_INDEX_TCPTXSEG,
- I40IW_HW_STAT_INDEX_RDMARXRDS,
- I40IW_HW_STAT_INDEX_RDMARXSNDS,
- I40IW_HW_STAT_INDEX_RDMARXWRS,
- I40IW_HW_STAT_INDEX_RDMATXRDS,
- I40IW_HW_STAT_INDEX_RDMATXSNDS,
- I40IW_HW_STAT_INDEX_RDMATXWRS,
- I40IW_HW_STAT_INDEX_RDMAVBND,
- I40IW_HW_STAT_INDEX_RDMAVINV,
- I40IW_HW_STAT_INDEX_MAX_64
-};
-
-enum i40iw_feature_type {
- I40IW_FEATURE_FW_INFO = 0,
- I40IW_MAX_FEATURES
-};
-
-struct i40iw_dev_hw_stats_offsets {
- u32 stats_offset_32[I40IW_HW_STAT_INDEX_MAX_32];
- u32 stats_offset_64[I40IW_HW_STAT_INDEX_MAX_64];
-};
-
-struct i40iw_dev_hw_stats {
- u64 stats_value_32[I40IW_HW_STAT_INDEX_MAX_32];
- u64 stats_value_64[I40IW_HW_STAT_INDEX_MAX_64];
-};
-
-struct i40iw_vsi_pestat {
- struct i40iw_hw *hw;
- struct i40iw_dev_hw_stats hw_stats;
- struct i40iw_dev_hw_stats last_read_hw_stats;
- struct i40iw_dev_hw_stats_offsets hw_stats_offsets;
- struct timer_list stats_timer;
- struct i40iw_sc_vsi *vsi;
- spinlock_t lock; /* rdma stats lock */
-};
-
-struct i40iw_hw {
- u8 __iomem *hw_addr;
- void *dev_context;
- struct i40iw_hmc_info hmc;
-};
-
-struct i40iw_pfpdu {
- struct list_head rxlist;
- u32 rcv_nxt;
- u32 fps;
- u32 max_fpdu_data;
- bool mode;
- bool mpa_crc_err;
- u64 total_ieq_bufs;
- u64 fpdu_processed;
- u64 bad_seq_num;
- u64 crc_err;
- u64 no_tx_bufs;
- u64 tx_err;
- u64 out_of_order;
- u64 pmode_count;
-};
-
-struct i40iw_sc_pd {
- u32 size;
- struct i40iw_sc_dev *dev;
- u16 pd_id;
- int abi_ver;
-};
-
-struct i40iw_cqp_quanta {
- u64 elem[I40IW_CQP_WQE_SIZE];
-};
-
-struct i40iw_sc_cqp {
- u32 size;
- u64 sq_pa;
- u64 host_ctx_pa;
- void *back_cqp;
- struct i40iw_sc_dev *dev;
- enum i40iw_status_code (*process_cqp_sds)(struct i40iw_sc_dev *,
- struct i40iw_update_sds_info *);
- struct i40iw_dma_mem sdbuf;
- struct i40iw_ring sq_ring;
- struct i40iw_cqp_quanta *sq_base;
- u64 *host_ctx;
- u64 *scratch_array;
- u32 cqp_id;
- u32 sq_size;
- u32 hw_sq_size;
- u8 struct_ver;
- u8 polarity;
- bool en_datacenter_tcp;
- u8 hmc_profile;
- u8 enabled_vf_count;
- u8 timeout_count;
-};
-
-struct i40iw_sc_aeq {
- u32 size;
- u64 aeq_elem_pa;
- struct i40iw_sc_dev *dev;
- struct i40iw_sc_aeqe *aeqe_base;
- void *pbl_list;
- u32 elem_cnt;
- struct i40iw_ring aeq_ring;
- bool virtual_map;
- u8 pbl_chunk_size;
- u32 first_pm_pbl_idx;
- u8 polarity;
-};
-
-struct i40iw_sc_ceq {
- u32 size;
- u64 ceq_elem_pa;
- struct i40iw_sc_dev *dev;
- struct i40iw_ceqe *ceqe_base;
- void *pbl_list;
- u32 ceq_id;
- u32 elem_cnt;
- struct i40iw_ring ceq_ring;
- bool virtual_map;
- u8 pbl_chunk_size;
- bool tph_en;
- u8 tph_val;
- u32 first_pm_pbl_idx;
- u8 polarity;
-};
-
-struct i40iw_sc_cq {
- struct i40iw_cq_uk cq_uk;
- u64 cq_pa;
- u64 shadow_area_pa;
- struct i40iw_sc_dev *dev;
- struct i40iw_sc_vsi *vsi;
- void *pbl_list;
- void *back_cq;
- u32 ceq_id;
- u32 shadow_read_threshold;
- bool ceqe_mask;
- bool virtual_map;
- u8 pbl_chunk_size;
- u8 cq_type;
- bool ceq_id_valid;
- bool tph_en;
- u8 tph_val;
- u32 first_pm_pbl_idx;
- bool check_overflow;
-};
-
-struct i40iw_sc_qp {
- struct i40iw_qp_uk qp_uk;
- u64 sq_pa;
- u64 rq_pa;
- u64 hw_host_ctx_pa;
- u64 shadow_area_pa;
- u64 q2_pa;
- struct i40iw_sc_dev *dev;
- struct i40iw_sc_vsi *vsi;
- struct i40iw_sc_pd *pd;
- u64 *hw_host_ctx;
- void *llp_stream_handle;
- void *back_qp;
- struct i40iw_pfpdu pfpdu;
- u8 *q2_buf;
- u64 qp_compl_ctx;
- u16 qs_handle;
- u16 push_idx;
- u8 sq_tph_val;
- u8 rq_tph_val;
- u8 qp_state;
- u8 qp_type;
- u8 hw_sq_size;
- u8 hw_rq_size;
- u8 src_mac_addr_idx;
- bool sq_tph_en;
- bool rq_tph_en;
- bool rcv_tph_en;
- bool xmit_tph_en;
- bool virtual_map;
- bool flush_sq;
- bool flush_rq;
- u8 user_pri;
- struct list_head list;
- bool on_qoslist;
- bool sq_flush;
- enum i40iw_flush_opcode flush_code;
- enum i40iw_term_eventtypes eventtype;
- u8 term_flags;
-};
-
-struct i40iw_hmc_fpm_misc {
- u32 max_ceqs;
- u32 max_sds;
- u32 xf_block_size;
- u32 q1_block_size;
- u32 ht_multiplier;
- u32 timer_bucket;
-};
-
-struct i40iw_vchnl_if {
- enum i40iw_status_code (*vchnl_recv)(struct i40iw_sc_dev *, u32, u8 *, u16);
- enum i40iw_status_code (*vchnl_send)(struct i40iw_sc_dev *dev, u32, u8 *, u16);
-};
-
-#define I40IW_VCHNL_MAX_VF_MSG_SIZE 512
-
-struct i40iw_vchnl_vf_msg_buffer {
- struct i40iw_virtchnl_op_buf vchnl_msg;
- char parm_buffer[I40IW_VCHNL_MAX_VF_MSG_SIZE - 1];
-};
-
-struct i40iw_qos {
- struct list_head qplist;
- spinlock_t lock; /* qos list */
- u16 qs_handle;
-};
-
-struct i40iw_vfdev {
- struct i40iw_sc_dev *pf_dev;
- u8 *hmc_info_mem;
- struct i40iw_vsi_pestat pestat;
- struct i40iw_hmc_pble_info *pble_info;
- struct i40iw_hmc_info hmc_info;
- struct i40iw_vchnl_vf_msg_buffer vf_msg_buffer;
- u64 fpm_query_buf_pa;
- u64 *fpm_query_buf;
- u32 vf_id;
- u32 msg_count;
- bool pf_hmc_initialized;
- u16 pmf_index;
- u16 iw_vf_idx; /* VF Device table index */
- bool stats_initialized;
-};
-
-#define I40IW_INVALID_FCN_ID 0xff
-struct i40iw_sc_vsi {
- struct i40iw_sc_dev *dev;
- void *back_vsi; /* Owned by OS */
- u32 ilq_count;
- struct i40iw_virt_mem ilq_mem;
- struct i40iw_puda_rsrc *ilq;
- u32 ieq_count;
- struct i40iw_virt_mem ieq_mem;
- struct i40iw_puda_rsrc *ieq;
- u16 exception_lan_queue;
- u16 mtu;
- u8 fcn_id;
- bool stats_fcn_id_alloc;
- struct i40iw_qos qos[I40IW_MAX_USER_PRIORITY];
- struct i40iw_vsi_pestat *pestat;
-};
-
-struct i40iw_sc_dev {
- struct list_head cqp_cmd_head; /* head of the CQP command list */
- spinlock_t cqp_lock; /* cqp list sync */
- struct i40iw_dev_uk dev_uk;
- bool fcn_id_array[I40IW_MAX_STATS_COUNT];
- struct i40iw_dma_mem vf_fpm_query_buf[I40IW_MAX_PE_ENABLED_VF_COUNT];
- u64 fpm_query_buf_pa;
- u64 fpm_commit_buf_pa;
- u64 *fpm_query_buf;
- u64 *fpm_commit_buf;
- void *back_dev;
- struct i40iw_hw *hw;
- u8 __iomem *db_addr;
- struct i40iw_hmc_info *hmc_info;
- struct i40iw_hmc_pble_info *pble_info;
- struct i40iw_vfdev *vf_dev[I40IW_MAX_PE_ENABLED_VF_COUNT];
- struct i40iw_sc_cqp *cqp;
- struct i40iw_sc_aeq *aeq;
- struct i40iw_sc_ceq *ceq[I40IW_CEQ_MAX_COUNT];
- struct i40iw_sc_cq *ccq;
- struct i40iw_cqp_ops *cqp_ops;
- struct i40iw_ccq_ops *ccq_ops;
- struct i40iw_ceq_ops *ceq_ops;
- struct i40iw_aeq_ops *aeq_ops;
- struct i40iw_pd_ops *iw_pd_ops;
- struct i40iw_priv_qp_ops *iw_priv_qp_ops;
- struct i40iw_priv_cq_ops *iw_priv_cq_ops;
- struct i40iw_mr_ops *mr_ops;
- struct i40iw_cqp_misc_ops *cqp_misc_ops;
- struct i40iw_hmc_ops *hmc_ops;
- struct i40iw_vchnl_if vchnl_if;
- const struct i40iw_vf_cqp_ops *iw_vf_cqp_ops;
-
- struct i40iw_hmc_fpm_misc hmc_fpm_misc;
- u64 feature_info[I40IW_MAX_FEATURES];
- u32 debug_mask;
- u8 hmc_fn_id;
- bool is_pf;
- bool vchnl_up;
- bool ceq_valid;
- u8 vf_id;
- wait_queue_head_t vf_reqs;
- u64 cqp_cmd_stats[OP_SIZE_CQP_STAT_ARRAY];
- struct i40iw_vchnl_vf_msg_buffer vchnl_vf_msg_buf;
- u8 hw_rev;
-};
-
-struct i40iw_modify_cq_info {
- u64 cq_pa;
- struct i40iw_cqe *cq_base;
- void *pbl_list;
- u32 ceq_id;
- u32 cq_size;
- u32 shadow_read_threshold;
- bool virtual_map;
- u8 pbl_chunk_size;
- bool check_overflow;
- bool cq_resize;
- bool ceq_change;
- bool check_overflow_change;
- u32 first_pm_pbl_idx;
- bool ceq_valid;
-};
-
-struct i40iw_create_qp_info {
- u8 next_iwarp_state;
- bool ord_valid;
- bool tcp_ctx_valid;
- bool cq_num_valid;
- bool arp_cache_idx_valid;
-};
-
-struct i40iw_modify_qp_info {
- u64 rx_win0;
- u64 rx_win1;
- u8 next_iwarp_state;
- u8 termlen;
- bool ord_valid;
- bool tcp_ctx_valid;
- bool cq_num_valid;
- bool arp_cache_idx_valid;
- bool reset_tcp_conn;
- bool remove_hash_idx;
- bool dont_send_term;
- bool dont_send_fin;
- bool cached_var_valid;
- bool force_loopback;
-};
-
-struct i40iw_ccq_cqe_info {
- struct i40iw_sc_cqp *cqp;
- u64 scratch;
- u32 op_ret_val;
- u16 maj_err_code;
- u16 min_err_code;
- u8 op_code;
- bool error;
-};
-
-struct i40iw_l2params {
- u16 qs_handle_list[I40IW_MAX_USER_PRIORITY];
- u16 mtu;
-};
-
-struct i40iw_vsi_init_info {
- struct i40iw_sc_dev *dev;
- void *back_vsi;
- struct i40iw_l2params *params;
- u16 exception_lan_queue;
-};
-
-struct i40iw_vsi_stats_info {
- struct i40iw_vsi_pestat *pestat;
- u8 fcn_id;
- bool alloc_fcn_id;
- bool stats_initialize;
-};
-
-struct i40iw_device_init_info {
- u64 fpm_query_buf_pa;
- u64 fpm_commit_buf_pa;
- u64 *fpm_query_buf;
- u64 *fpm_commit_buf;
- struct i40iw_hw *hw;
- void __iomem *bar0;
- enum i40iw_status_code (*vchnl_send)(struct i40iw_sc_dev *, u32, u8 *, u16);
- u8 hmc_fn_id;
- bool is_pf;
- u32 debug_mask;
-};
-
-enum i40iw_cqp_hmc_profile {
- I40IW_HMC_PROFILE_DEFAULT = 1,
- I40IW_HMC_PROFILE_FAVOR_VF = 2,
- I40IW_HMC_PROFILE_EQUAL = 3,
-};
-
-struct i40iw_cqp_init_info {
- u64 cqp_compl_ctx;
- u64 host_ctx_pa;
- u64 sq_pa;
- struct i40iw_sc_dev *dev;
- struct i40iw_cqp_quanta *sq;
- u64 *host_ctx;
- u64 *scratch_array;
- u32 sq_size;
- u8 struct_ver;
- bool en_datacenter_tcp;
- u8 hmc_profile;
- u8 enabled_vf_count;
-};
-
-struct i40iw_ceq_init_info {
- u64 ceqe_pa;
- struct i40iw_sc_dev *dev;
- u64 *ceqe_base;
- void *pbl_list;
- u32 elem_cnt;
- u32 ceq_id;
- bool virtual_map;
- u8 pbl_chunk_size;
- bool tph_en;
- u8 tph_val;
- u32 first_pm_pbl_idx;
-};
-
-struct i40iw_aeq_init_info {
- u64 aeq_elem_pa;
- struct i40iw_sc_dev *dev;
- u32 *aeqe_base;
- void *pbl_list;
- u32 elem_cnt;
- bool virtual_map;
- u8 pbl_chunk_size;
- u32 first_pm_pbl_idx;
-};
-
-struct i40iw_ccq_init_info {
- u64 cq_pa;
- u64 shadow_area_pa;
- struct i40iw_sc_dev *dev;
- struct i40iw_cqe *cq_base;
- u64 *shadow_area;
- void *pbl_list;
- u32 num_elem;
- u32 ceq_id;
- u32 shadow_read_threshold;
- bool ceqe_mask;
- bool ceq_id_valid;
- bool tph_en;
- u8 tph_val;
- bool avoid_mem_cflct;
- bool virtual_map;
- u8 pbl_chunk_size;
- u32 first_pm_pbl_idx;
-};
-
-struct i40iwarp_offload_info {
- u16 rcv_mark_offset;
- u16 snd_mark_offset;
- u16 pd_id;
- u8 ddp_ver;
- u8 rdmap_ver;
- u8 ord_size;
- u8 ird_size;
- bool wr_rdresp_en;
- bool rd_enable;
- bool snd_mark_en;
- bool rcv_mark_en;
- bool bind_en;
- bool fast_reg_en;
- bool priv_mode_en;
- bool lsmm_present;
- u8 iwarp_mode;
- bool align_hdrs;
- bool rcv_no_mpa_crc;
-
- u8 last_byte_sent;
-};
-
-struct i40iw_tcp_offload_info {
- bool ipv4;
- bool no_nagle;
- bool insert_vlan_tag;
- bool time_stamp;
- u8 cwnd_inc_limit;
- bool drop_ooo_seg;
- u8 dup_ack_thresh;
- u8 ttl;
- u8 src_mac_addr_idx;
- bool avoid_stretch_ack;
- u8 tos;
- u16 src_port;
- u16 dst_port;
- u32 dest_ip_addr0;
- u32 dest_ip_addr1;
- u32 dest_ip_addr2;
- u32 dest_ip_addr3;
- u32 snd_mss;
- u16 vlan_tag;
- u16 arp_idx;
- u32 flow_label;
- bool wscale;
- u8 tcp_state;
- u8 snd_wscale;
- u8 rcv_wscale;
- u32 time_stamp_recent;
- u32 time_stamp_age;
- u32 snd_nxt;
- u32 snd_wnd;
- u32 rcv_nxt;
- u32 rcv_wnd;
- u32 snd_max;
- u32 snd_una;
- u32 srtt;
- u32 rtt_var;
- u32 ss_thresh;
- u32 cwnd;
- u32 snd_wl1;
- u32 snd_wl2;
- u32 max_snd_window;
- u8 rexmit_thresh;
- u32 local_ipaddr0;
- u32 local_ipaddr1;
- u32 local_ipaddr2;
- u32 local_ipaddr3;
- bool ignore_tcp_opt;
- bool ignore_tcp_uns_opt;
-};
-
-struct i40iw_qp_host_ctx_info {
- u64 qp_compl_ctx;
- struct i40iw_tcp_offload_info *tcp_info;
- struct i40iwarp_offload_info *iwarp_info;
- u32 send_cq_num;
- u32 rcv_cq_num;
- u16 push_idx;
- bool push_mode_en;
- bool tcp_info_valid;
- bool iwarp_info_valid;
- bool err_rq_idx_valid;
- u16 err_rq_idx;
- bool add_to_qoslist;
- u8 user_pri;
-};
-
-struct i40iw_aeqe_info {
- u64 compl_ctx;
- u32 qp_cq_id;
- u16 ae_id;
- u16 wqe_idx;
- u8 tcp_state;
- u8 iwarp_state;
- bool qp;
- bool cq;
- bool sq;
- bool in_rdrsp_wr;
- bool out_rdrsp;
- u8 q2_data_written;
- bool aeqe_overflow;
-};
-
-struct i40iw_allocate_stag_info {
- u64 total_len;
- u32 chunk_size;
- u32 stag_idx;
- u32 page_size;
- u16 pd_id;
- u16 access_rights;
- bool remote_access;
- bool use_hmc_fcn_index;
- u8 hmc_fcn_index;
- bool use_pf_rid;
-};
-
-struct i40iw_reg_ns_stag_info {
- u64 reg_addr_pa;
- u64 fbo;
- void *va;
- u64 total_len;
- u32 page_size;
- u32 chunk_size;
- u32 first_pm_pbl_index;
- enum i40iw_addressing_type addr_type;
- i40iw_stag_index stag_idx;
- u16 access_rights;
- u16 pd_id;
- i40iw_stag_key stag_key;
- bool use_hmc_fcn_index;
- u8 hmc_fcn_index;
- bool use_pf_rid;
-};
-
-struct i40iw_fast_reg_stag_info {
- u64 wr_id;
- u64 reg_addr_pa;
- u64 fbo;
- void *va;
- u64 total_len;
- u32 page_size;
- u32 chunk_size;
- u32 first_pm_pbl_index;
- enum i40iw_addressing_type addr_type;
- i40iw_stag_index stag_idx;
- u16 access_rights;
- u16 pd_id;
- i40iw_stag_key stag_key;
- bool local_fence;
- bool read_fence;
- bool signaled;
- bool use_hmc_fcn_index;
- u8 hmc_fcn_index;
- bool use_pf_rid;
- bool defer_flag;
-};
-
-struct i40iw_dealloc_stag_info {
- u32 stag_idx;
- u16 pd_id;
- bool mr;
- bool dealloc_pbl;
-};
-
-struct i40iw_register_shared_stag {
- void *va;
- enum i40iw_addressing_type addr_type;
- i40iw_stag_index new_stag_idx;
- i40iw_stag_index parent_stag_idx;
- u32 access_rights;
- u16 pd_id;
- i40iw_stag_key new_stag_key;
-};
-
-struct i40iw_qp_init_info {
- struct i40iw_qp_uk_init_info qp_uk_init_info;
- struct i40iw_sc_pd *pd;
- struct i40iw_sc_vsi *vsi;
- u64 *host_ctx;
- u8 *q2;
- u64 sq_pa;
- u64 rq_pa;
- u64 host_ctx_pa;
- u64 q2_pa;
- u64 shadow_area_pa;
- int abi_ver;
- u8 sq_tph_val;
- u8 rq_tph_val;
- u8 type;
- bool sq_tph_en;
- bool rq_tph_en;
- bool rcv_tph_en;
- bool xmit_tph_en;
- bool virtual_map;
-};
-
-struct i40iw_cq_init_info {
- struct i40iw_sc_dev *dev;
- u64 cq_base_pa;
- u64 shadow_area_pa;
- u32 ceq_id;
- u32 shadow_read_threshold;
- bool virtual_map;
- bool ceqe_mask;
- u8 pbl_chunk_size;
- u32 first_pm_pbl_idx;
- bool ceq_id_valid;
- bool tph_en;
- u8 tph_val;
- u8 type;
- struct i40iw_cq_uk_init_info cq_uk_init_info;
-};
-
-struct i40iw_upload_context_info {
- u64 buf_pa;
- bool freeze_qp;
- bool raw_format;
- u32 qp_id;
- u8 qp_type;
-};
-
-struct i40iw_add_arp_cache_entry_info {
- u8 mac_addr[6];
- u32 reach_max;
- u16 arp_index;
- bool permanent;
-};
-
-struct i40iw_apbvt_info {
- u16 port;
- bool add;
-};
-
-enum i40iw_quad_entry_type {
- I40IW_QHASH_TYPE_TCP_ESTABLISHED = 1,
- I40IW_QHASH_TYPE_TCP_SYN,
-};
-
-enum i40iw_quad_hash_manage_type {
- I40IW_QHASH_MANAGE_TYPE_DELETE = 0,
- I40IW_QHASH_MANAGE_TYPE_ADD,
- I40IW_QHASH_MANAGE_TYPE_MODIFY
-};
-
-struct i40iw_qhash_table_info {
- struct i40iw_sc_vsi *vsi;
- enum i40iw_quad_hash_manage_type manage;
- enum i40iw_quad_entry_type entry_type;
- bool vlan_valid;
- bool ipv4_valid;
- u8 mac_addr[6];
- u16 vlan_id;
- u8 user_pri;
- u32 qp_num;
- u32 dest_ip[4];
- u32 src_ip[4];
- u16 dest_port;
- u16 src_port;
-};
-
-struct i40iw_local_mac_ipaddr_entry_info {
- u8 mac_addr[6];
- u8 entry_idx;
-};
-
-struct i40iw_cqp_manage_push_page_info {
- u32 push_idx;
- u16 qs_handle;
- u8 free_page;
-};
-
-struct i40iw_qp_flush_info {
- u16 sq_minor_code;
- u16 sq_major_code;
- u16 rq_minor_code;
- u16 rq_major_code;
- u16 ae_code;
- u8 ae_source;
- bool sq;
- bool rq;
- bool userflushcode;
- bool generate_ae;
-};
-
-struct i40iw_cqp_commit_fpm_values {
- u64 qp_base;
- u64 cq_base;
- u32 hte_base;
- u32 arp_base;
- u32 apbvt_inuse_base;
- u32 mr_base;
- u32 xf_base;
- u32 xffl_base;
- u32 q1_base;
- u32 q1fl_base;
- u32 fsimc_base;
- u32 fsiav_base;
- u32 pbl_base;
-
- u32 qp_cnt;
- u32 cq_cnt;
- u32 hte_cnt;
- u32 arp_cnt;
- u32 mr_cnt;
- u32 xf_cnt;
- u32 xffl_cnt;
- u32 q1_cnt;
- u32 q1fl_cnt;
- u32 fsimc_cnt;
- u32 fsiav_cnt;
- u32 pbl_cnt;
-};
-
-struct i40iw_cqp_query_fpm_values {
- u16 first_pe_sd_index;
- u32 qp_objsize;
- u32 cq_objsize;
- u32 hte_objsize;
- u32 arp_objsize;
- u32 mr_objsize;
- u32 xf_objsize;
- u32 q1_objsize;
- u32 fsimc_objsize;
- u32 fsiav_objsize;
-
- u32 qp_max;
- u32 cq_max;
- u32 hte_max;
- u32 arp_max;
- u32 mr_max;
- u32 xf_max;
- u32 xffl_max;
- u32 q1_max;
- u32 q1fl_max;
- u32 fsimc_max;
- u32 fsiav_max;
- u32 pbl_max;
-};
-
-struct i40iw_gen_ae_info {
- u16 ae_code;
- u8 ae_source;
-};
-
-struct i40iw_cqp_ops {
- enum i40iw_status_code (*cqp_init)(struct i40iw_sc_cqp *,
- struct i40iw_cqp_init_info *);
- enum i40iw_status_code (*cqp_create)(struct i40iw_sc_cqp *, u16 *, u16 *);
- void (*cqp_post_sq)(struct i40iw_sc_cqp *);
- u64 *(*cqp_get_next_send_wqe)(struct i40iw_sc_cqp *, u64 scratch);
- enum i40iw_status_code (*cqp_destroy)(struct i40iw_sc_cqp *);
- enum i40iw_status_code (*poll_for_cqp_op_done)(struct i40iw_sc_cqp *, u8,
- struct i40iw_ccq_cqe_info *);
-};
-
-struct i40iw_ccq_ops {
- enum i40iw_status_code (*ccq_init)(struct i40iw_sc_cq *,
- struct i40iw_ccq_init_info *);
- enum i40iw_status_code (*ccq_create)(struct i40iw_sc_cq *, u64, bool, bool);
- enum i40iw_status_code (*ccq_destroy)(struct i40iw_sc_cq *, u64, bool);
- enum i40iw_status_code (*ccq_create_done)(struct i40iw_sc_cq *);
- enum i40iw_status_code (*ccq_get_cqe_info)(struct i40iw_sc_cq *,
- struct i40iw_ccq_cqe_info *);
- void (*ccq_arm)(struct i40iw_sc_cq *);
-};
-
-struct i40iw_ceq_ops {
- enum i40iw_status_code (*ceq_init)(struct i40iw_sc_ceq *,
- struct i40iw_ceq_init_info *);
- enum i40iw_status_code (*ceq_create)(struct i40iw_sc_ceq *, u64, bool);
- enum i40iw_status_code (*cceq_create_done)(struct i40iw_sc_ceq *);
- enum i40iw_status_code (*cceq_destroy_done)(struct i40iw_sc_ceq *);
- enum i40iw_status_code (*cceq_create)(struct i40iw_sc_ceq *, u64);
- enum i40iw_status_code (*ceq_destroy)(struct i40iw_sc_ceq *, u64, bool);
- void *(*process_ceq)(struct i40iw_sc_dev *, struct i40iw_sc_ceq *);
-};
-
-struct i40iw_aeq_ops {
- enum i40iw_status_code (*aeq_init)(struct i40iw_sc_aeq *,
- struct i40iw_aeq_init_info *);
- enum i40iw_status_code (*aeq_create)(struct i40iw_sc_aeq *, u64, bool);
- enum i40iw_status_code (*aeq_destroy)(struct i40iw_sc_aeq *, u64, bool);
- enum i40iw_status_code (*get_next_aeqe)(struct i40iw_sc_aeq *,
- struct i40iw_aeqe_info *);
- enum i40iw_status_code (*repost_aeq_entries)(struct i40iw_sc_dev *, u32);
- enum i40iw_status_code (*aeq_create_done)(struct i40iw_sc_aeq *);
- enum i40iw_status_code (*aeq_destroy_done)(struct i40iw_sc_aeq *);
-};
-
-struct i40iw_pd_ops {
- void (*pd_init)(struct i40iw_sc_dev *, struct i40iw_sc_pd *, u16, int);
-};
-
-struct i40iw_priv_qp_ops {
- enum i40iw_status_code (*qp_init)(struct i40iw_sc_qp *, struct i40iw_qp_init_info *);
- enum i40iw_status_code (*qp_create)(struct i40iw_sc_qp *,
- struct i40iw_create_qp_info *, u64, bool);
- enum i40iw_status_code (*qp_modify)(struct i40iw_sc_qp *,
- struct i40iw_modify_qp_info *, u64, bool);
- enum i40iw_status_code (*qp_destroy)(struct i40iw_sc_qp *, u64, bool, bool, bool);
- enum i40iw_status_code (*qp_flush_wqes)(struct i40iw_sc_qp *,
- struct i40iw_qp_flush_info *, u64, bool);
- enum i40iw_status_code (*qp_upload_context)(struct i40iw_sc_dev *,
- struct i40iw_upload_context_info *,
- u64, bool);
- enum i40iw_status_code (*qp_setctx)(struct i40iw_sc_qp *, u64 *,
- struct i40iw_qp_host_ctx_info *);
-
- void (*qp_send_lsmm)(struct i40iw_sc_qp *, void *, u32, i40iw_stag);
- void (*qp_send_lsmm_nostag)(struct i40iw_sc_qp *, void *, u32);
- void (*qp_send_rtt)(struct i40iw_sc_qp *, bool);
- enum i40iw_status_code (*qp_post_wqe0)(struct i40iw_sc_qp *, u8);
- enum i40iw_status_code (*iw_mr_fast_register)(struct i40iw_sc_qp *,
- struct i40iw_fast_reg_stag_info *,
- bool);
-};
-
-struct i40iw_priv_cq_ops {
- enum i40iw_status_code (*cq_init)(struct i40iw_sc_cq *, struct i40iw_cq_init_info *);
- enum i40iw_status_code (*cq_create)(struct i40iw_sc_cq *, u64, bool, bool);
- enum i40iw_status_code (*cq_destroy)(struct i40iw_sc_cq *, u64, bool);
- enum i40iw_status_code (*cq_modify)(struct i40iw_sc_cq *,
- struct i40iw_modify_cq_info *, u64, bool);
-};
-
-struct i40iw_mr_ops {
- enum i40iw_status_code (*alloc_stag)(struct i40iw_sc_dev *,
- struct i40iw_allocate_stag_info *, u64, bool);
- enum i40iw_status_code (*mr_reg_non_shared)(struct i40iw_sc_dev *,
- struct i40iw_reg_ns_stag_info *,
- u64, bool);
- enum i40iw_status_code (*mr_reg_shared)(struct i40iw_sc_dev *,
- struct i40iw_register_shared_stag *,
- u64, bool);
- enum i40iw_status_code (*dealloc_stag)(struct i40iw_sc_dev *,
- struct i40iw_dealloc_stag_info *,
- u64, bool);
- enum i40iw_status_code (*query_stag)(struct i40iw_sc_dev *, u64, u32, bool);
- enum i40iw_status_code (*mw_alloc)(struct i40iw_sc_dev *, u64, u32, u16, bool);
-};
-
-struct i40iw_cqp_misc_ops {
- enum i40iw_status_code (*manage_push_page)(struct i40iw_sc_cqp *,
- struct i40iw_cqp_manage_push_page_info *,
- u64, bool);
- enum i40iw_status_code (*manage_hmc_pm_func_table)(struct i40iw_sc_cqp *,
- u64, u8, bool, bool);
- enum i40iw_status_code (*set_hmc_resource_profile)(struct i40iw_sc_cqp *,
- u64, u8, u8, bool, bool);
- enum i40iw_status_code (*commit_fpm_values)(struct i40iw_sc_cqp *, u64, u8,
- struct i40iw_dma_mem *, bool, u8);
- enum i40iw_status_code (*query_fpm_values)(struct i40iw_sc_cqp *, u64, u8,
- struct i40iw_dma_mem *, bool, u8);
- enum i40iw_status_code (*static_hmc_pages_allocated)(struct i40iw_sc_cqp *,
- u64, u8, bool, bool);
- enum i40iw_status_code (*add_arp_cache_entry)(struct i40iw_sc_cqp *,
- struct i40iw_add_arp_cache_entry_info *,
- u64, bool);
- enum i40iw_status_code (*del_arp_cache_entry)(struct i40iw_sc_cqp *, u64, u16, bool);
- enum i40iw_status_code (*query_arp_cache_entry)(struct i40iw_sc_cqp *, u64, u16, bool);
- enum i40iw_status_code (*manage_apbvt_entry)(struct i40iw_sc_cqp *,
- struct i40iw_apbvt_info *, u64, bool);
- enum i40iw_status_code (*manage_qhash_table_entry)(struct i40iw_sc_cqp *,
- struct i40iw_qhash_table_info *, u64, bool);
- enum i40iw_status_code (*alloc_local_mac_ipaddr_table_entry)(struct i40iw_sc_cqp *, u64, bool);
- enum i40iw_status_code (*add_local_mac_ipaddr_entry)(struct i40iw_sc_cqp *,
- struct i40iw_local_mac_ipaddr_entry_info *,
- u64, bool);
- enum i40iw_status_code (*del_local_mac_ipaddr_entry)(struct i40iw_sc_cqp *, u64, u8, u8, bool);
- enum i40iw_status_code (*cqp_nop)(struct i40iw_sc_cqp *, u64, bool);
- enum i40iw_status_code (*commit_fpm_values_done)(struct i40iw_sc_cqp
- *);
- enum i40iw_status_code (*query_fpm_values_done)(struct i40iw_sc_cqp *);
- enum i40iw_status_code (*manage_hmc_pm_func_table_done)(struct i40iw_sc_cqp *);
- enum i40iw_status_code (*update_suspend_qp)(struct i40iw_sc_cqp *, struct i40iw_sc_qp *, u64);
- enum i40iw_status_code (*update_resume_qp)(struct i40iw_sc_cqp *, struct i40iw_sc_qp *, u64);
-};
-
-struct i40iw_hmc_ops {
- enum i40iw_status_code (*init_iw_hmc)(struct i40iw_sc_dev *, u8);
- enum i40iw_status_code (*parse_fpm_query_buf)(u64 *, struct i40iw_hmc_info *,
- struct i40iw_hmc_fpm_misc *);
- enum i40iw_status_code (*configure_iw_fpm)(struct i40iw_sc_dev *, u8);
- enum i40iw_status_code (*parse_fpm_commit_buf)(u64 *, struct i40iw_hmc_obj_info *, u32 *sd);
- enum i40iw_status_code (*create_hmc_object)(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_create_obj_info *);
- enum i40iw_status_code (*del_hmc_object)(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_del_obj_info *,
- bool reset);
- enum i40iw_status_code (*pf_init_vfhmc)(struct i40iw_sc_dev *, u8, u32 *);
- enum i40iw_status_code (*vf_configure_vffpm)(struct i40iw_sc_dev *, u32 *);
-};
-
-struct cqp_info {
- union {
- struct {
- struct i40iw_sc_qp *qp;
- struct i40iw_create_qp_info info;
- u64 scratch;
- } qp_create;
-
- struct {
- struct i40iw_sc_qp *qp;
- struct i40iw_modify_qp_info info;
- u64 scratch;
- } qp_modify;
-
- struct {
- struct i40iw_sc_qp *qp;
- u64 scratch;
- bool remove_hash_idx;
- bool ignore_mw_bnd;
- } qp_destroy;
-
- struct {
- struct i40iw_sc_cq *cq;
- u64 scratch;
- bool check_overflow;
- } cq_create;
-
- struct {
- struct i40iw_sc_cq *cq;
- u64 scratch;
- } cq_destroy;
-
- struct {
- struct i40iw_sc_dev *dev;
- struct i40iw_allocate_stag_info info;
- u64 scratch;
- } alloc_stag;
-
- struct {
- struct i40iw_sc_dev *dev;
- u64 scratch;
- u32 mw_stag_index;
- u16 pd_id;
- } mw_alloc;
-
- struct {
- struct i40iw_sc_dev *dev;
- struct i40iw_reg_ns_stag_info info;
- u64 scratch;
- } mr_reg_non_shared;
-
- struct {
- struct i40iw_sc_dev *dev;
- struct i40iw_dealloc_stag_info info;
- u64 scratch;
- } dealloc_stag;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- struct i40iw_local_mac_ipaddr_entry_info info;
- u64 scratch;
- } add_local_mac_ipaddr_entry;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- struct i40iw_add_arp_cache_entry_info info;
- u64 scratch;
- } add_arp_cache_entry;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- u64 scratch;
- u8 entry_idx;
- u8 ignore_ref_count;
- } del_local_mac_ipaddr_entry;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- u64 scratch;
- u16 arp_index;
- } del_arp_cache_entry;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- struct i40iw_manage_vf_pble_info info;
- u64 scratch;
- } manage_vf_pble_bp;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- struct i40iw_cqp_manage_push_page_info info;
- u64 scratch;
- } manage_push_page;
-
- struct {
- struct i40iw_sc_dev *dev;
- struct i40iw_upload_context_info info;
- u64 scratch;
- } qp_upload_context;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- u64 scratch;
- } alloc_local_mac_ipaddr_entry;
-
- struct {
- struct i40iw_sc_dev *dev;
- struct i40iw_hmc_fcn_info info;
- u64 scratch;
- } manage_hmc_pm;
-
- struct {
- struct i40iw_sc_ceq *ceq;
- u64 scratch;
- } ceq_create;
-
- struct {
- struct i40iw_sc_ceq *ceq;
- u64 scratch;
- } ceq_destroy;
-
- struct {
- struct i40iw_sc_aeq *aeq;
- u64 scratch;
- } aeq_create;
-
- struct {
- struct i40iw_sc_aeq *aeq;
- u64 scratch;
- } aeq_destroy;
-
- struct {
- struct i40iw_sc_qp *qp;
- struct i40iw_qp_flush_info info;
- u64 scratch;
- } qp_flush_wqes;
-
- struct {
- struct i40iw_sc_qp *qp;
- struct i40iw_gen_ae_info info;
- u64 scratch;
- } gen_ae;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- void *fpm_values_va;
- u64 fpm_values_pa;
- u8 hmc_fn_id;
- u64 scratch;
- } query_fpm_values;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- void *fpm_values_va;
- u64 fpm_values_pa;
- u8 hmc_fn_id;
- u64 scratch;
- } commit_fpm_values;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- struct i40iw_apbvt_info info;
- u64 scratch;
- } manage_apbvt_entry;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- struct i40iw_qhash_table_info info;
- u64 scratch;
- } manage_qhash_table_entry;
-
- struct {
- struct i40iw_sc_dev *dev;
- struct i40iw_update_sds_info info;
- u64 scratch;
- } update_pe_sds;
-
- struct {
- struct i40iw_sc_cqp *cqp;
- struct i40iw_sc_qp *qp;
- u64 scratch;
- } suspend_resume;
- struct {
- struct i40iw_sc_cqp *cqp;
- void *cap_va;
- u64 cap_pa;
- u64 scratch;
- } query_rdma_features;
- } u;
-};
-
-struct cqp_commands_info {
- struct list_head cqp_cmd_entry;
- u8 cqp_cmd;
- u8 post_sq;
- struct cqp_info in;
-};
-
-struct i40iw_virtchnl_work_info {
- void (*callback_fcn)(void *vf_dev);
- void *worker_vf_dev;
-};
-
-struct i40iw_cqp_timeout {
- u64 compl_cqp_cmds;
- u8 count;
-};
-
-#endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_uk.c b/drivers/infiniband/hw/i40iw/i40iw_uk.c
deleted file mode 100644
index 8afa5a67a86b..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_uk.c
+++ /dev/null
@@ -1,1232 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include "i40iw_osdep.h"
-#include "i40iw_status.h"
-#include "i40iw_d.h"
-#include "i40iw_user.h"
-#include "i40iw_register.h"
-
-static u32 nop_signature = 0x55550000;
-
-/**
- * i40iw_nop_1 - insert a nop wqe and move head. no post work
- * @qp: hw qp ptr
- */
-static enum i40iw_status_code i40iw_nop_1(struct i40iw_qp_uk *qp)
-{
- u64 header, *wqe;
- u64 *wqe_0 = NULL;
- u32 wqe_idx, peek_head;
- bool signaled = false;
-
- if (!qp->sq_ring.head)
- return I40IW_ERR_PARAM;
-
- wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- wqe = qp->sq_base[wqe_idx].elem;
-
- qp->sq_wrtrk_array[wqe_idx].wqe_size = I40IW_QP_WQE_MIN_SIZE;
-
- peek_head = (qp->sq_ring.head + 1) % qp->sq_ring.size;
- wqe_0 = qp->sq_base[peek_head].elem;
- if (peek_head)
- wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID);
- else
- wqe_0[3] = LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- set_64bit_val(wqe, 0, 0);
- set_64bit_val(wqe, 8, 0);
- set_64bit_val(wqe, 16, 0);
-
- header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |
- LS_64(signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID) | nop_signature++;
-
- wmb(); /* Memory barrier to ensure data is written before valid bit is set */
-
- set_64bit_val(wqe, 24, header);
- return 0;
-}
-
-/**
- * i40iw_qp_post_wr - post wr to hrdware
- * @qp: hw qp ptr
- */
-void i40iw_qp_post_wr(struct i40iw_qp_uk *qp)
-{
- u64 temp;
- u32 hw_sq_tail;
- u32 sw_sq_head;
-
- mb(); /* valid bit is written and loads completed before reading shadow */
-
- /* read the doorbell shadow area */
- get_64bit_val(qp->shadow_area, 0, &temp);
-
- hw_sq_tail = (u32)RS_64(temp, I40IW_QP_DBSA_HW_SQ_TAIL);
- sw_sq_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- if (sw_sq_head != hw_sq_tail) {
- if (sw_sq_head > qp->initial_ring.head) {
- if ((hw_sq_tail >= qp->initial_ring.head) &&
- (hw_sq_tail < sw_sq_head)) {
- writel(qp->qp_id, qp->wqe_alloc_reg);
- }
- } else if (sw_sq_head != qp->initial_ring.head) {
- if ((hw_sq_tail >= qp->initial_ring.head) ||
- (hw_sq_tail < sw_sq_head)) {
- writel(qp->qp_id, qp->wqe_alloc_reg);
- }
- }
- }
-
- qp->initial_ring.head = qp->sq_ring.head;
-}
-
-/**
- * i40iw_qp_ring_push_db - ring qp doorbell
- * @qp: hw qp ptr
- * @wqe_idx: wqe index
- */
-static void i40iw_qp_ring_push_db(struct i40iw_qp_uk *qp, u32 wqe_idx)
-{
- set_32bit_val(qp->push_db, 0, LS_32((wqe_idx >> 2), I40E_PFPE_WQEALLOC_WQE_DESC_INDEX) | qp->qp_id);
- qp->initial_ring.head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
-}
-
-/**
- * i40iw_qp_get_next_send_wqe - return next wqe ptr
- * @qp: hw qp ptr
- * @wqe_idx: return wqe index
- * @wqe_size: size of sq wqe
- */
-u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp,
- u32 *wqe_idx,
- u8 wqe_size,
- u32 total_size,
- u64 wr_id
- )
-{
- u64 *wqe = NULL;
- u64 wqe_ptr;
- u32 peek_head = 0;
- u16 offset;
- enum i40iw_status_code ret_code = 0;
- u8 nop_wqe_cnt = 0, i;
- u64 *wqe_0 = NULL;
-
- *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
-
- if (!*wqe_idx)
- qp->swqe_polarity = !qp->swqe_polarity;
- wqe_ptr = (uintptr_t)qp->sq_base[*wqe_idx].elem;
- offset = (u16)(wqe_ptr) & 0x7F;
- if ((offset + wqe_size) > I40IW_QP_WQE_MAX_SIZE) {
- nop_wqe_cnt = (u8)(I40IW_QP_WQE_MAX_SIZE - offset) / I40IW_QP_WQE_MIN_SIZE;
- for (i = 0; i < nop_wqe_cnt; i++) {
- i40iw_nop_1(qp);
- I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
- if (ret_code)
- return NULL;
- }
-
- *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- if (!*wqe_idx)
- qp->swqe_polarity = !qp->swqe_polarity;
- }
-
- if (((*wqe_idx & 3) == 1) && (wqe_size == I40IW_WQE_SIZE_64)) {
- i40iw_nop_1(qp);
- I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
- if (ret_code)
- return NULL;
- *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- if (!*wqe_idx)
- qp->swqe_polarity = !qp->swqe_polarity;
- }
- I40IW_RING_MOVE_HEAD_BY_COUNT(qp->sq_ring,
- wqe_size / I40IW_QP_WQE_MIN_SIZE, ret_code);
- if (ret_code)
- return NULL;
-
- wqe = qp->sq_base[*wqe_idx].elem;
-
- peek_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- wqe_0 = qp->sq_base[peek_head].elem;
-
- if (((peek_head & 3) == 1) || ((peek_head & 3) == 3)) {
- if (RS_64(wqe_0[3], I40IWQPSQ_VALID) != !qp->swqe_polarity)
- wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID);
- }
-
- qp->sq_wrtrk_array[*wqe_idx].wrid = wr_id;
- qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
- qp->sq_wrtrk_array[*wqe_idx].wqe_size = wqe_size;
- return wqe;
-}
-
-/**
- * i40iw_set_fragment - set fragment in wqe
- * @wqe: wqe for setting fragment
- * @offset: offset value
- * @sge: sge length and stag
- */
-static void i40iw_set_fragment(u64 *wqe, u32 offset, struct i40iw_sge *sge)
-{
- if (sge) {
- set_64bit_val(wqe, offset, LS_64(sge->tag_off, I40IWQPSQ_FRAG_TO));
- set_64bit_val(wqe, (offset + 8),
- (LS_64(sge->len, I40IWQPSQ_FRAG_LEN) |
- LS_64(sge->stag, I40IWQPSQ_FRAG_STAG)));
- }
-}
-
-/**
- * i40iw_qp_get_next_recv_wqe - get next qp's rcv wqe
- * @qp: hw qp ptr
- * @wqe_idx: return wqe index
- */
-u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx)
-{
- u64 *wqe = NULL;
- enum i40iw_status_code ret_code;
-
- if (I40IW_RING_FULL_ERR(qp->rq_ring))
- return NULL;
-
- I40IW_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code);
- if (ret_code)
- return NULL;
- if (!*wqe_idx)
- qp->rwqe_polarity = !qp->rwqe_polarity;
- /* rq_wqe_size_multiplier is no of qwords in one rq wqe */
- wqe = qp->rq_base[*wqe_idx * (qp->rq_wqe_size_multiplier >> 2)].elem;
-
- return wqe;
-}
-
-/**
- * i40iw_rdma_write - rdma write operation
- * @qp: hw qp ptr
- * @info: post sq information
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_rdma_write(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- bool post_sq)
-{
- u64 header;
- u64 *wqe;
- struct i40iw_rdma_write *op_info;
- u32 i, wqe_idx;
- u32 total_size = 0, byte_off;
- enum i40iw_status_code ret_code;
- bool read_fence = false;
- u8 wqe_size;
-
- op_info = &info->op.rdma_write;
- if (op_info->num_lo_sges > qp->max_sq_frag_cnt)
- return I40IW_ERR_INVALID_FRAG_COUNT;
-
- for (i = 0; i < op_info->num_lo_sges; i++)
- total_size += op_info->lo_sg_list[i].len;
-
- if (total_size > I40IW_MAX_OUTBOUND_MESSAGE_SIZE)
- return I40IW_ERR_QP_INVALID_MSG_SIZE;
-
- read_fence |= info->read_fence;
-
- ret_code = i40iw_fragcnt_to_wqesize_sq(op_info->num_lo_sges, &wqe_size);
- if (ret_code)
- return ret_code;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, total_size, info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- set_64bit_val(wqe, 16,
- LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO));
- if (!op_info->rem_addr.stag)
- return I40IW_ERR_BAD_STAG;
-
- header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) |
- LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |
- LS_64((op_info->num_lo_sges > 1 ? (op_info->num_lo_sges - 1) : 0), I40IWQPSQ_ADDFRAGCNT) |
- LS_64(read_fence, I40IWQPSQ_READFENCE) |
- LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_set_fragment(wqe, 0, op_info->lo_sg_list);
-
- for (i = 1, byte_off = 32; i < op_info->num_lo_sges; i++) {
- i40iw_set_fragment(wqe, byte_off, &op_info->lo_sg_list[i]);
- byte_off += 16;
- }
-
- wmb(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, 24, header);
-
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_rdma_read - rdma read command
- * @qp: hw qp ptr
- * @info: post sq information
- * @inv_stag: flag for inv_stag
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_rdma_read(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- bool inv_stag,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_rdma_read *op_info;
- u64 header;
- u32 wqe_idx;
- enum i40iw_status_code ret_code;
- u8 wqe_size;
- bool local_fence = false;
-
- op_info = &info->op.rdma_read;
- ret_code = i40iw_fragcnt_to_wqesize_sq(1, &wqe_size);
- if (ret_code)
- return ret_code;
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->lo_addr.len, info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- local_fence |= info->local_fence;
-
- set_64bit_val(wqe, 16, LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO));
- header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) |
- LS_64((inv_stag ? I40IWQP_OP_RDMA_READ_LOC_INV : I40IWQP_OP_RDMA_READ), I40IWQPSQ_OPCODE) |
- LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
- LS_64(local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_set_fragment(wqe, 0, &op_info->lo_addr);
-
- wmb(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, 24, header);
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_send - rdma send command
- * @qp: hw qp ptr
- * @info: post sq information
- * @stag_to_inv: stag_to_inv value
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_send(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- u32 stag_to_inv,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_post_send *op_info;
- u64 header;
- u32 i, wqe_idx, total_size = 0, byte_off;
- enum i40iw_status_code ret_code;
- bool read_fence = false;
- u8 wqe_size;
-
- op_info = &info->op.send;
- if (qp->max_sq_frag_cnt < op_info->num_sges)
- return I40IW_ERR_INVALID_FRAG_COUNT;
-
- for (i = 0; i < op_info->num_sges; i++)
- total_size += op_info->sg_list[i].len;
- ret_code = i40iw_fragcnt_to_wqesize_sq(op_info->num_sges, &wqe_size);
- if (ret_code)
- return ret_code;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, total_size, info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
-
- read_fence |= info->read_fence;
- set_64bit_val(wqe, 16, 0);
- header = LS_64(stag_to_inv, I40IWQPSQ_REMSTAG) |
- LS_64(info->op_type, I40IWQPSQ_OPCODE) |
- LS_64((op_info->num_sges > 1 ? (op_info->num_sges - 1) : 0),
- I40IWQPSQ_ADDFRAGCNT) |
- LS_64(read_fence, I40IWQPSQ_READFENCE) |
- LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_set_fragment(wqe, 0, op_info->sg_list);
-
- for (i = 1, byte_off = 32; i < op_info->num_sges; i++) {
- i40iw_set_fragment(wqe, byte_off, &op_info->sg_list[i]);
- byte_off += 16;
- }
-
- wmb(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, 24, header);
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_inline_rdma_write - inline rdma write operation
- * @qp: hw qp ptr
- * @info: post sq information
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- bool post_sq)
-{
- u64 *wqe;
- u8 *dest, *src;
- struct i40iw_inline_rdma_write *op_info;
- u64 *push;
- u64 header = 0;
- u32 wqe_idx;
- enum i40iw_status_code ret_code;
- bool read_fence = false;
- u8 wqe_size;
-
- op_info = &info->op.inline_rdma_write;
- if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE)
- return I40IW_ERR_INVALID_INLINE_DATA_SIZE;
-
- ret_code = i40iw_inline_data_size_to_wqesize(op_info->len, &wqe_size);
- if (ret_code)
- return ret_code;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->len, info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
-
- read_fence |= info->read_fence;
- set_64bit_val(wqe, 16,
- LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO));
-
- header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) |
- LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |
- LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) |
- LS_64(1, I40IWQPSQ_INLINEDATAFLAG) |
- LS_64((qp->push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) |
- LS_64(read_fence, I40IWQPSQ_READFENCE) |
- LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- dest = (u8 *)wqe;
- src = (u8 *)(op_info->data);
-
- if (op_info->len <= 16) {
- memcpy(dest, src, op_info->len);
- } else {
- memcpy(dest, src, 16);
- src += 16;
- dest = (u8 *)wqe + 32;
- memcpy(dest, src, op_info->len - 16);
- }
-
- wmb(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, 24, header);
-
- if (qp->push_db) {
- push = (u64 *)((uintptr_t)qp->push_wqe + (wqe_idx & 0x3) * 0x20);
- memcpy(push, wqe, (op_info->len > 16) ? op_info->len + 16 : 32);
- i40iw_qp_ring_push_db(qp, wqe_idx);
- } else {
- if (post_sq)
- i40iw_qp_post_wr(qp);
- }
-
- return 0;
-}
-
-/**
- * i40iw_inline_send - inline send operation
- * @qp: hw qp ptr
- * @info: post sq information
- * @stag_to_inv: remote stag
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- u32 stag_to_inv,
- bool post_sq)
-{
- u64 *wqe;
- u8 *dest, *src;
- struct i40iw_post_inline_send *op_info;
- u64 header;
- u32 wqe_idx;
- enum i40iw_status_code ret_code;
- bool read_fence = false;
- u8 wqe_size;
- u64 *push;
-
- op_info = &info->op.inline_send;
- if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE)
- return I40IW_ERR_INVALID_INLINE_DATA_SIZE;
-
- ret_code = i40iw_inline_data_size_to_wqesize(op_info->len, &wqe_size);
- if (ret_code)
- return ret_code;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->len, info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
-
- read_fence |= info->read_fence;
- header = LS_64(stag_to_inv, I40IWQPSQ_REMSTAG) |
- LS_64(info->op_type, I40IWQPSQ_OPCODE) |
- LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) |
- LS_64(1, I40IWQPSQ_INLINEDATAFLAG) |
- LS_64((qp->push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) |
- LS_64(read_fence, I40IWQPSQ_READFENCE) |
- LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- dest = (u8 *)wqe;
- src = (u8 *)(op_info->data);
-
- if (op_info->len <= 16) {
- memcpy(dest, src, op_info->len);
- } else {
- memcpy(dest, src, 16);
- src += 16;
- dest = (u8 *)wqe + 32;
- memcpy(dest, src, op_info->len - 16);
- }
-
- wmb(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, 24, header);
-
- if (qp->push_db) {
- push = (u64 *)((uintptr_t)qp->push_wqe + (wqe_idx & 0x3) * 0x20);
- memcpy(push, wqe, (op_info->len > 16) ? op_info->len + 16 : 32);
- i40iw_qp_ring_push_db(qp, wqe_idx);
- } else {
- if (post_sq)
- i40iw_qp_post_wr(qp);
- }
-
- return 0;
-}
-
-/**
- * i40iw_stag_local_invalidate - stag invalidate operation
- * @qp: hw qp ptr
- * @info: post sq information
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_stag_local_invalidate(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_inv_local_stag *op_info;
- u64 header;
- u32 wqe_idx;
- bool local_fence = false;
-
- op_info = &info->op.inv_local_stag;
- local_fence = info->local_fence;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- set_64bit_val(wqe, 0, 0);
- set_64bit_val(wqe, 8,
- LS_64(op_info->target_stag, I40IWQPSQ_LOCSTAG));
- set_64bit_val(wqe, 16, 0);
- header = LS_64(I40IW_OP_TYPE_INV_STAG, I40IWQPSQ_OPCODE) |
- LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
- LS_64(local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- wmb(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, 24, header);
-
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_mw_bind - Memory Window bind operation
- * @qp: hw qp ptr
- * @info: post sq information
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_mw_bind(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_bind_window *op_info;
- u64 header;
- u32 wqe_idx;
- bool local_fence = false;
-
- op_info = &info->op.bind_window;
-
- local_fence |= info->local_fence;
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- set_64bit_val(wqe, 0, (uintptr_t)op_info->va);
- set_64bit_val(wqe, 8,
- LS_64(op_info->mr_stag, I40IWQPSQ_PARENTMRSTAG) |
- LS_64(op_info->mw_stag, I40IWQPSQ_MWSTAG));
- set_64bit_val(wqe, 16, op_info->bind_length);
- header = LS_64(I40IW_OP_TYPE_BIND_MW, I40IWQPSQ_OPCODE) |
- LS_64(((op_info->enable_reads << 2) |
- (op_info->enable_writes << 3)),
- I40IWQPSQ_STAGRIGHTS) |
- LS_64((op_info->addressing_type == I40IW_ADDR_TYPE_VA_BASED ? 1 : 0),
- I40IWQPSQ_VABASEDTO) |
- LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
- LS_64(local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- wmb(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, 24, header);
-
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_post_receive - post receive wqe
- * @qp: hw qp ptr
- * @info: post rq information
- */
-static enum i40iw_status_code i40iw_post_receive(struct i40iw_qp_uk *qp,
- struct i40iw_post_rq_info *info)
-{
- u64 *wqe;
- u64 header;
- u32 total_size = 0, wqe_idx, i, byte_off;
-
- if (qp->max_rq_frag_cnt < info->num_sges)
- return I40IW_ERR_INVALID_FRAG_COUNT;
- for (i = 0; i < info->num_sges; i++)
- total_size += info->sg_list[i].len;
- wqe = i40iw_qp_get_next_recv_wqe(qp, &wqe_idx);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
-
- qp->rq_wrid_array[wqe_idx] = info->wr_id;
- set_64bit_val(wqe, 16, 0);
-
- header = LS_64((info->num_sges > 1 ? (info->num_sges - 1) : 0),
- I40IWQPSQ_ADDFRAGCNT) |
- LS_64(qp->rwqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_set_fragment(wqe, 0, info->sg_list);
-
- for (i = 1, byte_off = 32; i < info->num_sges; i++) {
- i40iw_set_fragment(wqe, byte_off, &info->sg_list[i]);
- byte_off += 16;
- }
-
- wmb(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, 24, header);
-
- return 0;
-}
-
-/**
- * i40iw_cq_request_notification - cq notification request (door bell)
- * @cq: hw cq
- * @cq_notify: notification type
- */
-static void i40iw_cq_request_notification(struct i40iw_cq_uk *cq,
- enum i40iw_completion_notify cq_notify)
-{
- u64 temp_val;
- u16 sw_cq_sel;
- u8 arm_next_se = 0;
- u8 arm_next = 0;
- u8 arm_seq_num;
-
- get_64bit_val(cq->shadow_area, 32, &temp_val);
- arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);
- arm_seq_num++;
-
- sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);
- arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);
- arm_next_se |= 1;
- if (cq_notify == IW_CQ_COMPL_EVENT)
- arm_next = 1;
- temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) |
- LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) |
- LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) |
- LS_64(arm_next, I40IW_CQ_DBSA_ARM_NEXT);
-
- set_64bit_val(cq->shadow_area, 32, temp_val);
-
- wmb(); /* make sure WQE is populated before valid bit is set */
-
- writel(cq->cq_id, cq->cqe_alloc_reg);
-}
-
-/**
- * i40iw_cq_post_entries - update tail in shadow memory
- * @cq: hw cq
- * @count: # of entries processed
- */
-static enum i40iw_status_code i40iw_cq_post_entries(struct i40iw_cq_uk *cq,
- u8 count)
-{
- I40IW_RING_MOVE_TAIL_BY_COUNT(cq->cq_ring, count);
- set_64bit_val(cq->shadow_area, 0,
- I40IW_RING_GETCURRENT_HEAD(cq->cq_ring));
- return 0;
-}
-
-/**
- * i40iw_cq_poll_completion - get cq completion info
- * @cq: hw cq
- * @info: cq poll information returned
- * @post_cq: update cq tail
- */
-static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq,
- struct i40iw_cq_poll_info *info)
-{
- u64 comp_ctx, qword0, qword2, qword3, wqe_qword;
- u64 *cqe, *sw_wqe;
- struct i40iw_qp_uk *qp;
- struct i40iw_ring *pring = NULL;
- u32 wqe_idx, q_type, array_idx = 0;
- enum i40iw_status_code ret_code = 0;
- bool move_cq_head = true;
- u8 polarity;
- u8 addl_wqes = 0;
-
- if (cq->avoid_mem_cflct)
- cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(cq);
- else
- cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(cq);
-
- get_64bit_val(cqe, 24, &qword3);
- polarity = (u8)RS_64(qword3, I40IW_CQ_VALID);
-
- if (polarity != cq->polarity)
- return I40IW_ERR_QUEUE_EMPTY;
-
- q_type = (u8)RS_64(qword3, I40IW_CQ_SQ);
- info->error = (bool)RS_64(qword3, I40IW_CQ_ERROR);
- info->push_dropped = (bool)RS_64(qword3, I40IWCQ_PSHDROP);
- if (info->error) {
- info->comp_status = I40IW_COMPL_STATUS_FLUSHED;
- info->major_err = (bool)RS_64(qword3, I40IW_CQ_MAJERR);
- info->minor_err = (bool)RS_64(qword3, I40IW_CQ_MINERR);
- } else {
- info->comp_status = I40IW_COMPL_STATUS_SUCCESS;
- }
-
- get_64bit_val(cqe, 0, &qword0);
- get_64bit_val(cqe, 16, &qword2);
-
- info->tcp_seq_num = (u32)RS_64(qword0, I40IWCQ_TCPSEQNUM);
-
- info->qp_id = (u32)RS_64(qword2, I40IWCQ_QPID);
-
- get_64bit_val(cqe, 8, &comp_ctx);
-
- info->solicited_event = (bool)RS_64(qword3, I40IWCQ_SOEVENT);
- info->is_srq = (bool)RS_64(qword3, I40IWCQ_SRQ);
-
- qp = (struct i40iw_qp_uk *)(unsigned long)comp_ctx;
- if (!qp) {
- ret_code = I40IW_ERR_QUEUE_DESTROYED;
- goto exit;
- }
- wqe_idx = (u32)RS_64(qword3, I40IW_CQ_WQEIDX);
- info->qp_handle = (i40iw_qp_handle)(unsigned long)qp;
-
- if (q_type == I40IW_CQE_QTYPE_RQ) {
- array_idx = (wqe_idx * 4) / qp->rq_wqe_size_multiplier;
- if (info->comp_status == I40IW_COMPL_STATUS_FLUSHED) {
- info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail];
- array_idx = qp->rq_ring.tail;
- } else {
- info->wr_id = qp->rq_wrid_array[array_idx];
- }
-
- info->op_type = I40IW_OP_TYPE_REC;
- if (qword3 & I40IWCQ_STAG_MASK) {
- info->stag_invalid_set = true;
- info->inv_stag = (u32)RS_64(qword2, I40IWCQ_INVSTAG);
- } else {
- info->stag_invalid_set = false;
- }
- info->bytes_xfered = (u32)RS_64(qword0, I40IWCQ_PAYLDLEN);
- I40IW_RING_SET_TAIL(qp->rq_ring, array_idx + 1);
- pring = &qp->rq_ring;
- } else {
- if (qp->first_sq_wq) {
- qp->first_sq_wq = false;
- if (!wqe_idx && (qp->sq_ring.head == qp->sq_ring.tail)) {
- I40IW_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
- I40IW_RING_MOVE_TAIL(cq->cq_ring);
- set_64bit_val(cq->shadow_area, 0,
- I40IW_RING_GETCURRENT_HEAD(cq->cq_ring));
- memset(info, 0, sizeof(struct i40iw_cq_poll_info));
- return i40iw_cq_poll_completion(cq, info);
- }
- }
-
- if (info->comp_status != I40IW_COMPL_STATUS_FLUSHED) {
- info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
- info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;
-
- info->op_type = (u8)RS_64(qword3, I40IWCQ_OP);
- sw_wqe = qp->sq_base[wqe_idx].elem;
- get_64bit_val(sw_wqe, 24, &wqe_qword);
-
- addl_wqes = qp->sq_wrtrk_array[wqe_idx].wqe_size / I40IW_QP_WQE_MIN_SIZE;
- I40IW_RING_SET_TAIL(qp->sq_ring, (wqe_idx + addl_wqes));
- } else {
- do {
- u8 op_type;
- u32 tail;
-
- tail = qp->sq_ring.tail;
- sw_wqe = qp->sq_base[tail].elem;
- get_64bit_val(sw_wqe, 24, &wqe_qword);
- op_type = (u8)RS_64(wqe_qword, I40IWQPSQ_OPCODE);
- info->op_type = op_type;
- addl_wqes = qp->sq_wrtrk_array[tail].wqe_size / I40IW_QP_WQE_MIN_SIZE;
- I40IW_RING_SET_TAIL(qp->sq_ring, (tail + addl_wqes));
- if (op_type != I40IWQP_OP_NOP) {
- info->wr_id = qp->sq_wrtrk_array[tail].wrid;
- info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
- break;
- }
- } while (1);
- }
- pring = &qp->sq_ring;
- }
-
- ret_code = 0;
-
-exit:
- if (!ret_code &&
- (info->comp_status == I40IW_COMPL_STATUS_FLUSHED))
- if (pring && (I40IW_RING_MORE_WORK(*pring)))
- move_cq_head = false;
-
- if (move_cq_head) {
- I40IW_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
-
- if (I40IW_RING_GETCURRENT_HEAD(cq->cq_ring) == 0)
- cq->polarity ^= 1;
-
- I40IW_RING_MOVE_TAIL(cq->cq_ring);
- set_64bit_val(cq->shadow_area, 0,
- I40IW_RING_GETCURRENT_HEAD(cq->cq_ring));
- } else {
- if (info->is_srq)
- return ret_code;
- qword3 &= ~I40IW_CQ_WQEIDX_MASK;
- qword3 |= LS_64(pring->tail, I40IW_CQ_WQEIDX);
- set_64bit_val(cqe, 24, qword3);
- }
-
- return ret_code;
-}
-
-/**
- * i40iw_get_wqe_shift - get shift count for maximum wqe size
- * @sge: Maximum Scatter Gather Elements wqe
- * @inline_data: Maximum inline data size
- * @shift: Returns the shift needed based on sge
- *
- * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size.
- * For 1 SGE or inline data <= 16, shift = 0 (wqe size of 32 bytes).
- * For 2 or 3 SGEs or inline data <= 48, shift = 1 (wqe size of 64 bytes).
- * Shift of 2 otherwise (wqe size of 128 bytes).
- */
-void i40iw_get_wqe_shift(u32 sge, u32 inline_data, u8 *shift)
-{
- *shift = 0;
- if (sge > 1 || inline_data > 16)
- *shift = (sge < 4 && inline_data <= 48) ? 1 : 2;
-}
-
-/*
- * i40iw_get_sqdepth - get SQ depth (quantas)
- * @sq_size: SQ size
- * @shift: shift which determines size of WQE
- * @sqdepth: depth of SQ
- *
- */
-enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth)
-{
- *sqdepth = roundup_pow_of_two((sq_size << shift) + I40IW_SQ_RSVD);
-
- if (*sqdepth < (I40IW_QP_SW_MIN_WQSIZE << shift))
- *sqdepth = I40IW_QP_SW_MIN_WQSIZE << shift;
- else if (*sqdepth > I40IW_QP_SW_MAX_SQ_QUANTAS)
- return I40IW_ERR_INVALID_SIZE;
-
- return 0;
-}
-
-/*
- * i40iw_get_rq_depth - get RQ depth (quantas)
- * @rq_size: RQ size
- * @shift: shift which determines size of WQE
- * @rqdepth: depth of RQ
- *
- */
-enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth)
-{
- *rqdepth = roundup_pow_of_two((rq_size << shift) + I40IW_RQ_RSVD);
-
- if (*rqdepth < (I40IW_QP_SW_MIN_WQSIZE << shift))
- *rqdepth = I40IW_QP_SW_MIN_WQSIZE << shift;
- else if (*rqdepth > I40IW_QP_SW_MAX_RQ_QUANTAS)
- return I40IW_ERR_INVALID_SIZE;
-
- return 0;
-}
-
-static const struct i40iw_qp_uk_ops iw_qp_uk_ops = {
- .iw_qp_post_wr = i40iw_qp_post_wr,
- .iw_qp_ring_push_db = i40iw_qp_ring_push_db,
- .iw_rdma_write = i40iw_rdma_write,
- .iw_rdma_read = i40iw_rdma_read,
- .iw_send = i40iw_send,
- .iw_inline_rdma_write = i40iw_inline_rdma_write,
- .iw_inline_send = i40iw_inline_send,
- .iw_stag_local_invalidate = i40iw_stag_local_invalidate,
- .iw_mw_bind = i40iw_mw_bind,
- .iw_post_receive = i40iw_post_receive,
- .iw_post_nop = i40iw_nop
-};
-
-static const struct i40iw_cq_ops iw_cq_ops = {
- .iw_cq_request_notification = i40iw_cq_request_notification,
- .iw_cq_poll_completion = i40iw_cq_poll_completion,
- .iw_cq_post_entries = i40iw_cq_post_entries,
- .iw_cq_clean = i40iw_clean_cq
-};
-
-static const struct i40iw_device_uk_ops iw_device_uk_ops = {
- .iwarp_cq_uk_init = i40iw_cq_uk_init,
- .iwarp_qp_uk_init = i40iw_qp_uk_init,
-};
-
-/**
- * i40iw_qp_uk_init - initialize shared qp
- * @qp: hw qp (user and kernel)
- * @info: qp initialization info
- *
- * initializes the vars used in both user and kernel mode.
- * size of the wqe depends on numbers of max. fragements
- * allowed. Then size of wqe * the number of wqes should be the
- * amount of memory allocated for sq and rq. If srq is used,
- * then rq_base will point to one rq wqe only (not the whole
- * array of wqes)
- */
-enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp,
- struct i40iw_qp_uk_init_info *info)
-{
- enum i40iw_status_code ret_code = 0;
- u32 sq_ring_size;
- u8 sqshift, rqshift;
-
- if (info->max_sq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
- return I40IW_ERR_INVALID_FRAG_COUNT;
-
- if (info->max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
- return I40IW_ERR_INVALID_FRAG_COUNT;
- i40iw_get_wqe_shift(info->max_sq_frag_cnt, info->max_inline_data, &sqshift);
-
- qp->sq_base = info->sq;
- qp->rq_base = info->rq;
- qp->shadow_area = info->shadow_area;
- qp->sq_wrtrk_array = info->sq_wrtrk_array;
- qp->rq_wrid_array = info->rq_wrid_array;
-
- qp->wqe_alloc_reg = info->wqe_alloc_reg;
- qp->qp_id = info->qp_id;
-
- qp->sq_size = info->sq_size;
- qp->push_db = info->push_db;
- qp->push_wqe = info->push_wqe;
-
- qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
- sq_ring_size = qp->sq_size << sqshift;
-
- I40IW_RING_INIT(qp->sq_ring, sq_ring_size);
- I40IW_RING_INIT(qp->initial_ring, sq_ring_size);
- I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
- I40IW_RING_MOVE_TAIL(qp->sq_ring);
- I40IW_RING_MOVE_HEAD(qp->initial_ring, ret_code);
- qp->swqe_polarity = 1;
- qp->first_sq_wq = true;
- qp->swqe_polarity_deferred = 1;
- qp->rwqe_polarity = 0;
-
- if (!qp->use_srq) {
- qp->rq_size = info->rq_size;
- qp->max_rq_frag_cnt = info->max_rq_frag_cnt;
- I40IW_RING_INIT(qp->rq_ring, qp->rq_size);
- switch (info->abi_ver) {
- case 4:
- i40iw_get_wqe_shift(info->max_rq_frag_cnt, 0, &rqshift);
- break;
- case 5: /* fallthrough until next ABI version */
- default:
- rqshift = I40IW_MAX_RQ_WQE_SHIFT;
- break;
- }
- qp->rq_wqe_size = rqshift;
- qp->rq_wqe_size_multiplier = 4 << rqshift;
- }
- qp->ops = iw_qp_uk_ops;
-
- return ret_code;
-}
-
-/**
- * i40iw_cq_uk_init - initialize shared cq (user and kernel)
- * @cq: hw cq
- * @info: hw cq initialization info
- */
-enum i40iw_status_code i40iw_cq_uk_init(struct i40iw_cq_uk *cq,
- struct i40iw_cq_uk_init_info *info)
-{
- if ((info->cq_size < I40IW_MIN_CQ_SIZE) ||
- (info->cq_size > I40IW_MAX_CQ_SIZE))
- return I40IW_ERR_INVALID_SIZE;
- cq->cq_base = (struct i40iw_cqe *)info->cq_base;
- cq->cq_id = info->cq_id;
- cq->cq_size = info->cq_size;
- cq->cqe_alloc_reg = info->cqe_alloc_reg;
- cq->shadow_area = info->shadow_area;
- cq->avoid_mem_cflct = info->avoid_mem_cflct;
-
- I40IW_RING_INIT(cq->cq_ring, cq->cq_size);
- cq->polarity = 1;
- cq->ops = iw_cq_ops;
-
- return 0;
-}
-
-/**
- * i40iw_device_init_uk - setup routines for iwarp shared device
- * @dev: iwarp shared (user and kernel)
- */
-void i40iw_device_init_uk(struct i40iw_dev_uk *dev)
-{
- dev->ops_uk = iw_device_uk_ops;
-}
-
-/**
- * i40iw_clean_cq - clean cq entries
- * @ queue completion context
- * @cq: cq to clean
- */
-void i40iw_clean_cq(void *queue, struct i40iw_cq_uk *cq)
-{
- u64 *cqe;
- u64 qword3, comp_ctx;
- u32 cq_head;
- u8 polarity, temp;
-
- cq_head = cq->cq_ring.head;
- temp = cq->polarity;
- do {
- if (cq->avoid_mem_cflct)
- cqe = (u64 *)&(((struct i40iw_extended_cqe *)cq->cq_base)[cq_head]);
- else
- cqe = (u64 *)&cq->cq_base[cq_head];
- get_64bit_val(cqe, 24, &qword3);
- polarity = (u8)RS_64(qword3, I40IW_CQ_VALID);
-
- if (polarity != temp)
- break;
-
- get_64bit_val(cqe, 8, &comp_ctx);
- if ((void *)(unsigned long)comp_ctx == queue)
- set_64bit_val(cqe, 8, 0);
-
- cq_head = (cq_head + 1) % cq->cq_ring.size;
- if (!cq_head)
- temp ^= 1;
- } while (true);
-}
-
-/**
- * i40iw_nop - send a nop
- * @qp: hw qp ptr
- * @wr_id: work request id
- * @signaled: flag if signaled for completion
- * @post_sq: flag to post sq
- */
-enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp,
- u64 wr_id,
- bool signaled,
- bool post_sq)
-{
- u64 header, *wqe;
- u32 wqe_idx;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- set_64bit_val(wqe, 0, 0);
- set_64bit_val(wqe, 8, 0);
- set_64bit_val(wqe, 16, 0);
-
- header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |
- LS_64(signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- wmb(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, 24, header);
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_fragcnt_to_wqesize_sq - calculate wqe size based on fragment count for SQ
- * @frag_cnt: number of fragments
- * @wqe_size: size of sq wqe returned
- */
-enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size)
-{
- switch (frag_cnt) {
- case 0:
- case 1:
- *wqe_size = I40IW_QP_WQE_MIN_SIZE;
- break;
- case 2:
- case 3:
- *wqe_size = 64;
- break;
- case 4:
- case 5:
- *wqe_size = 96;
- break;
- case 6:
- case 7:
- *wqe_size = 128;
- break;
- default:
- return I40IW_ERR_INVALID_FRAG_COUNT;
- }
-
- return 0;
-}
-
-/**
- * i40iw_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ
- * @frag_cnt: number of fragments
- * @wqe_size: size of rq wqe returned
- */
-enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size)
-{
- switch (frag_cnt) {
- case 0:
- case 1:
- *wqe_size = 32;
- break;
- case 2:
- case 3:
- *wqe_size = 64;
- break;
- case 4:
- case 5:
- case 6:
- case 7:
- *wqe_size = 128;
- break;
- default:
- return I40IW_ERR_INVALID_FRAG_COUNT;
- }
-
- return 0;
-}
-
-/**
- * i40iw_inline_data_size_to_wqesize - based on inline data, wqe size
- * @data_size: data size for inline
- * @wqe_size: size of sq wqe returned
- */
-enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size,
- u8 *wqe_size)
-{
- if (data_size > I40IW_MAX_INLINE_DATA_SIZE)
- return I40IW_ERR_INVALID_INLINE_DATA_SIZE;
-
- if (data_size <= 16)
- *wqe_size = I40IW_QP_WQE_MIN_SIZE;
- else
- *wqe_size = 64;
-
- return 0;
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_user.h b/drivers/infiniband/hw/i40iw/i40iw_user.h
deleted file mode 100644
index b125925641e0..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_user.h
+++ /dev/null
@@ -1,430 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_USER_H
-#define I40IW_USER_H
-
-enum i40iw_device_capabilities_const {
- I40IW_WQE_SIZE = 4,
- I40IW_CQP_WQE_SIZE = 8,
- I40IW_CQE_SIZE = 4,
- I40IW_EXTENDED_CQE_SIZE = 8,
- I40IW_AEQE_SIZE = 2,
- I40IW_CEQE_SIZE = 1,
- I40IW_CQP_CTX_SIZE = 8,
- I40IW_SHADOW_AREA_SIZE = 8,
- I40IW_CEQ_MAX_COUNT = 256,
- I40IW_QUERY_FPM_BUF_SIZE = 128,
- I40IW_COMMIT_FPM_BUF_SIZE = 128,
- I40IW_MIN_IW_QP_ID = 1,
- I40IW_MAX_IW_QP_ID = 262143,
- I40IW_MIN_CEQID = 0,
- I40IW_MAX_CEQID = 256,
- I40IW_MIN_CQID = 0,
- I40IW_MAX_CQID = 131071,
- I40IW_MIN_AEQ_ENTRIES = 1,
- I40IW_MAX_AEQ_ENTRIES = 524287,
- I40IW_MIN_CEQ_ENTRIES = 1,
- I40IW_MAX_CEQ_ENTRIES = 131071,
- I40IW_MIN_CQ_SIZE = 1,
- I40IW_MAX_CQ_SIZE = 1048575,
- I40IW_DB_ID_ZERO = 0,
- I40IW_MAX_WQ_FRAGMENT_COUNT = 3,
- I40IW_MAX_SGE_RD = 1,
- I40IW_MAX_OUTBOUND_MESSAGE_SIZE = 2147483647,
- I40IW_MAX_INBOUND_MESSAGE_SIZE = 2147483647,
- I40IW_MAX_PUSH_PAGE_COUNT = 4096,
- I40IW_MAX_PE_ENABLED_VF_COUNT = 32,
- I40IW_MAX_VF_FPM_ID = 47,
- I40IW_MAX_VF_PER_PF = 127,
- I40IW_MAX_SQ_PAYLOAD_SIZE = 2145386496,
- I40IW_MAX_INLINE_DATA_SIZE = 48,
- I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE = 48,
- I40IW_MAX_IRD_SIZE = 64,
- I40IW_MAX_ORD_SIZE = 127,
- I40IW_MAX_WQ_ENTRIES = 2048,
- I40IW_Q2_BUFFER_SIZE = (248 + 100),
- I40IW_MAX_WQE_SIZE_RQ = 128,
- I40IW_QP_CTX_SIZE = 248,
- I40IW_MAX_PDS = 32768
-};
-
-#define i40iw_handle void *
-#define i40iw_adapter_handle i40iw_handle
-#define i40iw_qp_handle i40iw_handle
-#define i40iw_cq_handle i40iw_handle
-#define i40iw_srq_handle i40iw_handle
-#define i40iw_pd_id i40iw_handle
-#define i40iw_stag_handle i40iw_handle
-#define i40iw_stag_index u32
-#define i40iw_stag u32
-#define i40iw_stag_key u8
-
-#define i40iw_tagged_offset u64
-#define i40iw_access_privileges u32
-#define i40iw_physical_fragment u64
-#define i40iw_address_list u64 *
-
-#define I40IW_MAX_MR_SIZE 0x10000000000L
-#define I40IW_MAX_RQ_WQE_SHIFT 2
-
-struct i40iw_qp_uk;
-struct i40iw_cq_uk;
-struct i40iw_srq_uk;
-struct i40iw_qp_uk_init_info;
-struct i40iw_cq_uk_init_info;
-struct i40iw_srq_uk_init_info;
-
-struct i40iw_sge {
- i40iw_tagged_offset tag_off;
- u32 len;
- i40iw_stag stag;
-};
-
-#define i40iw_sgl struct i40iw_sge *
-
-struct i40iw_ring {
- u32 head;
- u32 tail;
- u32 size;
-};
-
-struct i40iw_cqe {
- u64 buf[I40IW_CQE_SIZE];
-};
-
-struct i40iw_extended_cqe {
- u64 buf[I40IW_EXTENDED_CQE_SIZE];
-};
-
-struct i40iw_wqe {
- u64 buf[I40IW_WQE_SIZE];
-};
-
-struct i40iw_qp_uk_ops;
-
-enum i40iw_addressing_type {
- I40IW_ADDR_TYPE_ZERO_BASED = 0,
- I40IW_ADDR_TYPE_VA_BASED = 1,
-};
-
-#define I40IW_ACCESS_FLAGS_LOCALREAD 0x01
-#define I40IW_ACCESS_FLAGS_LOCALWRITE 0x02
-#define I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04
-#define I40IW_ACCESS_FLAGS_REMOTEREAD 0x05
-#define I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08
-#define I40IW_ACCESS_FLAGS_REMOTEWRITE 0x0a
-#define I40IW_ACCESS_FLAGS_BIND_WINDOW 0x10
-#define I40IW_ACCESS_FLAGS_ALL 0x1F
-
-#define I40IW_OP_TYPE_RDMA_WRITE 0
-#define I40IW_OP_TYPE_RDMA_READ 1
-#define I40IW_OP_TYPE_SEND 3
-#define I40IW_OP_TYPE_SEND_INV 4
-#define I40IW_OP_TYPE_SEND_SOL 5
-#define I40IW_OP_TYPE_SEND_SOL_INV 6
-#define I40IW_OP_TYPE_REC 7
-#define I40IW_OP_TYPE_BIND_MW 8
-#define I40IW_OP_TYPE_FAST_REG_NSMR 9
-#define I40IW_OP_TYPE_INV_STAG 10
-#define I40IW_OP_TYPE_RDMA_READ_INV_STAG 11
-#define I40IW_OP_TYPE_NOP 12
-
-enum i40iw_completion_status {
- I40IW_COMPL_STATUS_SUCCESS = 0,
- I40IW_COMPL_STATUS_FLUSHED,
- I40IW_COMPL_STATUS_INVALID_WQE,
- I40IW_COMPL_STATUS_QP_CATASTROPHIC,
- I40IW_COMPL_STATUS_REMOTE_TERMINATION,
- I40IW_COMPL_STATUS_INVALID_STAG,
- I40IW_COMPL_STATUS_BASE_BOUND_VIOLATION,
- I40IW_COMPL_STATUS_ACCESS_VIOLATION,
- I40IW_COMPL_STATUS_INVALID_PD_ID,
- I40IW_COMPL_STATUS_WRAP_ERROR,
- I40IW_COMPL_STATUS_STAG_INVALID_PDID,
- I40IW_COMPL_STATUS_RDMA_READ_ZERO_ORD,
- I40IW_COMPL_STATUS_QP_NOT_PRIVLEDGED,
- I40IW_COMPL_STATUS_STAG_NOT_INVALID,
- I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_SIZE,
- I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_ENTRY,
- I40IW_COMPL_STATUS_INVALID_FBO,
- I40IW_COMPL_STATUS_INVALID_LENGTH,
- I40IW_COMPL_STATUS_INVALID_ACCESS,
- I40IW_COMPL_STATUS_PHYS_BUFFER_LIST_TOO_LONG,
- I40IW_COMPL_STATUS_INVALID_VIRT_ADDRESS,
- I40IW_COMPL_STATUS_INVALID_REGION,
- I40IW_COMPL_STATUS_INVALID_WINDOW,
- I40IW_COMPL_STATUS_INVALID_TOTAL_LENGTH
-};
-
-enum i40iw_completion_notify {
- IW_CQ_COMPL_EVENT = 0,
- IW_CQ_COMPL_SOLICITED = 1
-};
-
-struct i40iw_post_send {
- i40iw_sgl sg_list;
- u32 num_sges;
-};
-
-struct i40iw_post_inline_send {
- void *data;
- u32 len;
-};
-
-struct i40iw_rdma_write {
- i40iw_sgl lo_sg_list;
- u32 num_lo_sges;
- struct i40iw_sge rem_addr;
-};
-
-struct i40iw_inline_rdma_write {
- void *data;
- u32 len;
- struct i40iw_sge rem_addr;
-};
-
-struct i40iw_rdma_read {
- struct i40iw_sge lo_addr;
- struct i40iw_sge rem_addr;
-};
-
-struct i40iw_bind_window {
- i40iw_stag mr_stag;
- u64 bind_length;
- void *va;
- enum i40iw_addressing_type addressing_type;
- bool enable_reads;
- bool enable_writes;
- i40iw_stag mw_stag;
-};
-
-struct i40iw_inv_local_stag {
- i40iw_stag target_stag;
-};
-
-struct i40iw_post_sq_info {
- u64 wr_id;
- u8 op_type;
- bool signaled;
- bool read_fence;
- bool local_fence;
- bool inline_data;
- bool defer_flag;
- union {
- struct i40iw_post_send send;
- struct i40iw_rdma_write rdma_write;
- struct i40iw_rdma_read rdma_read;
- struct i40iw_rdma_read rdma_read_inv;
- struct i40iw_bind_window bind_window;
- struct i40iw_inv_local_stag inv_local_stag;
- struct i40iw_inline_rdma_write inline_rdma_write;
- struct i40iw_post_inline_send inline_send;
- } op;
-};
-
-struct i40iw_post_rq_info {
- u64 wr_id;
- i40iw_sgl sg_list;
- u32 num_sges;
-};
-
-struct i40iw_cq_poll_info {
- u64 wr_id;
- i40iw_qp_handle qp_handle;
- u32 bytes_xfered;
- u32 tcp_seq_num;
- u32 qp_id;
- i40iw_stag inv_stag;
- enum i40iw_completion_status comp_status;
- u16 major_err;
- u16 minor_err;
- u8 op_type;
- bool stag_invalid_set;
- bool push_dropped;
- bool error;
- bool is_srq;
- bool solicited_event;
-};
-
-struct i40iw_qp_uk_ops {
- void (*iw_qp_post_wr)(struct i40iw_qp_uk *);
- void (*iw_qp_ring_push_db)(struct i40iw_qp_uk *, u32);
- enum i40iw_status_code (*iw_rdma_write)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, bool);
- enum i40iw_status_code (*iw_rdma_read)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, bool, bool);
- enum i40iw_status_code (*iw_send)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, u32, bool);
- enum i40iw_status_code (*iw_inline_rdma_write)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, bool);
- enum i40iw_status_code (*iw_inline_send)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, u32, bool);
- enum i40iw_status_code (*iw_stag_local_invalidate)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, bool);
- enum i40iw_status_code (*iw_mw_bind)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, bool);
- enum i40iw_status_code (*iw_post_receive)(struct i40iw_qp_uk *,
- struct i40iw_post_rq_info *);
- enum i40iw_status_code (*iw_post_nop)(struct i40iw_qp_uk *, u64, bool, bool);
-};
-
-struct i40iw_cq_ops {
- void (*iw_cq_request_notification)(struct i40iw_cq_uk *,
- enum i40iw_completion_notify);
- enum i40iw_status_code (*iw_cq_poll_completion)(struct i40iw_cq_uk *,
- struct i40iw_cq_poll_info *);
- enum i40iw_status_code (*iw_cq_post_entries)(struct i40iw_cq_uk *, u8 count);
- void (*iw_cq_clean)(void *, struct i40iw_cq_uk *);
-};
-
-struct i40iw_dev_uk;
-
-struct i40iw_device_uk_ops {
- enum i40iw_status_code (*iwarp_cq_uk_init)(struct i40iw_cq_uk *,
- struct i40iw_cq_uk_init_info *);
- enum i40iw_status_code (*iwarp_qp_uk_init)(struct i40iw_qp_uk *,
- struct i40iw_qp_uk_init_info *);
-};
-
-struct i40iw_dev_uk {
- struct i40iw_device_uk_ops ops_uk;
-};
-
-struct i40iw_sq_uk_wr_trk_info {
- u64 wrid;
- u32 wr_len;
- u8 wqe_size;
- u8 reserved[3];
-};
-
-struct i40iw_qp_quanta {
- u64 elem[I40IW_WQE_SIZE];
-};
-
-struct i40iw_qp_uk {
- struct i40iw_qp_quanta *sq_base;
- struct i40iw_qp_quanta *rq_base;
- u32 __iomem *wqe_alloc_reg;
- struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
- u64 *rq_wrid_array;
- u64 *shadow_area;
- u32 *push_db;
- u64 *push_wqe;
- struct i40iw_ring sq_ring;
- struct i40iw_ring rq_ring;
- struct i40iw_ring initial_ring;
- u32 qp_id;
- u32 sq_size;
- u32 rq_size;
- u32 max_sq_frag_cnt;
- u32 max_rq_frag_cnt;
- struct i40iw_qp_uk_ops ops;
- bool use_srq;
- u8 swqe_polarity;
- u8 swqe_polarity_deferred;
- u8 rwqe_polarity;
- u8 rq_wqe_size;
- u8 rq_wqe_size_multiplier;
- bool first_sq_wq;
- bool deferred_flag;
-};
-
-struct i40iw_cq_uk {
- struct i40iw_cqe *cq_base;
- u32 __iomem *cqe_alloc_reg;
- u64 *shadow_area;
- u32 cq_id;
- u32 cq_size;
- struct i40iw_ring cq_ring;
- u8 polarity;
- bool avoid_mem_cflct;
-
- struct i40iw_cq_ops ops;
-};
-
-struct i40iw_qp_uk_init_info {
- struct i40iw_qp_quanta *sq;
- struct i40iw_qp_quanta *rq;
- u32 __iomem *wqe_alloc_reg;
- u64 *shadow_area;
- struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
- u64 *rq_wrid_array;
- u32 *push_db;
- u64 *push_wqe;
- u32 qp_id;
- u32 sq_size;
- u32 rq_size;
- u32 max_sq_frag_cnt;
- u32 max_rq_frag_cnt;
- u32 max_inline_data;
- int abi_ver;
-};
-
-struct i40iw_cq_uk_init_info {
- u32 __iomem *cqe_alloc_reg;
- struct i40iw_cqe *cq_base;
- u64 *shadow_area;
- u32 cq_size;
- u32 cq_id;
- bool avoid_mem_cflct;
-};
-
-void i40iw_device_init_uk(struct i40iw_dev_uk *dev);
-
-void i40iw_qp_post_wr(struct i40iw_qp_uk *qp);
-u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx,
- u8 wqe_size,
- u32 total_size,
- u64 wr_id
- );
-u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx);
-u64 *i40iw_qp_get_next_srq_wqe(struct i40iw_srq_uk *srq, u32 *wqe_idx);
-
-enum i40iw_status_code i40iw_cq_uk_init(struct i40iw_cq_uk *cq,
- struct i40iw_cq_uk_init_info *info);
-enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp,
- struct i40iw_qp_uk_init_info *info);
-
-void i40iw_clean_cq(void *queue, struct i40iw_cq_uk *cq);
-enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, u64 wr_id,
- bool signaled, bool post_sq);
-enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size);
-enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size);
-enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size,
- u8 *wqe_size);
-void i40iw_get_wqe_shift(u32 sge, u32 inline_data, u8 *shift);
-enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth);
-enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth);
-#endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c
deleted file mode 100644
index 016524683e17..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_utils.c
+++ /dev/null
@@ -1,1557 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/ethtool.h>
-#include <linux/mii.h>
-#include <linux/if_vlan.h>
-#include <linux/crc32.h>
-#include <linux/in.h>
-#include <linux/ip.h>
-#include <linux/tcp.h>
-#include <linux/init.h>
-#include <linux/io.h>
-#include <asm/irq.h>
-#include <asm/byteorder.h>
-#include <net/netevent.h>
-#include <net/neighbour.h>
-#include "i40iw.h"
-
-/**
- * i40iw_arp_table - manage arp table
- * @iwdev: iwarp device
- * @ip_addr: ip address for device
- * @mac_addr: mac address ptr
- * @action: modify, delete or add
- */
-int i40iw_arp_table(struct i40iw_device *iwdev,
- u32 *ip_addr,
- bool ipv4,
- u8 *mac_addr,
- u32 action)
-{
- int arp_index;
- int err;
- u32 ip[4];
-
- if (ipv4) {
- memset(ip, 0, sizeof(ip));
- ip[0] = *ip_addr;
- } else {
- memcpy(ip, ip_addr, sizeof(ip));
- }
-
- for (arp_index = 0; (u32)arp_index < iwdev->arp_table_size; arp_index++)
- if (memcmp(iwdev->arp_table[arp_index].ip_addr, ip, sizeof(ip)) == 0)
- break;
- switch (action) {
- case I40IW_ARP_ADD:
- if (arp_index != iwdev->arp_table_size)
- return -1;
-
- arp_index = 0;
- err = i40iw_alloc_resource(iwdev, iwdev->allocated_arps,
- iwdev->arp_table_size,
- (u32 *)&arp_index,
- &iwdev->next_arp_index);
-
- if (err)
- return err;
-
- memcpy(iwdev->arp_table[arp_index].ip_addr, ip, sizeof(ip));
- ether_addr_copy(iwdev->arp_table[arp_index].mac_addr, mac_addr);
- break;
- case I40IW_ARP_RESOLVE:
- if (arp_index == iwdev->arp_table_size)
- return -1;
- break;
- case I40IW_ARP_DELETE:
- if (arp_index == iwdev->arp_table_size)
- return -1;
- memset(iwdev->arp_table[arp_index].ip_addr, 0,
- sizeof(iwdev->arp_table[arp_index].ip_addr));
- eth_zero_addr(iwdev->arp_table[arp_index].mac_addr);
- i40iw_free_resource(iwdev, iwdev->allocated_arps, arp_index);
- break;
- default:
- return -1;
- }
- return arp_index;
-}
-
-/**
- * i40iw_wr32 - write 32 bits to hw register
- * @hw: hardware information including registers
- * @reg: register offset
- * @value: vvalue to write to register
- */
-inline void i40iw_wr32(struct i40iw_hw *hw, u32 reg, u32 value)
-{
- writel(value, hw->hw_addr + reg);
-}
-
-/**
- * i40iw_rd32 - read a 32 bit hw register
- * @hw: hardware information including registers
- * @reg: register offset
- *
- * Return value of register content
- */
-inline u32 i40iw_rd32(struct i40iw_hw *hw, u32 reg)
-{
- return readl(hw->hw_addr + reg);
-}
-
-/**
- * i40iw_inetaddr_event - system notifier for ipv4 addr events
- * @notfier: not used
- * @event: event for notifier
- * @ptr: if address
- */
-int i40iw_inetaddr_event(struct notifier_block *notifier,
- unsigned long event,
- void *ptr)
-{
- struct in_ifaddr *ifa = ptr;
- struct net_device *event_netdev = ifa->ifa_dev->dev;
- struct net_device *netdev;
- struct net_device *upper_dev;
- struct i40iw_device *iwdev;
- struct i40iw_handler *hdl;
- u32 local_ipaddr;
- u32 action = I40IW_ARP_ADD;
-
- hdl = i40iw_find_netdev(event_netdev);
- if (!hdl)
- return NOTIFY_DONE;
-
- iwdev = &hdl->device;
- if (iwdev->init_state < IP_ADDR_REGISTERED || iwdev->closing)
- return NOTIFY_DONE;
-
- netdev = iwdev->ldev->netdev;
- upper_dev = netdev_master_upper_dev_get(netdev);
- if (netdev != event_netdev)
- return NOTIFY_DONE;
-
- if (upper_dev) {
- struct in_device *in;
-
- rcu_read_lock();
- in = __in_dev_get_rcu(upper_dev);
-
- local_ipaddr = 0;
- if (in) {
- struct in_ifaddr *ifa;
-
- ifa = rcu_dereference(in->ifa_list);
- if (ifa)
- local_ipaddr = ntohl(ifa->ifa_address);
- }
-
- rcu_read_unlock();
- } else {
- local_ipaddr = ntohl(ifa->ifa_address);
- }
- switch (event) {
- case NETDEV_DOWN:
- action = I40IW_ARP_DELETE;
- /* Fall through */
- case NETDEV_UP:
- /* Fall through */
- case NETDEV_CHANGEADDR:
-
- /* Just skip if no need to handle ARP cache */
- if (!local_ipaddr)
- break;
-
- i40iw_manage_arp_cache(iwdev,
- netdev->dev_addr,
- &local_ipaddr,
- true,
- action);
- i40iw_if_notify(iwdev, netdev, &local_ipaddr, true,
- (action == I40IW_ARP_ADD) ? true : false);
- break;
- default:
- break;
- }
- return NOTIFY_DONE;
-}
-
-/**
- * i40iw_inet6addr_event - system notifier for ipv6 addr events
- * @notfier: not used
- * @event: event for notifier
- * @ptr: if address
- */
-int i40iw_inet6addr_event(struct notifier_block *notifier,
- unsigned long event,
- void *ptr)
-{
- struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)ptr;
- struct net_device *event_netdev = ifa->idev->dev;
- struct net_device *netdev;
- struct i40iw_device *iwdev;
- struct i40iw_handler *hdl;
- u32 local_ipaddr6[4];
- u32 action = I40IW_ARP_ADD;
-
- hdl = i40iw_find_netdev(event_netdev);
- if (!hdl)
- return NOTIFY_DONE;
-
- iwdev = &hdl->device;
- if (iwdev->init_state < IP_ADDR_REGISTERED || iwdev->closing)
- return NOTIFY_DONE;
-
- netdev = iwdev->ldev->netdev;
- if (netdev != event_netdev)
- return NOTIFY_DONE;
-
- i40iw_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
- switch (event) {
- case NETDEV_DOWN:
- action = I40IW_ARP_DELETE;
- /* Fall through */
- case NETDEV_UP:
- /* Fall through */
- case NETDEV_CHANGEADDR:
- i40iw_manage_arp_cache(iwdev,
- netdev->dev_addr,
- local_ipaddr6,
- false,
- action);
- i40iw_if_notify(iwdev, netdev, local_ipaddr6, false,
- (action == I40IW_ARP_ADD) ? true : false);
- break;
- default:
- break;
- }
- return NOTIFY_DONE;
-}
-
-/**
- * i40iw_net_event - system notifier for netevents
- * @notfier: not used
- * @event: event for notifier
- * @ptr: neighbor
- */
-int i40iw_net_event(struct notifier_block *notifier, unsigned long event, void *ptr)
-{
- struct neighbour *neigh = ptr;
- struct i40iw_device *iwdev;
- struct i40iw_handler *iwhdl;
- __be32 *p;
- u32 local_ipaddr[4];
-
- switch (event) {
- case NETEVENT_NEIGH_UPDATE:
- iwhdl = i40iw_find_netdev((struct net_device *)neigh->dev);
- if (!iwhdl)
- return NOTIFY_DONE;
- iwdev = &iwhdl->device;
- if (iwdev->init_state < IP_ADDR_REGISTERED || iwdev->closing)
- return NOTIFY_DONE;
- p = (__be32 *)neigh->primary_key;
- i40iw_copy_ip_ntohl(local_ipaddr, p);
- if (neigh->nud_state & NUD_VALID) {
- i40iw_manage_arp_cache(iwdev,
- neigh->ha,
- local_ipaddr,
- false,
- I40IW_ARP_ADD);
-
- } else {
- i40iw_manage_arp_cache(iwdev,
- neigh->ha,
- local_ipaddr,
- false,
- I40IW_ARP_DELETE);
- }
- break;
- default:
- break;
- }
- return NOTIFY_DONE;
-}
-
-/**
- * i40iw_netdevice_event - system notifier for netdev events
- * @notfier: not used
- * @event: event for notifier
- * @ptr: netdev
- */
-int i40iw_netdevice_event(struct notifier_block *notifier,
- unsigned long event,
- void *ptr)
-{
- struct net_device *event_netdev;
- struct net_device *netdev;
- struct i40iw_device *iwdev;
- struct i40iw_handler *hdl;
-
- event_netdev = netdev_notifier_info_to_dev(ptr);
-
- hdl = i40iw_find_netdev(event_netdev);
- if (!hdl)
- return NOTIFY_DONE;
-
- iwdev = &hdl->device;
- if (iwdev->init_state < RDMA_DEV_REGISTERED || iwdev->closing)
- return NOTIFY_DONE;
-
- netdev = iwdev->ldev->netdev;
- if (netdev != event_netdev)
- return NOTIFY_DONE;
-
- iwdev->iw_status = 1;
-
- switch (event) {
- case NETDEV_DOWN:
- iwdev->iw_status = 0;
- /* Fall through */
- case NETDEV_UP:
- i40iw_port_ibevent(iwdev);
- break;
- default:
- break;
- }
- return NOTIFY_DONE;
-}
-
-/**
- * i40iw_get_cqp_request - get cqp struct
- * @cqp: device cqp ptr
- * @wait: cqp to be used in wait mode
- */
-struct i40iw_cqp_request *i40iw_get_cqp_request(struct i40iw_cqp *cqp, bool wait)
-{
- struct i40iw_cqp_request *cqp_request = NULL;
- unsigned long flags;
-
- spin_lock_irqsave(&cqp->req_lock, flags);
- if (!list_empty(&cqp->cqp_avail_reqs)) {
- cqp_request = list_entry(cqp->cqp_avail_reqs.next,
- struct i40iw_cqp_request, list);
- list_del_init(&cqp_request->list);
- }
- spin_unlock_irqrestore(&cqp->req_lock, flags);
- if (!cqp_request) {
- cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
- if (cqp_request) {
- cqp_request->dynamic = true;
- INIT_LIST_HEAD(&cqp_request->list);
- init_waitqueue_head(&cqp_request->waitq);
- }
- }
- if (!cqp_request) {
- i40iw_pr_err("CQP Request Fail: No Memory");
- return NULL;
- }
-
- if (wait) {
- atomic_set(&cqp_request->refcount, 2);
- cqp_request->waiting = true;
- } else {
- atomic_set(&cqp_request->refcount, 1);
- }
- return cqp_request;
-}
-
-/**
- * i40iw_free_cqp_request - free cqp request
- * @cqp: cqp ptr
- * @cqp_request: to be put back in cqp list
- */
-void i40iw_free_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request)
-{
- struct i40iw_device *iwdev = container_of(cqp, struct i40iw_device, cqp);
- unsigned long flags;
-
- if (cqp_request->dynamic) {
- kfree(cqp_request);
- } else {
- cqp_request->request_done = false;
- cqp_request->callback_fcn = NULL;
- cqp_request->waiting = false;
-
- spin_lock_irqsave(&cqp->req_lock, flags);
- list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
- spin_unlock_irqrestore(&cqp->req_lock, flags);
- }
- wake_up(&iwdev->close_wq);
-}
-
-/**
- * i40iw_put_cqp_request - dec ref count and free if 0
- * @cqp: cqp ptr
- * @cqp_request: to be put back in cqp list
- */
-void i40iw_put_cqp_request(struct i40iw_cqp *cqp,
- struct i40iw_cqp_request *cqp_request)
-{
- if (atomic_dec_and_test(&cqp_request->refcount))
- i40iw_free_cqp_request(cqp, cqp_request);
-}
-
-/**
- * i40iw_free_pending_cqp_request -free pending cqp request objs
- * @cqp: cqp ptr
- * @cqp_request: to be put back in cqp list
- */
-static void i40iw_free_pending_cqp_request(struct i40iw_cqp *cqp,
- struct i40iw_cqp_request *cqp_request)
-{
- struct i40iw_device *iwdev = container_of(cqp, struct i40iw_device, cqp);
-
- if (cqp_request->waiting) {
- cqp_request->compl_info.error = true;
- cqp_request->request_done = true;
- wake_up(&cqp_request->waitq);
- }
- i40iw_put_cqp_request(cqp, cqp_request);
- wait_event_timeout(iwdev->close_wq,
- !atomic_read(&cqp_request->refcount),
- 1000);
-}
-
-/**
- * i40iw_cleanup_pending_cqp_op - clean-up cqp with no completions
- * @iwdev: iwarp device
- */
-void i40iw_cleanup_pending_cqp_op(struct i40iw_device *iwdev)
-{
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_cqp *cqp = &iwdev->cqp;
- struct i40iw_cqp_request *cqp_request = NULL;
- struct cqp_commands_info *pcmdinfo = NULL;
- u32 i, pending_work, wqe_idx;
-
- pending_work = I40IW_RING_WORK_AVAILABLE(cqp->sc_cqp.sq_ring);
- wqe_idx = I40IW_RING_GETCURRENT_TAIL(cqp->sc_cqp.sq_ring);
- for (i = 0; i < pending_work; i++) {
- cqp_request = (struct i40iw_cqp_request *)(unsigned long)cqp->scratch_array[wqe_idx];
- if (cqp_request)
- i40iw_free_pending_cqp_request(cqp, cqp_request);
- wqe_idx = (wqe_idx + 1) % I40IW_RING_GETSIZE(cqp->sc_cqp.sq_ring);
- }
-
- while (!list_empty(&dev->cqp_cmd_head)) {
- pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&dev->cqp_cmd_head);
- cqp_request = container_of(pcmdinfo, struct i40iw_cqp_request, info);
- if (cqp_request)
- i40iw_free_pending_cqp_request(cqp, cqp_request);
- }
-}
-
-/**
- * i40iw_free_qp - callback after destroy cqp completes
- * @cqp_request: cqp request for destroy qp
- * @num: not used
- */
-static void i40iw_free_qp(struct i40iw_cqp_request *cqp_request, u32 num)
-{
- struct i40iw_sc_qp *qp = (struct i40iw_sc_qp *)cqp_request->param;
- struct i40iw_qp *iwqp = (struct i40iw_qp *)qp->back_qp;
- struct i40iw_device *iwdev;
- u32 qp_num = iwqp->ibqp.qp_num;
-
- iwdev = iwqp->iwdev;
-
- i40iw_rem_pdusecount(iwqp->iwpd, iwdev);
- i40iw_free_qp_resources(iwdev, iwqp, qp_num);
- i40iw_rem_devusecount(iwdev);
-}
-
-/**
- * i40iw_wait_event - wait for completion
- * @iwdev: iwarp device
- * @cqp_request: cqp request to wait
- */
-static int i40iw_wait_event(struct i40iw_device *iwdev,
- struct i40iw_cqp_request *cqp_request)
-{
- struct cqp_commands_info *info = &cqp_request->info;
- struct i40iw_cqp *iwcqp = &iwdev->cqp;
- struct i40iw_cqp_timeout cqp_timeout;
- bool cqp_error = false;
- int err_code = 0;
- memset(&cqp_timeout, 0, sizeof(cqp_timeout));
- cqp_timeout.compl_cqp_cmds = iwdev->sc_dev.cqp_cmd_stats[OP_COMPLETED_COMMANDS];
- do {
- if (wait_event_timeout(cqp_request->waitq,
- cqp_request->request_done, CQP_COMPL_WAIT_TIME))
- break;
-
- i40iw_check_cqp_progress(&cqp_timeout, &iwdev->sc_dev);
-
- if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
- continue;
-
- i40iw_pr_err("error cqp command 0x%x timed out", info->cqp_cmd);
- err_code = -ETIME;
- if (!iwdev->reset) {
- iwdev->reset = true;
- i40iw_request_reset(iwdev);
- }
- goto done;
- } while (1);
- cqp_error = cqp_request->compl_info.error;
- if (cqp_error) {
- i40iw_pr_err("error cqp command 0x%x completion maj = 0x%x min=0x%x\n",
- info->cqp_cmd, cqp_request->compl_info.maj_err_code,
- cqp_request->compl_info.min_err_code);
- err_code = -EPROTO;
- goto done;
- }
-done:
- i40iw_put_cqp_request(iwcqp, cqp_request);
- return err_code;
-}
-
-/**
- * i40iw_handle_cqp_op - process cqp command
- * @iwdev: iwarp device
- * @cqp_request: cqp request to process
- */
-enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev,
- struct i40iw_cqp_request
- *cqp_request)
-{
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- enum i40iw_status_code status;
- struct cqp_commands_info *info = &cqp_request->info;
- int err_code = 0;
-
- if (iwdev->reset) {
- i40iw_free_cqp_request(&iwdev->cqp, cqp_request);
- return I40IW_ERR_CQP_COMPL_ERROR;
- }
-
- status = i40iw_process_cqp_cmd(dev, info);
- if (status) {
- i40iw_pr_err("error cqp command 0x%x failed\n", info->cqp_cmd);
- i40iw_free_cqp_request(&iwdev->cqp, cqp_request);
- return status;
- }
- if (cqp_request->waiting)
- err_code = i40iw_wait_event(iwdev, cqp_request);
- if (err_code)
- status = I40IW_ERR_CQP_COMPL_ERROR;
- return status;
-}
-
-/**
- * i40iw_add_devusecount - add dev refcount
- * @iwdev: dev for refcount
- */
-void i40iw_add_devusecount(struct i40iw_device *iwdev)
-{
- atomic64_inc(&iwdev->use_count);
-}
-
-/**
- * i40iw_rem_devusecount - decrement refcount for dev
- * @iwdev: device
- */
-void i40iw_rem_devusecount(struct i40iw_device *iwdev)
-{
- if (!atomic64_dec_and_test(&iwdev->use_count))
- return;
- wake_up(&iwdev->close_wq);
-}
-
-/**
- * i40iw_add_pdusecount - add pd refcount
- * @iwpd: pd for refcount
- */
-void i40iw_add_pdusecount(struct i40iw_pd *iwpd)
-{
- atomic_inc(&iwpd->usecount);
-}
-
-/**
- * i40iw_rem_pdusecount - decrement refcount for pd and free if 0
- * @iwpd: pd for refcount
- * @iwdev: iwarp device
- */
-void i40iw_rem_pdusecount(struct i40iw_pd *iwpd, struct i40iw_device *iwdev)
-{
- if (!atomic_dec_and_test(&iwpd->usecount))
- return;
- i40iw_free_resource(iwdev, iwdev->allocated_pds, iwpd->sc_pd.pd_id);
-}
-
-/**
- * i40iw_add_ref - add refcount for qp
- * @ibqp: iqarp qp
- */
-void i40iw_add_ref(struct ib_qp *ibqp)
-{
- struct i40iw_qp *iwqp = (struct i40iw_qp *)ibqp;
-
- atomic_inc(&iwqp->refcount);
-}
-
-/**
- * i40iw_rem_ref - rem refcount for qp and free if 0
- * @ibqp: iqarp qp
- */
-void i40iw_rem_ref(struct ib_qp *ibqp)
-{
- struct i40iw_qp *iwqp;
- enum i40iw_status_code status;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- struct i40iw_device *iwdev;
- u32 qp_num;
- unsigned long flags;
-
- iwqp = to_iwqp(ibqp);
- iwdev = iwqp->iwdev;
- spin_lock_irqsave(&iwdev->qptable_lock, flags);
- if (!atomic_dec_and_test(&iwqp->refcount)) {
- spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
- return;
- }
-
- qp_num = iwqp->ibqp.qp_num;
- iwdev->qp_table[qp_num] = NULL;
- spin_unlock_irqrestore(&iwdev->qptable_lock, flags);
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false);
- if (!cqp_request)
- return;
-
- cqp_request->callback_fcn = i40iw_free_qp;
- cqp_request->param = (void *)&iwqp->sc_qp;
- cqp_info = &cqp_request->info;
- cqp_info->cqp_cmd = OP_QP_DESTROY;
- cqp_info->post_sq = 1;
- cqp_info->in.u.qp_destroy.qp = &iwqp->sc_qp;
- cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
- cqp_info->in.u.qp_destroy.remove_hash_idx = true;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (!status)
- return;
-
- i40iw_rem_pdusecount(iwqp->iwpd, iwdev);
- i40iw_free_qp_resources(iwdev, iwqp, qp_num);
- i40iw_rem_devusecount(iwdev);
-}
-
-/**
- * i40iw_get_qp - get qp address
- * @device: iwarp device
- * @qpn: qp number
- */
-struct ib_qp *i40iw_get_qp(struct ib_device *device, int qpn)
-{
- struct i40iw_device *iwdev = to_iwdev(device);
-
- if ((qpn < IW_FIRST_QPN) || (qpn >= iwdev->max_qp))
- return NULL;
-
- return &iwdev->qp_table[qpn]->ibqp;
-}
-
-/**
- * i40iw_debug_buf - print debug msg and buffer is mask set
- * @dev: hardware control device structure
- * @mask: mask to compare if to print debug buffer
- * @buf: points buffer addr
- * @size: saize of buffer to print
- */
-void i40iw_debug_buf(struct i40iw_sc_dev *dev,
- enum i40iw_debug_flag mask,
- char *desc,
- u64 *buf,
- u32 size)
-{
- u32 i;
-
- if (!(dev->debug_mask & mask))
- return;
- i40iw_debug(dev, mask, "%s\n", desc);
- i40iw_debug(dev, mask, "starting address virt=%p phy=%llxh\n", buf,
- (unsigned long long)virt_to_phys(buf));
-
- for (i = 0; i < size; i += 8)
- i40iw_debug(dev, mask, "index %03d val: %016llx\n", i, buf[i / 8]);
-}
-
-/**
- * i40iw_get_hw_addr - return hw addr
- * @par: points to shared dev
- */
-u8 __iomem *i40iw_get_hw_addr(void *par)
-{
- struct i40iw_sc_dev *dev = (struct i40iw_sc_dev *)par;
-
- return dev->hw->hw_addr;
-}
-
-/**
- * i40iw_remove_head - return head entry and remove from list
- * @list: list for entry
- */
-void *i40iw_remove_head(struct list_head *list)
-{
- struct list_head *entry;
-
- if (list_empty(list))
- return NULL;
-
- entry = (void *)list->next;
- list_del(entry);
- return (void *)entry;
-}
-
-/**
- * i40iw_allocate_dma_mem - Memory alloc helper fn
- * @hw: pointer to the HW structure
- * @mem: ptr to mem struct to fill out
- * @size: size of memory requested
- * @alignment: what to align the allocation to
- */
-enum i40iw_status_code i40iw_allocate_dma_mem(struct i40iw_hw *hw,
- struct i40iw_dma_mem *mem,
- u64 size,
- u32 alignment)
-{
- struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context;
-
- if (!mem)
- return I40IW_ERR_PARAM;
- mem->size = ALIGN(size, alignment);
- mem->va = dma_alloc_coherent(&pcidev->dev, mem->size,
- (dma_addr_t *)&mem->pa, GFP_KERNEL);
- if (!mem->va)
- return I40IW_ERR_NO_MEMORY;
- return 0;
-}
-
-/**
- * i40iw_free_dma_mem - Memory free helper fn
- * @hw: pointer to the HW structure
- * @mem: ptr to mem struct to free
- */
-void i40iw_free_dma_mem(struct i40iw_hw *hw, struct i40iw_dma_mem *mem)
-{
- struct pci_dev *pcidev = (struct pci_dev *)hw->dev_context;
-
- if (!mem || !mem->va)
- return;
-
- dma_free_coherent(&pcidev->dev, mem->size,
- mem->va, (dma_addr_t)mem->pa);
- mem->va = NULL;
-}
-
-/**
- * i40iw_allocate_virt_mem - virtual memory alloc helper fn
- * @hw: pointer to the HW structure
- * @mem: ptr to mem struct to fill out
- * @size: size of memory requested
- */
-enum i40iw_status_code i40iw_allocate_virt_mem(struct i40iw_hw *hw,
- struct i40iw_virt_mem *mem,
- u32 size)
-{
- if (!mem)
- return I40IW_ERR_PARAM;
-
- mem->size = size;
- mem->va = kzalloc(size, GFP_KERNEL);
-
- if (mem->va)
- return 0;
- else
- return I40IW_ERR_NO_MEMORY;
-}
-
-/**
- * i40iw_free_virt_mem - virtual memory free helper fn
- * @hw: pointer to the HW structure
- * @mem: ptr to mem struct to free
- */
-enum i40iw_status_code i40iw_free_virt_mem(struct i40iw_hw *hw,
- struct i40iw_virt_mem *mem)
-{
- if (!mem)
- return I40IW_ERR_PARAM;
- /*
- * mem->va points to the parent of mem, so both mem and mem->va
- * can not be touched once mem->va is freed
- */
- kfree(mem->va);
- return 0;
-}
-
-/**
- * i40iw_cqp_sds_cmd - create cqp command for sd
- * @dev: hardware control device structure
- * @sd_info: information for sd cqp
- *
- */
-enum i40iw_status_code i40iw_cqp_sds_cmd(struct i40iw_sc_dev *dev,
- struct i40iw_update_sds_info *sdinfo)
-{
- enum i40iw_status_code status;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
- if (!cqp_request)
- return I40IW_ERR_NO_MEMORY;
- cqp_info = &cqp_request->info;
- memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
- sizeof(cqp_info->in.u.update_pe_sds.info));
- cqp_info->cqp_cmd = OP_UPDATE_PE_SDS;
- cqp_info->post_sq = 1;
- cqp_info->in.u.update_pe_sds.dev = dev;
- cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Update SD's fail");
- return status;
-}
-
-/**
- * i40iw_qp_suspend_resume - cqp command for suspend/resume
- * @dev: hardware control device structure
- * @qp: hardware control qp
- * @suspend: flag if suspend or resume
- */
-void i40iw_qp_suspend_resume(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp, bool suspend)
-{
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
- struct i40iw_cqp_request *cqp_request;
- struct i40iw_sc_cqp *cqp = dev->cqp;
- struct cqp_commands_info *cqp_info;
- enum i40iw_status_code status;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false);
- if (!cqp_request)
- return;
-
- cqp_info = &cqp_request->info;
- cqp_info->cqp_cmd = (suspend) ? OP_SUSPEND : OP_RESUME;
- cqp_info->in.u.suspend_resume.cqp = cqp;
- cqp_info->in.u.suspend_resume.qp = qp;
- cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP QP Suspend/Resume fail");
-}
-
-/**
- * i40iw_term_modify_qp - modify qp for term message
- * @qp: hardware control qp
- * @next_state: qp's next state
- * @term: terminate code
- * @term_len: length
- */
-void i40iw_term_modify_qp(struct i40iw_sc_qp *qp, u8 next_state, u8 term, u8 term_len)
-{
- struct i40iw_qp *iwqp;
-
- iwqp = (struct i40iw_qp *)qp->back_qp;
- i40iw_next_iw_state(iwqp, next_state, 0, term, term_len);
-};
-
-/**
- * i40iw_terminate_done - after terminate is completed
- * @qp: hardware control qp
- * @timeout_occurred: indicates if terminate timer expired
- */
-void i40iw_terminate_done(struct i40iw_sc_qp *qp, int timeout_occurred)
-{
- struct i40iw_qp *iwqp;
- u32 next_iwarp_state = I40IW_QP_STATE_ERROR;
- u8 hte = 0;
- bool first_time;
- unsigned long flags;
-
- iwqp = (struct i40iw_qp *)qp->back_qp;
- spin_lock_irqsave(&iwqp->lock, flags);
- if (iwqp->hte_added) {
- iwqp->hte_added = 0;
- hte = 1;
- }
- first_time = !(qp->term_flags & I40IW_TERM_DONE);
- qp->term_flags |= I40IW_TERM_DONE;
- spin_unlock_irqrestore(&iwqp->lock, flags);
- if (first_time) {
- if (!timeout_occurred)
- i40iw_terminate_del_timer(qp);
- else
- next_iwarp_state = I40IW_QP_STATE_CLOSING;
-
- i40iw_next_iw_state(iwqp, next_iwarp_state, hte, 0, 0);
- i40iw_cm_disconn(iwqp);
- }
-}
-
-/**
- * i40iw_terminate_imeout - timeout happened
- * @context: points to iwarp qp
- */
-static void i40iw_terminate_timeout(struct timer_list *t)
-{
- struct i40iw_qp *iwqp = from_timer(iwqp, t, terminate_timer);
- struct i40iw_sc_qp *qp = (struct i40iw_sc_qp *)&iwqp->sc_qp;
-
- i40iw_terminate_done(qp, 1);
- i40iw_rem_ref(&iwqp->ibqp);
-}
-
-/**
- * i40iw_terminate_start_timer - start terminate timeout
- * @qp: hardware control qp
- */
-void i40iw_terminate_start_timer(struct i40iw_sc_qp *qp)
-{
- struct i40iw_qp *iwqp;
-
- iwqp = (struct i40iw_qp *)qp->back_qp;
- i40iw_add_ref(&iwqp->ibqp);
- timer_setup(&iwqp->terminate_timer, i40iw_terminate_timeout, 0);
- iwqp->terminate_timer.expires = jiffies + HZ;
- add_timer(&iwqp->terminate_timer);
-}
-
-/**
- * i40iw_terminate_del_timer - delete terminate timeout
- * @qp: hardware control qp
- */
-void i40iw_terminate_del_timer(struct i40iw_sc_qp *qp)
-{
- struct i40iw_qp *iwqp;
-
- iwqp = (struct i40iw_qp *)qp->back_qp;
- if (del_timer(&iwqp->terminate_timer))
- i40iw_rem_ref(&iwqp->ibqp);
-}
-
-/**
- * i40iw_cqp_generic_worker - generic worker for cqp
- * @work: work pointer
- */
-static void i40iw_cqp_generic_worker(struct work_struct *work)
-{
- struct i40iw_virtchnl_work_info *work_info =
- &((struct virtchnl_work *)work)->work_info;
-
- if (work_info->worker_vf_dev)
- work_info->callback_fcn(work_info->worker_vf_dev);
-}
-
-/**
- * i40iw_cqp_spawn_worker - spawn worket thread
- * @iwdev: device struct pointer
- * @work_info: work request info
- * @iw_vf_idx: virtual function index
- */
-void i40iw_cqp_spawn_worker(struct i40iw_sc_dev *dev,
- struct i40iw_virtchnl_work_info *work_info,
- u32 iw_vf_idx)
-{
- struct virtchnl_work *work;
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
-
- work = &iwdev->virtchnl_w[iw_vf_idx];
- memcpy(&work->work_info, work_info, sizeof(*work_info));
- INIT_WORK(&work->work, i40iw_cqp_generic_worker);
- queue_work(iwdev->virtchnl_wq, &work->work);
-}
-
-/**
- * i40iw_cqp_manage_hmc_fcn_worker -
- * @work: work pointer for hmc info
- */
-static void i40iw_cqp_manage_hmc_fcn_worker(struct work_struct *work)
-{
- struct i40iw_cqp_request *cqp_request =
- ((struct virtchnl_work *)work)->cqp_request;
- struct i40iw_ccq_cqe_info ccq_cqe_info;
- struct i40iw_hmc_fcn_info *hmcfcninfo =
- &cqp_request->info.in.u.manage_hmc_pm.info;
- struct i40iw_device *iwdev =
- (struct i40iw_device *)cqp_request->info.in.u.manage_hmc_pm.dev->back_dev;
-
- ccq_cqe_info.cqp = NULL;
- ccq_cqe_info.maj_err_code = cqp_request->compl_info.maj_err_code;
- ccq_cqe_info.min_err_code = cqp_request->compl_info.min_err_code;
- ccq_cqe_info.op_code = cqp_request->compl_info.op_code;
- ccq_cqe_info.op_ret_val = cqp_request->compl_info.op_ret_val;
- ccq_cqe_info.scratch = 0;
- ccq_cqe_info.error = cqp_request->compl_info.error;
- hmcfcninfo->callback_fcn(cqp_request->info.in.u.manage_hmc_pm.dev,
- hmcfcninfo->cqp_callback_param, &ccq_cqe_info);
- i40iw_put_cqp_request(&iwdev->cqp, cqp_request);
-}
-
-/**
- * i40iw_cqp_manage_hmc_fcn_callback - called function after cqp completion
- * @cqp_request: cqp request info struct for hmc fun
- * @unused: unused param of callback
- */
-static void i40iw_cqp_manage_hmc_fcn_callback(struct i40iw_cqp_request *cqp_request,
- u32 unused)
-{
- struct virtchnl_work *work;
- struct i40iw_hmc_fcn_info *hmcfcninfo =
- &cqp_request->info.in.u.manage_hmc_pm.info;
- struct i40iw_device *iwdev =
- (struct i40iw_device *)cqp_request->info.in.u.manage_hmc_pm.dev->
- back_dev;
-
- if (hmcfcninfo && hmcfcninfo->callback_fcn) {
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_HMC, "%s1\n", __func__);
- atomic_inc(&cqp_request->refcount);
- work = &iwdev->virtchnl_w[hmcfcninfo->iw_vf_idx];
- work->cqp_request = cqp_request;
- INIT_WORK(&work->work, i40iw_cqp_manage_hmc_fcn_worker);
- queue_work(iwdev->virtchnl_wq, &work->work);
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_HMC, "%s2\n", __func__);
- } else {
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_HMC, "%s: Something wrong\n", __func__);
- }
-}
-
-/**
- * i40iw_cqp_manage_hmc_fcn_cmd - issue cqp command to manage hmc
- * @dev: hardware control device structure
- * @hmcfcninfo: info for hmc
- */
-enum i40iw_status_code i40iw_cqp_manage_hmc_fcn_cmd(struct i40iw_sc_dev *dev,
- struct i40iw_hmc_fcn_info *hmcfcninfo)
-{
- enum i40iw_status_code status;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
-
- i40iw_debug(&iwdev->sc_dev, I40IW_DEBUG_HMC, "%s\n", __func__);
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false);
- if (!cqp_request)
- return I40IW_ERR_NO_MEMORY;
- cqp_info = &cqp_request->info;
- cqp_request->callback_fcn = i40iw_cqp_manage_hmc_fcn_callback;
- cqp_request->param = hmcfcninfo;
- memcpy(&cqp_info->in.u.manage_hmc_pm.info, hmcfcninfo,
- sizeof(*hmcfcninfo));
- cqp_info->in.u.manage_hmc_pm.dev = dev;
- cqp_info->cqp_cmd = OP_MANAGE_HMC_PM_FUNC_TABLE;
- cqp_info->post_sq = 1;
- cqp_info->in.u.manage_hmc_pm.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Manage HMC fail");
- return status;
-}
-
-/**
- * i40iw_cqp_query_fpm_values_cmd - send cqp command for fpm
- * @iwdev: function device struct
- * @values_mem: buffer for fpm
- * @hmc_fn_id: function id for fpm
- */
-enum i40iw_status_code i40iw_cqp_query_fpm_values_cmd(struct i40iw_sc_dev *dev,
- struct i40iw_dma_mem *values_mem,
- u8 hmc_fn_id)
-{
- enum i40iw_status_code status;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
- if (!cqp_request)
- return I40IW_ERR_NO_MEMORY;
- cqp_info = &cqp_request->info;
- cqp_request->param = NULL;
- cqp_info->in.u.query_fpm_values.cqp = dev->cqp;
- cqp_info->in.u.query_fpm_values.fpm_values_pa = values_mem->pa;
- cqp_info->in.u.query_fpm_values.fpm_values_va = values_mem->va;
- cqp_info->in.u.query_fpm_values.hmc_fn_id = hmc_fn_id;
- cqp_info->cqp_cmd = OP_QUERY_FPM_VALUES;
- cqp_info->post_sq = 1;
- cqp_info->in.u.query_fpm_values.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Query FPM fail");
- return status;
-}
-
-/**
- * i40iw_cqp_commit_fpm_values_cmd - commit fpm values in hw
- * @dev: hardware control device structure
- * @values_mem: buffer with fpm values
- * @hmc_fn_id: function id for fpm
- */
-enum i40iw_status_code i40iw_cqp_commit_fpm_values_cmd(struct i40iw_sc_dev *dev,
- struct i40iw_dma_mem *values_mem,
- u8 hmc_fn_id)
-{
- enum i40iw_status_code status;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
- if (!cqp_request)
- return I40IW_ERR_NO_MEMORY;
- cqp_info = &cqp_request->info;
- cqp_request->param = NULL;
- cqp_info->in.u.commit_fpm_values.cqp = dev->cqp;
- cqp_info->in.u.commit_fpm_values.fpm_values_pa = values_mem->pa;
- cqp_info->in.u.commit_fpm_values.fpm_values_va = values_mem->va;
- cqp_info->in.u.commit_fpm_values.hmc_fn_id = hmc_fn_id;
- cqp_info->cqp_cmd = OP_COMMIT_FPM_VALUES;
- cqp_info->post_sq = 1;
- cqp_info->in.u.commit_fpm_values.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Commit FPM fail");
- return status;
-}
-
-/**
- * i40iw_vf_wait_vchnl_resp - wait for channel msg
- * @iwdev: function's device struct
- */
-enum i40iw_status_code i40iw_vf_wait_vchnl_resp(struct i40iw_sc_dev *dev)
-{
- struct i40iw_device *iwdev = dev->back_dev;
- int timeout_ret;
-
- i40iw_debug(dev, I40IW_DEBUG_VIRT, "%s[%u] dev %p, iwdev %p\n",
- __func__, __LINE__, dev, iwdev);
-
- atomic_set(&iwdev->vchnl_msgs, 2);
- timeout_ret = wait_event_timeout(iwdev->vchnl_waitq,
- (atomic_read(&iwdev->vchnl_msgs) == 1),
- I40IW_VCHNL_EVENT_TIMEOUT);
- atomic_dec(&iwdev->vchnl_msgs);
- if (!timeout_ret) {
- i40iw_pr_err("virt channel completion timeout = 0x%x\n", timeout_ret);
- atomic_set(&iwdev->vchnl_msgs, 0);
- dev->vchnl_up = false;
- return I40IW_ERR_TIMEOUT;
- }
- wake_up(&dev->vf_reqs);
- return 0;
-}
-
-/**
- * i40iw_cqp_cq_create_cmd - create a cq for the cqp
- * @dev: device pointer
- * @cq: pointer to created cq
- */
-enum i40iw_status_code i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev *dev,
- struct i40iw_sc_cq *cq)
-{
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
- struct i40iw_cqp *iwcqp = &iwdev->cqp;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- enum i40iw_status_code status;
-
- cqp_request = i40iw_get_cqp_request(iwcqp, true);
- if (!cqp_request)
- return I40IW_ERR_NO_MEMORY;
-
- cqp_info = &cqp_request->info;
- cqp_info->cqp_cmd = OP_CQ_CREATE;
- cqp_info->post_sq = 1;
- cqp_info->in.u.cq_create.cq = cq;
- cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Create QP fail");
-
- return status;
-}
-
-/**
- * i40iw_cqp_qp_create_cmd - create a qp for the cqp
- * @dev: device pointer
- * @qp: pointer to created qp
- */
-enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev,
- struct i40iw_sc_qp *qp)
-{
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
- struct i40iw_cqp *iwcqp = &iwdev->cqp;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- struct i40iw_create_qp_info *qp_info;
- enum i40iw_status_code status;
-
- cqp_request = i40iw_get_cqp_request(iwcqp, true);
- if (!cqp_request)
- return I40IW_ERR_NO_MEMORY;
-
- cqp_info = &cqp_request->info;
- qp_info = &cqp_request->info.in.u.qp_create.info;
-
- memset(qp_info, 0, sizeof(*qp_info));
-
- qp_info->cq_num_valid = true;
- qp_info->next_iwarp_state = I40IW_QP_STATE_RTS;
-
- cqp_info->cqp_cmd = OP_QP_CREATE;
- cqp_info->post_sq = 1;
- cqp_info->in.u.qp_create.qp = qp;
- cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP QP create fail");
- return status;
-}
-
-/**
- * i40iw_cqp_cq_destroy_cmd - destroy the cqp cq
- * @dev: device pointer
- * @cq: pointer to cq
- */
-void i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq)
-{
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
-
- i40iw_cq_wq_destroy(iwdev, cq);
-}
-
-/**
- * i40iw_cqp_qp_destroy_cmd - destroy the cqp
- * @dev: device pointer
- * @qp: pointer to qp
- */
-void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp)
-{
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
- struct i40iw_cqp *iwcqp = &iwdev->cqp;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- enum i40iw_status_code status;
-
- cqp_request = i40iw_get_cqp_request(iwcqp, true);
- if (!cqp_request)
- return;
-
- cqp_info = &cqp_request->info;
- memset(cqp_info, 0, sizeof(*cqp_info));
-
- cqp_info->cqp_cmd = OP_QP_DESTROY;
- cqp_info->post_sq = 1;
- cqp_info->in.u.qp_destroy.qp = qp;
- cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
- cqp_info->in.u.qp_destroy.remove_hash_idx = true;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP QP_DESTROY fail");
-}
-
-
-/**
- * i40iw_ieq_mpa_crc_ae - generate AE for crc error
- * @dev: hardware control device structure
- * @qp: hardware control qp
- */
-void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp)
-{
- struct i40iw_gen_ae_info info;
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
-
- i40iw_debug(dev, I40IW_DEBUG_AEQ, "%s entered\n", __func__);
- info.ae_code = I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR;
- info.ae_source = I40IW_AE_SOURCE_RQ;
- i40iw_gen_ae(iwdev, qp, &info, false);
-}
-
-/**
- * i40iw_init_hash_desc - initialize hash for crc calculation
- * @desc: cryption type
- */
-enum i40iw_status_code i40iw_init_hash_desc(struct shash_desc **desc)
-{
- struct crypto_shash *tfm;
- struct shash_desc *tdesc;
-
- tfm = crypto_alloc_shash("crc32c", 0, 0);
- if (IS_ERR(tfm))
- return I40IW_ERR_MPA_CRC;
-
- tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm),
- GFP_KERNEL);
- if (!tdesc) {
- crypto_free_shash(tfm);
- return I40IW_ERR_MPA_CRC;
- }
- tdesc->tfm = tfm;
- *desc = tdesc;
-
- return 0;
-}
-
-/**
- * i40iw_free_hash_desc - free hash desc
- * @desc: to be freed
- */
-void i40iw_free_hash_desc(struct shash_desc *desc)
-{
- if (desc) {
- crypto_free_shash(desc->tfm);
- kfree(desc);
- }
-}
-
-/**
- * i40iw_alloc_query_fpm_buf - allocate buffer for fpm
- * @dev: hardware control device structure
- * @mem: buffer ptr for fpm to be allocated
- * @return: memory allocation status
- */
-enum i40iw_status_code i40iw_alloc_query_fpm_buf(struct i40iw_sc_dev *dev,
- struct i40iw_dma_mem *mem)
-{
- enum i40iw_status_code status;
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
-
- status = i40iw_obj_aligned_mem(iwdev, mem, I40IW_QUERY_FPM_BUF_SIZE,
- I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK);
- return status;
-}
-
-/**
- * i40iw_ieq_check_mpacrc - check if mpa crc is OK
- * @desc: desc for hash
- * @addr: address of buffer for crc
- * @length: length of buffer
- * @value: value to be compared
- */
-enum i40iw_status_code i40iw_ieq_check_mpacrc(struct shash_desc *desc,
- void *addr,
- u32 length,
- u32 value)
-{
- u32 crc = 0;
- int ret;
- enum i40iw_status_code ret_code = 0;
-
- crypto_shash_init(desc);
- ret = crypto_shash_update(desc, addr, length);
- if (!ret)
- crypto_shash_final(desc, (u8 *)&crc);
- if (crc != value) {
- i40iw_pr_err("mpa crc check fail\n");
- ret_code = I40IW_ERR_MPA_CRC;
- }
- return ret_code;
-}
-
-/**
- * i40iw_ieq_get_qp - get qp based on quad in puda buffer
- * @dev: hardware control device structure
- * @buf: receive puda buffer on exception q
- */
-struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev,
- struct i40iw_puda_buf *buf)
-{
- struct i40iw_device *iwdev = (struct i40iw_device *)dev->back_dev;
- struct i40iw_qp *iwqp;
- struct i40iw_cm_node *cm_node;
- u32 loc_addr[4], rem_addr[4];
- u16 loc_port, rem_port;
- struct ipv6hdr *ip6h;
- struct iphdr *iph = (struct iphdr *)buf->iph;
- struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
-
- if (iph->version == 4) {
- memset(loc_addr, 0, sizeof(loc_addr));
- loc_addr[0] = ntohl(iph->daddr);
- memset(rem_addr, 0, sizeof(rem_addr));
- rem_addr[0] = ntohl(iph->saddr);
- } else {
- ip6h = (struct ipv6hdr *)buf->iph;
- i40iw_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32);
- i40iw_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32);
- }
- loc_port = ntohs(tcph->dest);
- rem_port = ntohs(tcph->source);
-
- cm_node = i40iw_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
- loc_addr, false, true);
- if (!cm_node)
- return NULL;
- iwqp = cm_node->iwqp;
- return &iwqp->sc_qp;
-}
-
-/**
- * i40iw_ieq_update_tcpip_info - update tcpip in the buffer
- * @buf: puda to update
- * @length: length of buffer
- * @seqnum: seq number for tcp
- */
-void i40iw_ieq_update_tcpip_info(struct i40iw_puda_buf *buf, u16 length, u32 seqnum)
-{
- struct tcphdr *tcph;
- struct iphdr *iph;
- u16 iphlen;
- u16 packetsize;
- u8 *addr = (u8 *)buf->mem.va;
-
- iphlen = (buf->ipv4) ? 20 : 40;
- iph = (struct iphdr *)(addr + buf->maclen);
- tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
- packetsize = length + buf->tcphlen + iphlen;
-
- iph->tot_len = htons(packetsize);
- tcph->seq = htonl(seqnum);
-}
-
-/**
- * i40iw_puda_get_tcpip_info - get tcpip info from puda buffer
- * @info: to get information
- * @buf: puda buffer
- */
-enum i40iw_status_code i40iw_puda_get_tcpip_info(struct i40iw_puda_completion_info *info,
- struct i40iw_puda_buf *buf)
-{
- struct iphdr *iph;
- struct ipv6hdr *ip6h;
- struct tcphdr *tcph;
- u16 iphlen;
- u16 pkt_len;
- u8 *mem = (u8 *)buf->mem.va;
- struct ethhdr *ethh = (struct ethhdr *)buf->mem.va;
-
- if (ethh->h_proto == htons(0x8100)) {
- info->vlan_valid = true;
- buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) & VLAN_VID_MASK;
- }
- buf->maclen = (info->vlan_valid) ? 18 : 14;
- iphlen = (info->l3proto) ? 40 : 20;
- buf->ipv4 = (info->l3proto) ? false : true;
- buf->iph = mem + buf->maclen;
- iph = (struct iphdr *)buf->iph;
-
- buf->tcph = buf->iph + iphlen;
- tcph = (struct tcphdr *)buf->tcph;
-
- if (buf->ipv4) {
- pkt_len = ntohs(iph->tot_len);
- } else {
- ip6h = (struct ipv6hdr *)buf->iph;
- pkt_len = ntohs(ip6h->payload_len) + iphlen;
- }
-
- buf->totallen = pkt_len + buf->maclen;
-
- if (info->payload_len < buf->totallen) {
- i40iw_pr_err("payload_len = 0x%x totallen expected0x%x\n",
- info->payload_len, buf->totallen);
- return I40IW_ERR_INVALID_SIZE;
- }
-
- buf->tcphlen = (tcph->doff) << 2;
- buf->datalen = pkt_len - iphlen - buf->tcphlen;
- buf->data = (buf->datalen) ? buf->tcph + buf->tcphlen : NULL;
- buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
- buf->seqnum = ntohl(tcph->seq);
- return 0;
-}
-
-/**
- * i40iw_hw_stats_timeout - Stats timer-handler which updates all HW stats
- * @vsi: pointer to the vsi structure
- */
-static void i40iw_hw_stats_timeout(struct timer_list *t)
-{
- struct i40iw_vsi_pestat *pf_devstat = from_timer(pf_devstat, t,
- stats_timer);
- struct i40iw_sc_vsi *sc_vsi = pf_devstat->vsi;
- struct i40iw_sc_dev *pf_dev = sc_vsi->dev;
- struct i40iw_vsi_pestat *vf_devstat = NULL;
- u16 iw_vf_idx;
- unsigned long flags;
-
- /*PF*/
- i40iw_hw_stats_read_all(pf_devstat, &pf_devstat->hw_stats);
-
- for (iw_vf_idx = 0; iw_vf_idx < I40IW_MAX_PE_ENABLED_VF_COUNT; iw_vf_idx++) {
- spin_lock_irqsave(&pf_devstat->lock, flags);
- if (pf_dev->vf_dev[iw_vf_idx]) {
- if (pf_dev->vf_dev[iw_vf_idx]->stats_initialized) {
- vf_devstat = &pf_dev->vf_dev[iw_vf_idx]->pestat;
- i40iw_hw_stats_read_all(vf_devstat, &vf_devstat->hw_stats);
- }
- }
- spin_unlock_irqrestore(&pf_devstat->lock, flags);
- }
-
- mod_timer(&pf_devstat->stats_timer,
- jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
-}
-
-/**
- * i40iw_hw_stats_start_timer - Start periodic stats timer
- * @vsi: pointer to the vsi structure
- */
-void i40iw_hw_stats_start_timer(struct i40iw_sc_vsi *vsi)
-{
- struct i40iw_vsi_pestat *devstat = vsi->pestat;
-
- timer_setup(&devstat->stats_timer, i40iw_hw_stats_timeout, 0);
- mod_timer(&devstat->stats_timer,
- jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
-}
-
-/**
- * i40iw_hw_stats_stop_timer - Delete periodic stats timer
- * @vsi: pointer to the vsi structure
- */
-void i40iw_hw_stats_stop_timer(struct i40iw_sc_vsi *vsi)
-{
- struct i40iw_vsi_pestat *devstat = vsi->pestat;
-
- del_timer_sync(&devstat->stats_timer);
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
deleted file mode 100644
index 1b6fb1380961..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ /dev/null
@@ -1,2791 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/random.h>
-#include <linux/highmem.h>
-#include <linux/time.h>
-#include <linux/hugetlb.h>
-#include <linux/irq.h>
-#include <asm/byteorder.h>
-#include <net/ip.h>
-#include <rdma/ib_verbs.h>
-#include <rdma/iw_cm.h>
-#include <rdma/ib_user_verbs.h>
-#include <rdma/ib_umem.h>
-#include <rdma/uverbs_ioctl.h>
-#include "i40iw.h"
-
-/**
- * i40iw_query_device - get device attributes
- * @ibdev: device pointer from stack
- * @props: returning device attributes
- * @udata: user data
- */
-static int i40iw_query_device(struct ib_device *ibdev,
- struct ib_device_attr *props,
- struct ib_udata *udata)
-{
- struct i40iw_device *iwdev = to_iwdev(ibdev);
-
- if (udata->inlen || udata->outlen)
- return -EINVAL;
- memset(props, 0, sizeof(*props));
- ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr);
- props->fw_ver = i40iw_fw_major_ver(&iwdev->sc_dev) << 32 |
- i40iw_fw_minor_ver(&iwdev->sc_dev);
- props->device_cap_flags = iwdev->device_cap_flags;
- props->vendor_id = iwdev->ldev->pcidev->vendor;
- props->vendor_part_id = iwdev->ldev->pcidev->device;
- props->hw_ver = (u32)iwdev->sc_dev.hw_rev;
- props->max_mr_size = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
- props->max_qp = iwdev->max_qp - iwdev->used_qps;
- props->max_qp_wr = I40IW_MAX_QP_WRS;
- props->max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
- props->max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
- props->max_cq = iwdev->max_cq - iwdev->used_cqs;
- props->max_cqe = iwdev->max_cqe;
- props->max_mr = iwdev->max_mr - iwdev->used_mrs;
- props->max_pd = iwdev->max_pd - iwdev->used_pds;
- props->max_sge_rd = I40IW_MAX_SGE_RD;
- props->max_qp_rd_atom = I40IW_MAX_IRD_SIZE;
- props->max_qp_init_rd_atom = props->max_qp_rd_atom;
- props->atomic_cap = IB_ATOMIC_NONE;
- props->max_map_per_fmr = 1;
- props->max_fast_reg_page_list_len = I40IW_MAX_PAGES_PER_FMR;
- return 0;
-}
-
-/**
- * i40iw_query_port - get port attrubutes
- * @ibdev: device pointer from stack
- * @port: port number for query
- * @props: returning device attributes
- */
-static int i40iw_query_port(struct ib_device *ibdev,
- u8 port,
- struct ib_port_attr *props)
-{
- props->lid = 1;
- props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
- IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
- props->gid_tbl_len = 1;
- props->pkey_tbl_len = 1;
- props->active_width = IB_WIDTH_4X;
- props->active_speed = 1;
- props->max_msg_sz = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
- return 0;
-}
-
-/**
- * i40iw_alloc_ucontext - Allocate the user context data structure
- * @uctx: Uverbs context pointer from stack
- * @udata: user data
- *
- * This keeps track of all objects associated with a particular
- * user-mode client.
- */
-static int i40iw_alloc_ucontext(struct ib_ucontext *uctx,
- struct ib_udata *udata)
-{
- struct ib_device *ibdev = uctx->device;
- struct i40iw_device *iwdev = to_iwdev(ibdev);
- struct i40iw_alloc_ucontext_req req;
- struct i40iw_alloc_ucontext_resp uresp = {};
- struct i40iw_ucontext *ucontext = to_ucontext(uctx);
-
- if (ib_copy_from_udata(&req, udata, sizeof(req)))
- return -EINVAL;
-
- if (req.userspace_ver < 4 || req.userspace_ver > I40IW_ABI_VER) {
- i40iw_pr_err("Unsupported provider library version %u.\n", req.userspace_ver);
- return -EINVAL;
- }
-
- uresp.max_qps = iwdev->max_qp;
- uresp.max_pds = iwdev->max_pd;
- uresp.wq_size = iwdev->max_qp_wr * 2;
- uresp.kernel_ver = req.userspace_ver;
-
- ucontext->iwdev = iwdev;
- ucontext->abi_ver = req.userspace_ver;
-
- if (ib_copy_to_udata(udata, &uresp, sizeof(uresp)))
- return -EFAULT;
-
- INIT_LIST_HEAD(&ucontext->cq_reg_mem_list);
- spin_lock_init(&ucontext->cq_reg_mem_list_lock);
- INIT_LIST_HEAD(&ucontext->qp_reg_mem_list);
- spin_lock_init(&ucontext->qp_reg_mem_list_lock);
-
- return 0;
-}
-
-/**
- * i40iw_dealloc_ucontext - deallocate the user context data structure
- * @context: user context created during alloc
- */
-static void i40iw_dealloc_ucontext(struct ib_ucontext *context)
-{
- return;
-}
-
-/**
- * i40iw_mmap - user memory map
- * @context: context created during alloc
- * @vma: kernel info for user memory map
- */
-static int i40iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
-{
- struct i40iw_ucontext *ucontext;
- u64 db_addr_offset, push_offset, pfn;
-
- ucontext = to_ucontext(context);
- if (ucontext->iwdev->sc_dev.is_pf) {
- db_addr_offset = I40IW_DB_ADDR_OFFSET;
- push_offset = I40IW_PUSH_OFFSET;
- if (vma->vm_pgoff)
- vma->vm_pgoff += I40IW_PF_FIRST_PUSH_PAGE_INDEX - 1;
- } else {
- db_addr_offset = I40IW_VF_DB_ADDR_OFFSET;
- push_offset = I40IW_VF_PUSH_OFFSET;
- if (vma->vm_pgoff)
- vma->vm_pgoff += I40IW_VF_FIRST_PUSH_PAGE_INDEX - 1;
- }
-
- vma->vm_pgoff += db_addr_offset >> PAGE_SHIFT;
-
- if (vma->vm_pgoff == (db_addr_offset >> PAGE_SHIFT)) {
- vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
- } else {
- if ((vma->vm_pgoff - (push_offset >> PAGE_SHIFT)) % 2)
- vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
- else
- vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
- }
-
- pfn = vma->vm_pgoff +
- (pci_resource_start(ucontext->iwdev->ldev->pcidev, 0) >>
- PAGE_SHIFT);
-
- return rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,
- vma->vm_page_prot, NULL);
-}
-
-/**
- * i40iw_alloc_push_page - allocate a push page for qp
- * @iwdev: iwarp device
- * @qp: hardware control qp
- */
-static void i40iw_alloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp)
-{
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- enum i40iw_status_code status;
-
- if (qp->push_idx != I40IW_INVALID_PUSH_PAGE_INDEX)
- return;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
- if (!cqp_request)
- return;
-
- atomic_inc(&cqp_request->refcount);
-
- cqp_info = &cqp_request->info;
- cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE;
- cqp_info->post_sq = 1;
-
- cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
- cqp_info->in.u.manage_push_page.info.free_page = 0;
- cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;
- cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
-
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (!status)
- qp->push_idx = cqp_request->compl_info.op_ret_val;
- else
- i40iw_pr_err("CQP-OP Push page fail");
- i40iw_put_cqp_request(&iwdev->cqp, cqp_request);
-}
-
-/**
- * i40iw_dealloc_push_page - free a push page for qp
- * @iwdev: iwarp device
- * @qp: hardware control qp
- */
-static void i40iw_dealloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_qp *qp)
-{
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- enum i40iw_status_code status;
-
- if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX)
- return;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false);
- if (!cqp_request)
- return;
-
- cqp_info = &cqp_request->info;
- cqp_info->cqp_cmd = OP_MANAGE_PUSH_PAGE;
- cqp_info->post_sq = 1;
-
- cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
- cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
- cqp_info->in.u.manage_push_page.info.free_page = 1;
- cqp_info->in.u.manage_push_page.cqp = &iwdev->cqp.sc_cqp;
- cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
-
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (!status)
- qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;
- else
- i40iw_pr_err("CQP-OP Push page fail");
-}
-
-/**
- * i40iw_alloc_pd - allocate protection domain
- * @pd: PD pointer
- * @udata: user data
- */
-static int i40iw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
-{
- struct i40iw_pd *iwpd = to_iwpd(pd);
- struct i40iw_device *iwdev = to_iwdev(pd->device);
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_alloc_pd_resp uresp;
- struct i40iw_sc_pd *sc_pd;
- u32 pd_id = 0;
- int err;
-
- if (iwdev->closing)
- return -ENODEV;
-
- err = i40iw_alloc_resource(iwdev, iwdev->allocated_pds,
- iwdev->max_pd, &pd_id, &iwdev->next_pd);
- if (err) {
- i40iw_pr_err("alloc resource failed\n");
- return err;
- }
-
- sc_pd = &iwpd->sc_pd;
-
- if (udata) {
- struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(
- udata, struct i40iw_ucontext, ibucontext);
- dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
- memset(&uresp, 0, sizeof(uresp));
- uresp.pd_id = pd_id;
- if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
- err = -EFAULT;
- goto error;
- }
- } else {
- dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id, -1);
- }
-
- i40iw_add_pdusecount(iwpd);
- return 0;
-
-error:
- i40iw_free_resource(iwdev, iwdev->allocated_pds, pd_id);
- return err;
-}
-
-/**
- * i40iw_dealloc_pd - deallocate pd
- * @ibpd: ptr of pd to be deallocated
- * @udata: user data or null for kernel object
- */
-static void i40iw_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
-{
- struct i40iw_pd *iwpd = to_iwpd(ibpd);
- struct i40iw_device *iwdev = to_iwdev(ibpd->device);
-
- i40iw_rem_pdusecount(iwpd, iwdev);
-}
-
-/**
- * i40iw_get_pbl - Retrieve pbl from a list given a virtual
- * address
- * @va: user virtual address
- * @pbl_list: pbl list to search in (QP's or CQ's)
- */
-static struct i40iw_pbl *i40iw_get_pbl(unsigned long va,
- struct list_head *pbl_list)
-{
- struct i40iw_pbl *iwpbl;
-
- list_for_each_entry(iwpbl, pbl_list, list) {
- if (iwpbl->user_base == va) {
- iwpbl->on_list = false;
- list_del(&iwpbl->list);
- return iwpbl;
- }
- }
- return NULL;
-}
-
-/**
- * i40iw_free_qp_resources - free up memory resources for qp
- * @iwdev: iwarp device
- * @iwqp: qp ptr (user or kernel)
- * @qp_num: qp number assigned
- */
-void i40iw_free_qp_resources(struct i40iw_device *iwdev,
- struct i40iw_qp *iwqp,
- u32 qp_num)
-{
- struct i40iw_pbl *iwpbl = &iwqp->iwpbl;
-
- i40iw_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
- i40iw_dealloc_push_page(iwdev, &iwqp->sc_qp);
- if (qp_num)
- i40iw_free_resource(iwdev, iwdev->allocated_qps, qp_num);
- if (iwpbl->pbl_allocated)
- i40iw_free_pble(iwdev->pble_rsrc, &iwpbl->pble_alloc);
- i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->q2_ctx_mem);
- i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->kqp.dma_mem);
- kfree(iwqp->kqp.wrid_mem);
- iwqp->kqp.wrid_mem = NULL;
- kfree(iwqp->allocated_buffer);
-}
-
-/**
- * i40iw_clean_cqes - clean cq entries for qp
- * @iwqp: qp ptr (user or kernel)
- * @iwcq: cq ptr
- */
-static void i40iw_clean_cqes(struct i40iw_qp *iwqp, struct i40iw_cq *iwcq)
-{
- struct i40iw_cq_uk *ukcq = &iwcq->sc_cq.cq_uk;
-
- ukcq->ops.iw_cq_clean(&iwqp->sc_qp.qp_uk, ukcq);
-}
-
-/**
- * i40iw_destroy_qp - destroy qp
- * @ibqp: qp's ib pointer also to get to device's qp address
- */
-static int i40iw_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
-{
- struct i40iw_qp *iwqp = to_iwqp(ibqp);
-
- iwqp->destroyed = 1;
-
- if (iwqp->ibqp_state >= IB_QPS_INIT && iwqp->ibqp_state < IB_QPS_RTS)
- i40iw_next_iw_state(iwqp, I40IW_QP_STATE_ERROR, 0, 0, 0);
-
- if (!iwqp->user_mode) {
- if (iwqp->iwscq) {
- i40iw_clean_cqes(iwqp, iwqp->iwscq);
- if (iwqp->iwrcq != iwqp->iwscq)
- i40iw_clean_cqes(iwqp, iwqp->iwrcq);
- }
- }
-
- i40iw_rem_ref(&iwqp->ibqp);
- return 0;
-}
-
-/**
- * i40iw_setup_virt_qp - setup for allocation of virtual qp
- * @dev: iwarp device
- * @qp: qp ptr
- * @init_info: initialize info to return
- */
-static int i40iw_setup_virt_qp(struct i40iw_device *iwdev,
- struct i40iw_qp *iwqp,
- struct i40iw_qp_init_info *init_info)
-{
- struct i40iw_pbl *iwpbl = &iwqp->iwpbl;
- struct i40iw_qp_mr *qpmr = &iwpbl->qp_mr;
-
- iwqp->page = qpmr->sq_page;
- init_info->shadow_area_pa = cpu_to_le64(qpmr->shadow);
- if (iwpbl->pbl_allocated) {
- init_info->virtual_map = true;
- init_info->sq_pa = qpmr->sq_pbl.idx;
- init_info->rq_pa = qpmr->rq_pbl.idx;
- } else {
- init_info->sq_pa = qpmr->sq_pbl.addr;
- init_info->rq_pa = qpmr->rq_pbl.addr;
- }
- return 0;
-}
-
-/**
- * i40iw_setup_kmode_qp - setup initialization for kernel mode qp
- * @iwdev: iwarp device
- * @iwqp: qp ptr (user or kernel)
- * @info: initialize info to return
- */
-static int i40iw_setup_kmode_qp(struct i40iw_device *iwdev,
- struct i40iw_qp *iwqp,
- struct i40iw_qp_init_info *info)
-{
- struct i40iw_dma_mem *mem = &iwqp->kqp.dma_mem;
- u32 sqdepth, rqdepth;
- u8 sqshift;
- u32 size;
- enum i40iw_status_code status;
- struct i40iw_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
-
- i40iw_get_wqe_shift(ukinfo->max_sq_frag_cnt, ukinfo->max_inline_data, &sqshift);
- status = i40iw_get_sqdepth(ukinfo->sq_size, sqshift, &sqdepth);
- if (status)
- return -ENOMEM;
-
- status = i40iw_get_rqdepth(ukinfo->rq_size, I40IW_MAX_RQ_WQE_SHIFT, &rqdepth);
- if (status)
- return -ENOMEM;
-
- size = sqdepth * sizeof(struct i40iw_sq_uk_wr_trk_info) + (rqdepth << 3);
- iwqp->kqp.wrid_mem = kzalloc(size, GFP_KERNEL);
-
- ukinfo->sq_wrtrk_array = (struct i40iw_sq_uk_wr_trk_info *)iwqp->kqp.wrid_mem;
- if (!ukinfo->sq_wrtrk_array)
- return -ENOMEM;
-
- ukinfo->rq_wrid_array = (u64 *)&ukinfo->sq_wrtrk_array[sqdepth];
-
- size = (sqdepth + rqdepth) * I40IW_QP_WQE_MIN_SIZE;
- size += (I40IW_SHADOW_AREA_SIZE << 3);
-
- status = i40iw_allocate_dma_mem(iwdev->sc_dev.hw, mem, size, 256);
- if (status) {
- kfree(ukinfo->sq_wrtrk_array);
- ukinfo->sq_wrtrk_array = NULL;
- return -ENOMEM;
- }
-
- ukinfo->sq = mem->va;
- info->sq_pa = mem->pa;
-
- ukinfo->rq = &ukinfo->sq[sqdepth];
- info->rq_pa = info->sq_pa + (sqdepth * I40IW_QP_WQE_MIN_SIZE);
-
- ukinfo->shadow_area = ukinfo->rq[rqdepth].elem;
- info->shadow_area_pa = info->rq_pa + (rqdepth * I40IW_QP_WQE_MIN_SIZE);
-
- ukinfo->sq_size = sqdepth >> sqshift;
- ukinfo->rq_size = rqdepth >> I40IW_MAX_RQ_WQE_SHIFT;
- ukinfo->qp_id = iwqp->ibqp.qp_num;
- return 0;
-}
-
-/**
- * i40iw_create_qp - create qp
- * @ibpd: ptr of pd
- * @init_attr: attributes for qp
- * @udata: user data for create qp
- */
-static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd,
- struct ib_qp_init_attr *init_attr,
- struct ib_udata *udata)
-{
- struct i40iw_pd *iwpd = to_iwpd(ibpd);
- struct i40iw_device *iwdev = to_iwdev(ibpd->device);
- struct i40iw_cqp *iwcqp = &iwdev->cqp;
- struct i40iw_qp *iwqp;
- struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(
- udata, struct i40iw_ucontext, ibucontext);
- struct i40iw_create_qp_req req;
- struct i40iw_create_qp_resp uresp;
- u32 qp_num = 0;
- void *mem;
- enum i40iw_status_code ret;
- int err_code;
- int sq_size;
- int rq_size;
- struct i40iw_sc_qp *qp;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_qp_init_info init_info;
- struct i40iw_create_qp_info *qp_info;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
-
- struct i40iw_qp_host_ctx_info *ctx_info;
- struct i40iwarp_offload_info *iwarp_info;
- unsigned long flags;
-
- if (iwdev->closing)
- return ERR_PTR(-ENODEV);
-
- if (init_attr->create_flags)
- return ERR_PTR(-EINVAL);
- if (init_attr->cap.max_inline_data > I40IW_MAX_INLINE_DATA_SIZE)
- init_attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;
-
- if (init_attr->cap.max_send_sge > I40IW_MAX_WQ_FRAGMENT_COUNT)
- init_attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
-
- if (init_attr->cap.max_recv_sge > I40IW_MAX_WQ_FRAGMENT_COUNT)
- init_attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
-
- memset(&init_info, 0, sizeof(init_info));
-
- sq_size = init_attr->cap.max_send_wr;
- rq_size = init_attr->cap.max_recv_wr;
-
- init_info.vsi = &iwdev->vsi;
- init_info.qp_uk_init_info.sq_size = sq_size;
- init_info.qp_uk_init_info.rq_size = rq_size;
- init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge;
- init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge;
- init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data;
-
- mem = kzalloc(sizeof(*iwqp), GFP_KERNEL);
- if (!mem)
- return ERR_PTR(-ENOMEM);
-
- iwqp = (struct i40iw_qp *)mem;
- iwqp->allocated_buffer = mem;
- qp = &iwqp->sc_qp;
- qp->back_qp = (void *)iwqp;
- qp->push_idx = I40IW_INVALID_PUSH_PAGE_INDEX;
-
- iwqp->ctx_info.iwarp_info = &iwqp->iwarp_info;
-
- if (i40iw_allocate_dma_mem(dev->hw,
- &iwqp->q2_ctx_mem,
- I40IW_Q2_BUFFER_SIZE + I40IW_QP_CTX_SIZE,
- 256)) {
- i40iw_pr_err("dma_mem failed\n");
- err_code = -ENOMEM;
- goto error;
- }
-
- init_info.q2 = iwqp->q2_ctx_mem.va;
- init_info.q2_pa = iwqp->q2_ctx_mem.pa;
-
- init_info.host_ctx = (void *)init_info.q2 + I40IW_Q2_BUFFER_SIZE;
- init_info.host_ctx_pa = init_info.q2_pa + I40IW_Q2_BUFFER_SIZE;
-
- err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_qps, iwdev->max_qp,
- &qp_num, &iwdev->next_qp);
- if (err_code) {
- i40iw_pr_err("qp resource\n");
- goto error;
- }
-
- iwqp->iwdev = iwdev;
- iwqp->iwpd = iwpd;
- iwqp->ibqp.qp_num = qp_num;
- qp = &iwqp->sc_qp;
- iwqp->iwscq = to_iwcq(init_attr->send_cq);
- iwqp->iwrcq = to_iwcq(init_attr->recv_cq);
-
- iwqp->host_ctx.va = init_info.host_ctx;
- iwqp->host_ctx.pa = init_info.host_ctx_pa;
- iwqp->host_ctx.size = I40IW_QP_CTX_SIZE;
-
- init_info.pd = &iwpd->sc_pd;
- init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num;
- iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;
-
- if (init_attr->qp_type != IB_QPT_RC) {
- err_code = -EOPNOTSUPP;
- goto error;
- }
- if (iwdev->push_mode)
- i40iw_alloc_push_page(iwdev, qp);
- if (udata) {
- err_code = ib_copy_from_udata(&req, udata, sizeof(req));
- if (err_code) {
- i40iw_pr_err("ib_copy_from_data\n");
- goto error;
- }
- iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;
- iwqp->user_mode = 1;
-
- if (req.user_wqe_buffers) {
- struct i40iw_pbl *iwpbl;
-
- spin_lock_irqsave(
- &ucontext->qp_reg_mem_list_lock, flags);
- iwpbl = i40iw_get_pbl(
- (unsigned long)req.user_wqe_buffers,
- &ucontext->qp_reg_mem_list);
- spin_unlock_irqrestore(
- &ucontext->qp_reg_mem_list_lock, flags);
-
- if (!iwpbl) {
- err_code = -ENODATA;
- i40iw_pr_err("no pbl info\n");
- goto error;
- }
- memcpy(&iwqp->iwpbl, iwpbl, sizeof(iwqp->iwpbl));
- }
- err_code = i40iw_setup_virt_qp(iwdev, iwqp, &init_info);
- } else {
- err_code = i40iw_setup_kmode_qp(iwdev, iwqp, &init_info);
- }
-
- if (err_code) {
- i40iw_pr_err("setup qp failed\n");
- goto error;
- }
-
- init_info.type = I40IW_QP_TYPE_IWARP;
- ret = dev->iw_priv_qp_ops->qp_init(qp, &init_info);
- if (ret) {
- err_code = -EPROTO;
- i40iw_pr_err("qp_init fail\n");
- goto error;
- }
- ctx_info = &iwqp->ctx_info;
- iwarp_info = &iwqp->iwarp_info;
- iwarp_info->rd_enable = true;
- iwarp_info->wr_rdresp_en = true;
- if (!iwqp->user_mode) {
- iwarp_info->fast_reg_en = true;
- iwarp_info->priv_mode_en = true;
- }
- iwarp_info->ddp_ver = 1;
- iwarp_info->rdmap_ver = 1;
-
- ctx_info->iwarp_info_valid = true;
- ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
- ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
- if (qp->push_idx == I40IW_INVALID_PUSH_PAGE_INDEX) {
- ctx_info->push_mode_en = false;
- } else {
- ctx_info->push_mode_en = true;
- ctx_info->push_idx = qp->push_idx;
- }
-
- ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
- (u64 *)iwqp->host_ctx.va,
- ctx_info);
- ctx_info->iwarp_info_valid = false;
- cqp_request = i40iw_get_cqp_request(iwcqp, true);
- if (!cqp_request) {
- err_code = -ENOMEM;
- goto error;
- }
- cqp_info = &cqp_request->info;
- qp_info = &cqp_request->info.in.u.qp_create.info;
-
- memset(qp_info, 0, sizeof(*qp_info));
-
- qp_info->cq_num_valid = true;
- qp_info->next_iwarp_state = I40IW_QP_STATE_IDLE;
-
- cqp_info->cqp_cmd = OP_QP_CREATE;
- cqp_info->post_sq = 1;
- cqp_info->in.u.qp_create.qp = qp;
- cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
- ret = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (ret) {
- i40iw_pr_err("CQP-OP QP create fail");
- err_code = -EACCES;
- goto error;
- }
-
- i40iw_add_ref(&iwqp->ibqp);
- spin_lock_init(&iwqp->lock);
- iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
- iwdev->qp_table[qp_num] = iwqp;
- i40iw_add_pdusecount(iwqp->iwpd);
- i40iw_add_devusecount(iwdev);
- if (udata) {
- memset(&uresp, 0, sizeof(uresp));
- uresp.actual_sq_size = sq_size;
- uresp.actual_rq_size = rq_size;
- uresp.qp_id = qp_num;
- uresp.push_idx = qp->push_idx;
- err_code = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
- if (err_code) {
- i40iw_pr_err("copy_to_udata failed\n");
- i40iw_destroy_qp(&iwqp->ibqp, udata);
- /* let the completion of the qp destroy free the qp */
- return ERR_PTR(err_code);
- }
- }
- init_completion(&iwqp->sq_drained);
- init_completion(&iwqp->rq_drained);
-
- return &iwqp->ibqp;
-error:
- i40iw_free_qp_resources(iwdev, iwqp, qp_num);
- return ERR_PTR(err_code);
-}
-
-/**
- * i40iw_query - query qp attributes
- * @ibqp: qp pointer
- * @attr: attributes pointer
- * @attr_mask: Not used
- * @init_attr: qp attributes to return
- */
-static int i40iw_query_qp(struct ib_qp *ibqp,
- struct ib_qp_attr *attr,
- int attr_mask,
- struct ib_qp_init_attr *init_attr)
-{
- struct i40iw_qp *iwqp = to_iwqp(ibqp);
- struct i40iw_sc_qp *qp = &iwqp->sc_qp;
-
- attr->qp_state = iwqp->ibqp_state;
- attr->cur_qp_state = attr->qp_state;
- attr->qp_access_flags = 0;
- attr->cap.max_send_wr = qp->qp_uk.sq_size;
- attr->cap.max_recv_wr = qp->qp_uk.rq_size;
- attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;
- attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
- attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
- attr->port_num = 1;
- init_attr->event_handler = iwqp->ibqp.event_handler;
- init_attr->qp_context = iwqp->ibqp.qp_context;
- init_attr->send_cq = iwqp->ibqp.send_cq;
- init_attr->recv_cq = iwqp->ibqp.recv_cq;
- init_attr->srq = iwqp->ibqp.srq;
- init_attr->cap = attr->cap;
- init_attr->port_num = 1;
- return 0;
-}
-
-/**
- * i40iw_hw_modify_qp - setup cqp for modify qp
- * @iwdev: iwarp device
- * @iwqp: qp ptr (user or kernel)
- * @info: info for modify qp
- * @wait: flag to wait or not for modify qp completion
- */
-void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp,
- struct i40iw_modify_qp_info *info, bool wait)
-{
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- struct i40iw_modify_qp_info *m_info;
- struct i40iw_gen_ae_info ae_info;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, wait);
- if (!cqp_request)
- return;
-
- cqp_info = &cqp_request->info;
- m_info = &cqp_info->in.u.qp_modify.info;
- memcpy(m_info, info, sizeof(*m_info));
- cqp_info->cqp_cmd = OP_QP_MODIFY;
- cqp_info->post_sq = 1;
- cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
- cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
- if (!i40iw_handle_cqp_op(iwdev, cqp_request))
- return;
-
- switch (m_info->next_iwarp_state) {
- case I40IW_QP_STATE_RTS:
- if (iwqp->iwarp_state == I40IW_QP_STATE_IDLE)
- i40iw_send_reset(iwqp->cm_node);
- /* fall through */
- case I40IW_QP_STATE_IDLE:
- case I40IW_QP_STATE_TERMINATE:
- case I40IW_QP_STATE_CLOSING:
- ae_info.ae_code = I40IW_AE_BAD_CLOSE;
- ae_info.ae_source = 0;
- i40iw_gen_ae(iwdev, &iwqp->sc_qp, &ae_info, false);
- break;
- case I40IW_QP_STATE_ERROR:
- default:
- break;
- }
-}
-
-/**
- * i40iw_modify_qp - modify qp request
- * @ibqp: qp's pointer for modify
- * @attr: access attributes
- * @attr_mask: state mask
- * @udata: user data
- */
-int i40iw_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
- int attr_mask, struct ib_udata *udata)
-{
- struct i40iw_qp *iwqp = to_iwqp(ibqp);
- struct i40iw_device *iwdev = iwqp->iwdev;
- struct i40iw_qp_host_ctx_info *ctx_info;
- struct i40iwarp_offload_info *iwarp_info;
- struct i40iw_modify_qp_info info;
- u8 issue_modify_qp = 0;
- u8 dont_wait = 0;
- u32 err;
- unsigned long flags;
-
- memset(&info, 0, sizeof(info));
- ctx_info = &iwqp->ctx_info;
- iwarp_info = &iwqp->iwarp_info;
-
- spin_lock_irqsave(&iwqp->lock, flags);
-
- if (attr_mask & IB_QP_STATE) {
- if (iwdev->closing && attr->qp_state != IB_QPS_ERR) {
- err = -EINVAL;
- goto exit;
- }
-
- switch (attr->qp_state) {
- case IB_QPS_INIT:
- case IB_QPS_RTR:
- if (iwqp->iwarp_state > (u32)I40IW_QP_STATE_IDLE) {
- err = -EINVAL;
- goto exit;
- }
- if (iwqp->iwarp_state == I40IW_QP_STATE_INVALID) {
- info.next_iwarp_state = I40IW_QP_STATE_IDLE;
- issue_modify_qp = 1;
- }
- break;
- case IB_QPS_RTS:
- if ((iwqp->iwarp_state > (u32)I40IW_QP_STATE_RTS) ||
- (!iwqp->cm_id)) {
- err = -EINVAL;
- goto exit;
- }
-
- issue_modify_qp = 1;
- iwqp->hw_tcp_state = I40IW_TCP_STATE_ESTABLISHED;
- iwqp->hte_added = 1;
- info.next_iwarp_state = I40IW_QP_STATE_RTS;
- info.tcp_ctx_valid = true;
- info.ord_valid = true;
- info.arp_cache_idx_valid = true;
- info.cq_num_valid = true;
- break;
- case IB_QPS_SQD:
- if (iwqp->hw_iwarp_state > (u32)I40IW_QP_STATE_RTS) {
- err = 0;
- goto exit;
- }
- if ((iwqp->iwarp_state == (u32)I40IW_QP_STATE_CLOSING) ||
- (iwqp->iwarp_state < (u32)I40IW_QP_STATE_RTS)) {
- err = 0;
- goto exit;
- }
- if (iwqp->iwarp_state > (u32)I40IW_QP_STATE_CLOSING) {
- err = -EINVAL;
- goto exit;
- }
- info.next_iwarp_state = I40IW_QP_STATE_CLOSING;
- issue_modify_qp = 1;
- break;
- case IB_QPS_SQE:
- if (iwqp->iwarp_state >= (u32)I40IW_QP_STATE_TERMINATE) {
- err = -EINVAL;
- goto exit;
- }
- info.next_iwarp_state = I40IW_QP_STATE_TERMINATE;
- issue_modify_qp = 1;
- break;
- case IB_QPS_ERR:
- case IB_QPS_RESET:
- if (iwqp->iwarp_state == (u32)I40IW_QP_STATE_ERROR) {
- err = -EINVAL;
- goto exit;
- }
- if (iwqp->sc_qp.term_flags)
- i40iw_terminate_del_timer(&iwqp->sc_qp);
- info.next_iwarp_state = I40IW_QP_STATE_ERROR;
- if ((iwqp->hw_tcp_state > I40IW_TCP_STATE_CLOSED) &&
- iwdev->iw_status &&
- (iwqp->hw_tcp_state != I40IW_TCP_STATE_TIME_WAIT))
- info.reset_tcp_conn = true;
- else
- dont_wait = 1;
- issue_modify_qp = 1;
- info.next_iwarp_state = I40IW_QP_STATE_ERROR;
- break;
- default:
- err = -EINVAL;
- goto exit;
- }
-
- iwqp->ibqp_state = attr->qp_state;
-
- }
- if (attr_mask & IB_QP_ACCESS_FLAGS) {
- ctx_info->iwarp_info_valid = true;
- if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE)
- iwarp_info->wr_rdresp_en = true;
- if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
- iwarp_info->wr_rdresp_en = true;
- if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
- iwarp_info->rd_enable = true;
- if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
- iwarp_info->bind_en = true;
-
- if (iwqp->user_mode) {
- iwarp_info->rd_enable = true;
- iwarp_info->wr_rdresp_en = true;
- iwarp_info->priv_mode_en = false;
- }
- }
-
- if (ctx_info->iwarp_info_valid) {
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- int ret;
-
- ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
- ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
- ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
- (u64 *)iwqp->host_ctx.va,
- ctx_info);
- if (ret) {
- i40iw_pr_err("setting QP context\n");
- err = -EINVAL;
- goto exit;
- }
- }
-
- spin_unlock_irqrestore(&iwqp->lock, flags);
-
- if (issue_modify_qp) {
- i40iw_hw_modify_qp(iwdev, iwqp, &info, true);
-
- spin_lock_irqsave(&iwqp->lock, flags);
- iwqp->iwarp_state = info.next_iwarp_state;
- spin_unlock_irqrestore(&iwqp->lock, flags);
- }
-
- if (issue_modify_qp && (iwqp->ibqp_state > IB_QPS_RTS)) {
- if (dont_wait) {
- if (iwqp->cm_id && iwqp->hw_tcp_state) {
- spin_lock_irqsave(&iwqp->lock, flags);
- iwqp->hw_tcp_state = I40IW_TCP_STATE_CLOSED;
- iwqp->last_aeq = I40IW_AE_RESET_SENT;
- spin_unlock_irqrestore(&iwqp->lock, flags);
- i40iw_cm_disconn(iwqp);
- }
- } else {
- spin_lock_irqsave(&iwqp->lock, flags);
- if (iwqp->cm_id) {
- if (atomic_inc_return(&iwqp->close_timer_started) == 1) {
- iwqp->cm_id->add_ref(iwqp->cm_id);
- i40iw_schedule_cm_timer(iwqp->cm_node,
- (struct i40iw_puda_buf *)iwqp,
- I40IW_TIMER_TYPE_CLOSE, 1, 0);
- }
- }
- spin_unlock_irqrestore(&iwqp->lock, flags);
- }
- }
- return 0;
-exit:
- spin_unlock_irqrestore(&iwqp->lock, flags);
- return err;
-}
-
-/**
- * cq_free_resources - free up recources for cq
- * @iwdev: iwarp device
- * @iwcq: cq ptr
- */
-static void cq_free_resources(struct i40iw_device *iwdev, struct i40iw_cq *iwcq)
-{
- struct i40iw_sc_cq *cq = &iwcq->sc_cq;
-
- if (!iwcq->user_mode)
- i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwcq->kmem);
- i40iw_free_resource(iwdev, iwdev->allocated_cqs, cq->cq_uk.cq_id);
-}
-
-/**
- * i40iw_cq_wq_destroy - send cq destroy cqp
- * @iwdev: iwarp device
- * @cq: hardware control cq
- */
-void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq)
-{
- enum i40iw_status_code status;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
- if (!cqp_request)
- return;
-
- cqp_info = &cqp_request->info;
-
- cqp_info->cqp_cmd = OP_CQ_DESTROY;
- cqp_info->post_sq = 1;
- cqp_info->in.u.cq_destroy.cq = cq;
- cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP Destroy QP fail");
-}
-
-/**
- * i40iw_destroy_cq - destroy cq
- * @ib_cq: cq pointer
- * @udata: user data or NULL for kernel object
- */
-static void i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
-{
- struct i40iw_cq *iwcq;
- struct i40iw_device *iwdev;
- struct i40iw_sc_cq *cq;
-
- iwcq = to_iwcq(ib_cq);
- iwdev = to_iwdev(ib_cq->device);
- cq = &iwcq->sc_cq;
- i40iw_cq_wq_destroy(iwdev, cq);
- cq_free_resources(iwdev, iwcq);
- i40iw_rem_devusecount(iwdev);
-}
-
-/**
- * i40iw_create_cq - create cq
- * @ibcq: CQ allocated
- * @attr: attributes for cq
- * @udata: user data
- */
-static int i40iw_create_cq(struct ib_cq *ibcq,
- const struct ib_cq_init_attr *attr,
- struct ib_udata *udata)
-{
- struct ib_device *ibdev = ibcq->device;
- struct i40iw_device *iwdev = to_iwdev(ibdev);
- struct i40iw_cq *iwcq = to_iwcq(ibcq);
- struct i40iw_pbl *iwpbl;
- u32 cq_num = 0;
- struct i40iw_sc_cq *cq;
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_cq_init_info info = {};
- enum i40iw_status_code status;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- struct i40iw_cq_uk_init_info *ukinfo = &info.cq_uk_init_info;
- unsigned long flags;
- int err_code;
- int entries = attr->cqe;
-
- if (iwdev->closing)
- return -ENODEV;
-
- if (entries > iwdev->max_cqe)
- return -EINVAL;
-
- err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_cqs,
- iwdev->max_cq, &cq_num,
- &iwdev->next_cq);
- if (err_code)
- return err_code;
-
- cq = &iwcq->sc_cq;
- cq->back_cq = (void *)iwcq;
- spin_lock_init(&iwcq->lock);
-
- info.dev = dev;
- ukinfo->cq_size = max(entries, 4);
- ukinfo->cq_id = cq_num;
- iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size;
- info.ceqe_mask = 0;
- if (attr->comp_vector < iwdev->ceqs_count)
- info.ceq_id = attr->comp_vector;
- info.ceq_id_valid = true;
- info.ceqe_mask = 1;
- info.type = I40IW_CQ_TYPE_IWARP;
- if (udata) {
- struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(
- udata, struct i40iw_ucontext, ibucontext);
- struct i40iw_create_cq_req req;
- struct i40iw_cq_mr *cqmr;
-
- memset(&req, 0, sizeof(req));
- iwcq->user_mode = true;
- if (ib_copy_from_udata(&req, udata, sizeof(struct i40iw_create_cq_req))) {
- err_code = -EFAULT;
- goto cq_free_resources;
- }
-
- spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
- iwpbl = i40iw_get_pbl((unsigned long)req.user_cq_buffer,
- &ucontext->cq_reg_mem_list);
- spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
- if (!iwpbl) {
- err_code = -EPROTO;
- goto cq_free_resources;
- }
-
- iwcq->iwpbl = iwpbl;
- iwcq->cq_mem_size = 0;
- cqmr = &iwpbl->cq_mr;
- info.shadow_area_pa = cpu_to_le64(cqmr->shadow);
- if (iwpbl->pbl_allocated) {
- info.virtual_map = true;
- info.pbl_chunk_size = 1;
- info.first_pm_pbl_idx = cqmr->cq_pbl.idx;
- } else {
- info.cq_base_pa = cqmr->cq_pbl.addr;
- }
- } else {
- /* Kmode allocations */
- int rsize;
- int shadow;
-
- rsize = info.cq_uk_init_info.cq_size * sizeof(struct i40iw_cqe);
- rsize = round_up(rsize, 256);
- shadow = I40IW_SHADOW_AREA_SIZE << 3;
- status = i40iw_allocate_dma_mem(dev->hw, &iwcq->kmem,
- rsize + shadow, 256);
- if (status) {
- err_code = -ENOMEM;
- goto cq_free_resources;
- }
- ukinfo->cq_base = iwcq->kmem.va;
- info.cq_base_pa = iwcq->kmem.pa;
- info.shadow_area_pa = info.cq_base_pa + rsize;
- ukinfo->shadow_area = iwcq->kmem.va + rsize;
- }
-
- if (dev->iw_priv_cq_ops->cq_init(cq, &info)) {
- i40iw_pr_err("init cq fail\n");
- err_code = -EPROTO;
- goto cq_free_resources;
- }
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
- if (!cqp_request) {
- err_code = -ENOMEM;
- goto cq_free_resources;
- }
-
- cqp_info = &cqp_request->info;
- cqp_info->cqp_cmd = OP_CQ_CREATE;
- cqp_info->post_sq = 1;
- cqp_info->in.u.cq_create.cq = cq;
- cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status) {
- i40iw_pr_err("CQP-OP Create QP fail");
- err_code = -EPROTO;
- goto cq_free_resources;
- }
-
- if (udata) {
- struct i40iw_create_cq_resp resp;
-
- memset(&resp, 0, sizeof(resp));
- resp.cq_id = info.cq_uk_init_info.cq_id;
- resp.cq_size = info.cq_uk_init_info.cq_size;
- if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
- i40iw_pr_err("copy to user data\n");
- err_code = -EPROTO;
- goto cq_destroy;
- }
- }
-
- i40iw_add_devusecount(iwdev);
- return 0;
-
-cq_destroy:
- i40iw_cq_wq_destroy(iwdev, cq);
-cq_free_resources:
- cq_free_resources(iwdev, iwcq);
- return err_code;
-}
-
-/**
- * i40iw_get_user_access - get hw access from IB access
- * @acc: IB access to return hw access
- */
-static inline u16 i40iw_get_user_access(int acc)
-{
- u16 access = 0;
-
- access |= (acc & IB_ACCESS_LOCAL_WRITE) ? I40IW_ACCESS_FLAGS_LOCALWRITE : 0;
- access |= (acc & IB_ACCESS_REMOTE_WRITE) ? I40IW_ACCESS_FLAGS_REMOTEWRITE : 0;
- access |= (acc & IB_ACCESS_REMOTE_READ) ? I40IW_ACCESS_FLAGS_REMOTEREAD : 0;
- access |= (acc & IB_ACCESS_MW_BIND) ? I40IW_ACCESS_FLAGS_BIND_WINDOW : 0;
- return access;
-}
-
-/**
- * i40iw_free_stag - free stag resource
- * @iwdev: iwarp device
- * @stag: stag to free
- */
-static void i40iw_free_stag(struct i40iw_device *iwdev, u32 stag)
-{
- u32 stag_idx;
-
- stag_idx = (stag & iwdev->mr_stagmask) >> I40IW_CQPSQ_STAG_IDX_SHIFT;
- i40iw_free_resource(iwdev, iwdev->allocated_mrs, stag_idx);
- i40iw_rem_devusecount(iwdev);
-}
-
-/**
- * i40iw_create_stag - create random stag
- * @iwdev: iwarp device
- */
-static u32 i40iw_create_stag(struct i40iw_device *iwdev)
-{
- u32 stag = 0;
- u32 stag_index = 0;
- u32 next_stag_index;
- u32 driver_key;
- u32 random;
- u8 consumer_key;
- int ret;
-
- get_random_bytes(&random, sizeof(random));
- consumer_key = (u8)random;
-
- driver_key = random & ~iwdev->mr_stagmask;
- next_stag_index = (random & iwdev->mr_stagmask) >> 8;
- next_stag_index %= iwdev->max_mr;
-
- ret = i40iw_alloc_resource(iwdev,
- iwdev->allocated_mrs, iwdev->max_mr,
- &stag_index, &next_stag_index);
- if (!ret) {
- stag = stag_index << I40IW_CQPSQ_STAG_IDX_SHIFT;
- stag |= driver_key;
- stag += (u32)consumer_key;
- i40iw_add_devusecount(iwdev);
- }
- return stag;
-}
-
-/**
- * i40iw_next_pbl_addr - Get next pbl address
- * @pbl: pointer to a pble
- * @pinfo: info pointer
- * @idx: index
- */
-static inline u64 *i40iw_next_pbl_addr(u64 *pbl,
- struct i40iw_pble_info **pinfo,
- u32 *idx)
-{
- *idx += 1;
- if ((!(*pinfo)) || (*idx != (*pinfo)->cnt))
- return ++pbl;
- *idx = 0;
- (*pinfo)++;
- return (u64 *)(*pinfo)->addr;
-}
-
-/**
- * i40iw_copy_user_pgaddrs - copy user page address to pble's os locally
- * @iwmr: iwmr for IB's user page addresses
- * @pbl: ple pointer to save 1 level or 0 level pble
- * @level: indicated level 0, 1 or 2
- */
-static void i40iw_copy_user_pgaddrs(struct i40iw_mr *iwmr,
- u64 *pbl,
- enum i40iw_pble_level level)
-{
- struct ib_umem *region = iwmr->region;
- struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
- struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
- struct i40iw_pble_info *pinfo;
- struct ib_block_iter biter;
- u32 idx = 0;
-
- pinfo = (level == I40IW_LEVEL_1) ? NULL : palloc->level2.leaf;
-
- if (iwmr->type == IW_MEMREG_TYPE_QP)
- iwpbl->qp_mr.sq_page = sg_page(region->sg_head.sgl);
-
- rdma_for_each_block(region->sg_head.sgl, &biter, region->nmap,
- iwmr->page_size) {
- *pbl = rdma_block_iter_dma_address(&biter);
- pbl = i40iw_next_pbl_addr(pbl, &pinfo, &idx);
- }
-}
-
-/**
- * i40iw_check_mem_contiguous - check if pbls stored in arr are contiguous
- * @arr: lvl1 pbl array
- * @npages: page count
- * pg_size: page size
- *
- */
-static bool i40iw_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size)
-{
- u32 pg_idx;
-
- for (pg_idx = 0; pg_idx < npages; pg_idx++) {
- if ((*arr + (pg_size * pg_idx)) != arr[pg_idx])
- return false;
- }
- return true;
-}
-
-/**
- * i40iw_check_mr_contiguous - check if MR is physically contiguous
- * @palloc: pbl allocation struct
- * pg_size: page size
- */
-static bool i40iw_check_mr_contiguous(struct i40iw_pble_alloc *palloc, u32 pg_size)
-{
- struct i40iw_pble_level2 *lvl2 = &palloc->level2;
- struct i40iw_pble_info *leaf = lvl2->leaf;
- u64 *arr = NULL;
- u64 *start_addr = NULL;
- int i;
- bool ret;
-
- if (palloc->level == I40IW_LEVEL_1) {
- arr = (u64 *)palloc->level1.addr;
- ret = i40iw_check_mem_contiguous(arr, palloc->total_cnt, pg_size);
- return ret;
- }
-
- start_addr = (u64 *)leaf->addr;
-
- for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
- arr = (u64 *)leaf->addr;
- if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr)
- return false;
- ret = i40iw_check_mem_contiguous(arr, leaf->cnt, pg_size);
- if (!ret)
- return false;
- }
-
- return true;
-}
-
-/**
- * i40iw_setup_pbles - copy user pg address to pble's
- * @iwdev: iwarp device
- * @iwmr: mr pointer for this memory registration
- * @use_pbles: flag if to use pble's
- */
-static int i40iw_setup_pbles(struct i40iw_device *iwdev,
- struct i40iw_mr *iwmr,
- bool use_pbles)
-{
- struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
- struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
- struct i40iw_pble_info *pinfo;
- u64 *pbl;
- enum i40iw_status_code status;
- enum i40iw_pble_level level = I40IW_LEVEL_1;
-
- if (use_pbles) {
- mutex_lock(&iwdev->pbl_mutex);
- status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);
- mutex_unlock(&iwdev->pbl_mutex);
- if (status)
- return -ENOMEM;
-
- iwpbl->pbl_allocated = true;
- level = palloc->level;
- pinfo = (level == I40IW_LEVEL_1) ? &palloc->level1 : palloc->level2.leaf;
- pbl = (u64 *)pinfo->addr;
- } else {
- pbl = iwmr->pgaddrmem;
- }
-
- i40iw_copy_user_pgaddrs(iwmr, pbl, level);
-
- if (use_pbles)
- iwmr->pgaddrmem[0] = *pbl;
-
- return 0;
-}
-
-/**
- * i40iw_handle_q_mem - handle memory for qp and cq
- * @iwdev: iwarp device
- * @req: information for q memory management
- * @iwpbl: pble struct
- * @use_pbles: flag to use pble
- */
-static int i40iw_handle_q_mem(struct i40iw_device *iwdev,
- struct i40iw_mem_reg_req *req,
- struct i40iw_pbl *iwpbl,
- bool use_pbles)
-{
- struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
- struct i40iw_mr *iwmr = iwpbl->iwmr;
- struct i40iw_qp_mr *qpmr = &iwpbl->qp_mr;
- struct i40iw_cq_mr *cqmr = &iwpbl->cq_mr;
- struct i40iw_hmc_pble *hmc_p;
- u64 *arr = iwmr->pgaddrmem;
- u32 pg_size;
- int err;
- int total;
- bool ret = true;
-
- total = req->sq_pages + req->rq_pages + req->cq_pages;
- pg_size = iwmr->page_size;
-
- err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);
- if (err)
- return err;
-
- if (use_pbles && (palloc->level != I40IW_LEVEL_1)) {
- i40iw_free_pble(iwdev->pble_rsrc, palloc);
- iwpbl->pbl_allocated = false;
- return -ENOMEM;
- }
-
- if (use_pbles)
- arr = (u64 *)palloc->level1.addr;
-
- if (iwmr->type == IW_MEMREG_TYPE_QP) {
- hmc_p = &qpmr->sq_pbl;
- qpmr->shadow = (dma_addr_t)arr[total];
-
- if (use_pbles) {
- ret = i40iw_check_mem_contiguous(arr, req->sq_pages, pg_size);
- if (ret)
- ret = i40iw_check_mem_contiguous(&arr[req->sq_pages], req->rq_pages, pg_size);
- }
-
- if (!ret) {
- hmc_p->idx = palloc->level1.idx;
- hmc_p = &qpmr->rq_pbl;
- hmc_p->idx = palloc->level1.idx + req->sq_pages;
- } else {
- hmc_p->addr = arr[0];
- hmc_p = &qpmr->rq_pbl;
- hmc_p->addr = arr[req->sq_pages];
- }
- } else { /* CQ */
- hmc_p = &cqmr->cq_pbl;
- cqmr->shadow = (dma_addr_t)arr[total];
-
- if (use_pbles)
- ret = i40iw_check_mem_contiguous(arr, req->cq_pages, pg_size);
-
- if (!ret)
- hmc_p->idx = palloc->level1.idx;
- else
- hmc_p->addr = arr[0];
- }
-
- if (use_pbles && ret) {
- i40iw_free_pble(iwdev->pble_rsrc, palloc);
- iwpbl->pbl_allocated = false;
- }
-
- return err;
-}
-
-/**
- * i40iw_hw_alloc_stag - cqp command to allocate stag
- * @iwdev: iwarp device
- * @iwmr: iwarp mr pointer
- */
-static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr)
-{
- struct i40iw_allocate_stag_info *info;
- struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
- enum i40iw_status_code status;
- int err = 0;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
- if (!cqp_request)
- return -ENOMEM;
-
- cqp_info = &cqp_request->info;
- info = &cqp_info->in.u.alloc_stag.info;
- memset(info, 0, sizeof(*info));
- info->page_size = PAGE_SIZE;
- info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT;
- info->pd_id = iwpd->sc_pd.pd_id;
- info->total_len = iwmr->length;
- info->remote_access = true;
- cqp_info->cqp_cmd = OP_ALLOC_STAG;
- cqp_info->post_sq = 1;
- cqp_info->in.u.alloc_stag.dev = &iwdev->sc_dev;
- cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request;
-
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status) {
- err = -ENOMEM;
- i40iw_pr_err("CQP-OP MR Reg fail");
- }
- return err;
-}
-
-/**
- * i40iw_alloc_mr - register stag for fast memory registration
- * @pd: ibpd pointer
- * @mr_type: memory for stag registrion
- * @max_num_sg: man number of pages
- * @udata: user data or NULL for kernel objects
- */
-static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg, struct ib_udata *udata)
-{
- struct i40iw_pd *iwpd = to_iwpd(pd);
- struct i40iw_device *iwdev = to_iwdev(pd->device);
- struct i40iw_pble_alloc *palloc;
- struct i40iw_pbl *iwpbl;
- struct i40iw_mr *iwmr;
- enum i40iw_status_code status;
- u32 stag;
- int err_code = -ENOMEM;
-
- iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
- if (!iwmr)
- return ERR_PTR(-ENOMEM);
-
- stag = i40iw_create_stag(iwdev);
- if (!stag) {
- err_code = -EOVERFLOW;
- goto err;
- }
- stag &= ~I40IW_CQPSQ_STAG_KEY_MASK;
- iwmr->stag = stag;
- iwmr->ibmr.rkey = stag;
- iwmr->ibmr.lkey = stag;
- iwmr->ibmr.pd = pd;
- iwmr->ibmr.device = pd->device;
- iwpbl = &iwmr->iwpbl;
- iwpbl->iwmr = iwmr;
- iwmr->type = IW_MEMREG_TYPE_MEM;
- palloc = &iwpbl->pble_alloc;
- iwmr->page_cnt = max_num_sg;
- mutex_lock(&iwdev->pbl_mutex);
- status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt);
- mutex_unlock(&iwdev->pbl_mutex);
- if (status)
- goto err1;
-
- if (palloc->level != I40IW_LEVEL_1)
- goto err2;
- err_code = i40iw_hw_alloc_stag(iwdev, iwmr);
- if (err_code)
- goto err2;
- iwpbl->pbl_allocated = true;
- i40iw_add_pdusecount(iwpd);
- return &iwmr->ibmr;
-err2:
- i40iw_free_pble(iwdev->pble_rsrc, palloc);
-err1:
- i40iw_free_stag(iwdev, stag);
-err:
- kfree(iwmr);
- return ERR_PTR(err_code);
-}
-
-/**
- * i40iw_set_page - populate pbl list for fmr
- * @ibmr: ib mem to access iwarp mr pointer
- * @addr: page dma address fro pbl list
- */
-static int i40iw_set_page(struct ib_mr *ibmr, u64 addr)
-{
- struct i40iw_mr *iwmr = to_iwmr(ibmr);
- struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
- struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
- u64 *pbl;
-
- if (unlikely(iwmr->npages == iwmr->page_cnt))
- return -ENOMEM;
-
- pbl = (u64 *)palloc->level1.addr;
- pbl[iwmr->npages++] = cpu_to_le64(addr);
- return 0;
-}
-
-/**
- * i40iw_map_mr_sg - map of sg list for fmr
- * @ibmr: ib mem to access iwarp mr pointer
- * @sg: scatter gather list for fmr
- * @sg_nents: number of sg pages
- */
-static int i40iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
- int sg_nents, unsigned int *sg_offset)
-{
- struct i40iw_mr *iwmr = to_iwmr(ibmr);
-
- iwmr->npages = 0;
- return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, i40iw_set_page);
-}
-
-/**
- * i40iw_drain_sq - drain the send queue
- * @ibqp: ib qp pointer
- */
-static void i40iw_drain_sq(struct ib_qp *ibqp)
-{
- struct i40iw_qp *iwqp = to_iwqp(ibqp);
- struct i40iw_sc_qp *qp = &iwqp->sc_qp;
-
- if (I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring))
- wait_for_completion(&iwqp->sq_drained);
-}
-
-/**
- * i40iw_drain_rq - drain the receive queue
- * @ibqp: ib qp pointer
- */
-static void i40iw_drain_rq(struct ib_qp *ibqp)
-{
- struct i40iw_qp *iwqp = to_iwqp(ibqp);
- struct i40iw_sc_qp *qp = &iwqp->sc_qp;
-
- if (I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring))
- wait_for_completion(&iwqp->rq_drained);
-}
-
-/**
- * i40iw_hwreg_mr - send cqp command for memory registration
- * @iwdev: iwarp device
- * @iwmr: iwarp mr pointer
- * @access: access for MR
- */
-static int i40iw_hwreg_mr(struct i40iw_device *iwdev,
- struct i40iw_mr *iwmr,
- u16 access)
-{
- struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
- struct i40iw_reg_ns_stag_info *stag_info;
- struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
- struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
- enum i40iw_status_code status;
- int err = 0;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
- if (!cqp_request)
- return -ENOMEM;
-
- cqp_info = &cqp_request->info;
- stag_info = &cqp_info->in.u.mr_reg_non_shared.info;
- memset(stag_info, 0, sizeof(*stag_info));
- stag_info->va = (void *)(unsigned long)iwpbl->user_base;
- stag_info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT;
- stag_info->stag_key = (u8)iwmr->stag;
- stag_info->total_len = iwmr->length;
- stag_info->access_rights = access;
- stag_info->pd_id = iwpd->sc_pd.pd_id;
- stag_info->addr_type = I40IW_ADDR_TYPE_VA_BASED;
- stag_info->page_size = iwmr->page_size;
-
- if (iwpbl->pbl_allocated) {
- if (palloc->level == I40IW_LEVEL_1) {
- stag_info->first_pm_pbl_index = palloc->level1.idx;
- stag_info->chunk_size = 1;
- } else {
- stag_info->first_pm_pbl_index = palloc->level2.root.idx;
- stag_info->chunk_size = 3;
- }
- } else {
- stag_info->reg_addr_pa = iwmr->pgaddrmem[0];
- }
-
- cqp_info->cqp_cmd = OP_MR_REG_NON_SHARED;
- cqp_info->post_sq = 1;
- cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->sc_dev;
- cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request;
-
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status) {
- err = -ENOMEM;
- i40iw_pr_err("CQP-OP MR Reg fail");
- }
- return err;
-}
-
-/**
- * i40iw_reg_user_mr - Register a user memory region
- * @pd: ptr of pd
- * @start: virtual start address
- * @length: length of mr
- * @virt: virtual address
- * @acc: access of mr
- * @udata: user data
- */
-static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd,
- u64 start,
- u64 length,
- u64 virt,
- int acc,
- struct ib_udata *udata)
-{
- struct i40iw_pd *iwpd = to_iwpd(pd);
- struct i40iw_device *iwdev = to_iwdev(pd->device);
- struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(
- udata, struct i40iw_ucontext, ibucontext);
- struct i40iw_pble_alloc *palloc;
- struct i40iw_pbl *iwpbl;
- struct i40iw_mr *iwmr;
- struct ib_umem *region;
- struct i40iw_mem_reg_req req;
- u64 pbl_depth = 0;
- u32 stag = 0;
- u16 access;
- u64 region_length;
- bool use_pbles = false;
- unsigned long flags;
- int err = -ENOSYS;
- int ret;
- int pg_shift;
-
- if (!udata)
- return ERR_PTR(-EOPNOTSUPP);
-
- if (iwdev->closing)
- return ERR_PTR(-ENODEV);
-
- if (length > I40IW_MAX_MR_SIZE)
- return ERR_PTR(-EINVAL);
- region = ib_umem_get(pd->device, start, length, acc);
- if (IS_ERR(region))
- return (struct ib_mr *)region;
-
- if (ib_copy_from_udata(&req, udata, sizeof(req))) {
- ib_umem_release(region);
- return ERR_PTR(-EFAULT);
- }
-
- iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
- if (!iwmr) {
- ib_umem_release(region);
- return ERR_PTR(-ENOMEM);
- }
-
- iwpbl = &iwmr->iwpbl;
- iwpbl->iwmr = iwmr;
- iwmr->region = region;
- iwmr->ibmr.pd = pd;
- iwmr->ibmr.device = pd->device;
-
- iwmr->page_size = PAGE_SIZE;
- if (req.reg_type == IW_MEMREG_TYPE_MEM)
- iwmr->page_size = ib_umem_find_best_pgsz(region, SZ_4K | SZ_2M,
- virt);
-
- region_length = region->length + (start & (iwmr->page_size - 1));
- pg_shift = ffs(iwmr->page_size) - 1;
- pbl_depth = region_length >> pg_shift;
- pbl_depth += (region_length & (iwmr->page_size - 1)) ? 1 : 0;
- iwmr->length = region->length;
-
- iwpbl->user_base = virt;
- palloc = &iwpbl->pble_alloc;
-
- iwmr->type = req.reg_type;
- iwmr->page_cnt = (u32)pbl_depth;
-
- switch (req.reg_type) {
- case IW_MEMREG_TYPE_QP:
- use_pbles = ((req.sq_pages + req.rq_pages) > 2);
- err = i40iw_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
- if (err)
- goto error;
- spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
- list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
- iwpbl->on_list = true;
- spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
- break;
- case IW_MEMREG_TYPE_CQ:
- use_pbles = (req.cq_pages > 1);
- err = i40iw_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
- if (err)
- goto error;
-
- spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
- list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
- iwpbl->on_list = true;
- spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
- break;
- case IW_MEMREG_TYPE_MEM:
- use_pbles = (iwmr->page_cnt != 1);
- access = I40IW_ACCESS_FLAGS_LOCALREAD;
-
- err = i40iw_setup_pbles(iwdev, iwmr, use_pbles);
- if (err)
- goto error;
-
- if (use_pbles) {
- ret = i40iw_check_mr_contiguous(palloc, iwmr->page_size);
- if (ret) {
- i40iw_free_pble(iwdev->pble_rsrc, palloc);
- iwpbl->pbl_allocated = false;
- }
- }
-
- access |= i40iw_get_user_access(acc);
- stag = i40iw_create_stag(iwdev);
- if (!stag) {
- err = -ENOMEM;
- goto error;
- }
-
- iwmr->stag = stag;
- iwmr->ibmr.rkey = stag;
- iwmr->ibmr.lkey = stag;
-
- err = i40iw_hwreg_mr(iwdev, iwmr, access);
- if (err) {
- i40iw_free_stag(iwdev, stag);
- goto error;
- }
-
- break;
- default:
- goto error;
- }
-
- iwmr->type = req.reg_type;
- if (req.reg_type == IW_MEMREG_TYPE_MEM)
- i40iw_add_pdusecount(iwpd);
- return &iwmr->ibmr;
-
-error:
- if (palloc->level != I40IW_LEVEL_0 && iwpbl->pbl_allocated)
- i40iw_free_pble(iwdev->pble_rsrc, palloc);
- ib_umem_release(region);
- kfree(iwmr);
- return ERR_PTR(err);
-}
-
-/**
- * i40iw_reg_phys_mr - register kernel physical memory
- * @pd: ibpd pointer
- * @addr: physical address of memory to register
- * @size: size of memory to register
- * @acc: Access rights
- * @iova_start: start of virtual address for physical buffers
- */
-struct ib_mr *i40iw_reg_phys_mr(struct ib_pd *pd,
- u64 addr,
- u64 size,
- int acc,
- u64 *iova_start)
-{
- struct i40iw_pd *iwpd = to_iwpd(pd);
- struct i40iw_device *iwdev = to_iwdev(pd->device);
- struct i40iw_pbl *iwpbl;
- struct i40iw_mr *iwmr;
- enum i40iw_status_code status;
- u32 stag;
- u16 access = I40IW_ACCESS_FLAGS_LOCALREAD;
- int ret;
-
- iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
- if (!iwmr)
- return ERR_PTR(-ENOMEM);
- iwmr->ibmr.pd = pd;
- iwmr->ibmr.device = pd->device;
- iwpbl = &iwmr->iwpbl;
- iwpbl->iwmr = iwmr;
- iwmr->type = IW_MEMREG_TYPE_MEM;
- iwpbl->user_base = *iova_start;
- stag = i40iw_create_stag(iwdev);
- if (!stag) {
- ret = -EOVERFLOW;
- goto err;
- }
- access |= i40iw_get_user_access(acc);
- iwmr->stag = stag;
- iwmr->ibmr.rkey = stag;
- iwmr->ibmr.lkey = stag;
- iwmr->page_cnt = 1;
- iwmr->pgaddrmem[0] = addr;
- iwmr->length = size;
- status = i40iw_hwreg_mr(iwdev, iwmr, access);
- if (status) {
- i40iw_free_stag(iwdev, stag);
- ret = -ENOMEM;
- goto err;
- }
-
- i40iw_add_pdusecount(iwpd);
- return &iwmr->ibmr;
- err:
- kfree(iwmr);
- return ERR_PTR(ret);
-}
-
-/**
- * i40iw_get_dma_mr - register physical mem
- * @pd: ptr of pd
- * @acc: access for memory
- */
-static struct ib_mr *i40iw_get_dma_mr(struct ib_pd *pd, int acc)
-{
- u64 kva = 0;
-
- return i40iw_reg_phys_mr(pd, 0, 0, acc, &kva);
-}
-
-/**
- * i40iw_del_mem_list - Deleting pbl list entries for CQ/QP
- * @iwmr: iwmr for IB's user page addresses
- * @ucontext: ptr to user context
- */
-static void i40iw_del_memlist(struct i40iw_mr *iwmr,
- struct i40iw_ucontext *ucontext)
-{
- struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
- unsigned long flags;
-
- switch (iwmr->type) {
- case IW_MEMREG_TYPE_CQ:
- spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
- if (iwpbl->on_list) {
- iwpbl->on_list = false;
- list_del(&iwpbl->list);
- }
- spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
- break;
- case IW_MEMREG_TYPE_QP:
- spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
- if (iwpbl->on_list) {
- iwpbl->on_list = false;
- list_del(&iwpbl->list);
- }
- spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
- break;
- default:
- break;
- }
-}
-
-/**
- * i40iw_dereg_mr - deregister mr
- * @ib_mr: mr ptr for dereg
- */
-static int i40iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
-{
- struct ib_pd *ibpd = ib_mr->pd;
- struct i40iw_pd *iwpd = to_iwpd(ibpd);
- struct i40iw_mr *iwmr = to_iwmr(ib_mr);
- struct i40iw_device *iwdev = to_iwdev(ib_mr->device);
- enum i40iw_status_code status;
- struct i40iw_dealloc_stag_info *info;
- struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
- struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
- struct i40iw_cqp_request *cqp_request;
- struct cqp_commands_info *cqp_info;
- u32 stag_idx;
-
- ib_umem_release(iwmr->region);
-
- if (iwmr->type != IW_MEMREG_TYPE_MEM) {
- /* region is released. only test for userness. */
- if (iwmr->region) {
- struct i40iw_ucontext *ucontext =
- rdma_udata_to_drv_context(
- udata,
- struct i40iw_ucontext,
- ibucontext);
-
- i40iw_del_memlist(iwmr, ucontext);
- }
- if (iwpbl->pbl_allocated && iwmr->type != IW_MEMREG_TYPE_QP)
- i40iw_free_pble(iwdev->pble_rsrc, palloc);
- kfree(iwmr);
- return 0;
- }
-
- cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true);
- if (!cqp_request)
- return -ENOMEM;
-
- cqp_info = &cqp_request->info;
- info = &cqp_info->in.u.dealloc_stag.info;
- memset(info, 0, sizeof(*info));
-
- info->pd_id = cpu_to_le32(iwpd->sc_pd.pd_id & 0x00007fff);
- info->stag_idx = RS_64_1(ib_mr->rkey, I40IW_CQPSQ_STAG_IDX_SHIFT);
- stag_idx = info->stag_idx;
- info->mr = true;
- if (iwpbl->pbl_allocated)
- info->dealloc_pbl = true;
-
- cqp_info->cqp_cmd = OP_DEALLOC_STAG;
- cqp_info->post_sq = 1;
- cqp_info->in.u.dealloc_stag.dev = &iwdev->sc_dev;
- cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
- status = i40iw_handle_cqp_op(iwdev, cqp_request);
- if (status)
- i40iw_pr_err("CQP-OP dealloc failed for stag_idx = 0x%x\n", stag_idx);
- i40iw_rem_pdusecount(iwpd, iwdev);
- i40iw_free_stag(iwdev, iwmr->stag);
- if (iwpbl->pbl_allocated)
- i40iw_free_pble(iwdev->pble_rsrc, palloc);
- kfree(iwmr);
- return 0;
-}
-
-/**
- * hw_rev_show
- */
-static ssize_t hw_rev_show(struct device *dev,
- struct device_attribute *attr, char *buf)
-{
- struct i40iw_ib_device *iwibdev =
- rdma_device_to_drv_device(dev, struct i40iw_ib_device, ibdev);
- u32 hw_rev = iwibdev->iwdev->sc_dev.hw_rev;
-
- return sprintf(buf, "%x\n", hw_rev);
-}
-static DEVICE_ATTR_RO(hw_rev);
-
-/**
- * hca_type_show
- */
-static ssize_t hca_type_show(struct device *dev,
- struct device_attribute *attr, char *buf)
-{
- return sprintf(buf, "I40IW\n");
-}
-static DEVICE_ATTR_RO(hca_type);
-
-/**
- * board_id_show
- */
-static ssize_t board_id_show(struct device *dev,
- struct device_attribute *attr, char *buf)
-{
- return sprintf(buf, "%.*s\n", 32, "I40IW Board ID");
-}
-static DEVICE_ATTR_RO(board_id);
-
-static struct attribute *i40iw_dev_attributes[] = {
- &dev_attr_hw_rev.attr,
- &dev_attr_hca_type.attr,
- &dev_attr_board_id.attr,
- NULL
-};
-
-static const struct attribute_group i40iw_attr_group = {
- .attrs = i40iw_dev_attributes,
-};
-
-/**
- * i40iw_copy_sg_list - copy sg list for qp
- * @sg_list: copied into sg_list
- * @sgl: copy from sgl
- * @num_sges: count of sg entries
- */
-static void i40iw_copy_sg_list(struct i40iw_sge *sg_list, struct ib_sge *sgl, int num_sges)
-{
- unsigned int i;
-
- for (i = 0; (i < num_sges) && (i < I40IW_MAX_WQ_FRAGMENT_COUNT); i++) {
- sg_list[i].tag_off = sgl[i].addr;
- sg_list[i].len = sgl[i].length;
- sg_list[i].stag = sgl[i].lkey;
- }
-}
-
-/**
- * i40iw_post_send - kernel application wr
- * @ibqp: qp ptr for wr
- * @ib_wr: work request ptr
- * @bad_wr: return of bad wr if err
- */
-static int i40iw_post_send(struct ib_qp *ibqp,
- const struct ib_send_wr *ib_wr,
- const struct ib_send_wr **bad_wr)
-{
- struct i40iw_qp *iwqp;
- struct i40iw_qp_uk *ukqp;
- struct i40iw_post_sq_info info;
- enum i40iw_status_code ret;
- int err = 0;
- unsigned long flags;
- bool inv_stag;
-
- iwqp = (struct i40iw_qp *)ibqp;
- ukqp = &iwqp->sc_qp.qp_uk;
-
- spin_lock_irqsave(&iwqp->lock, flags);
-
- if (iwqp->flush_issued) {
- err = -EINVAL;
- goto out;
- }
-
- while (ib_wr) {
- inv_stag = false;
- memset(&info, 0, sizeof(info));
- info.wr_id = (u64)(ib_wr->wr_id);
- if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all)
- info.signaled = true;
- if (ib_wr->send_flags & IB_SEND_FENCE)
- info.read_fence = true;
-
- switch (ib_wr->opcode) {
- case IB_WR_SEND:
- /* fall-through */
- case IB_WR_SEND_WITH_INV:
- if (ib_wr->opcode == IB_WR_SEND) {
- if (ib_wr->send_flags & IB_SEND_SOLICITED)
- info.op_type = I40IW_OP_TYPE_SEND_SOL;
- else
- info.op_type = I40IW_OP_TYPE_SEND;
- } else {
- if (ib_wr->send_flags & IB_SEND_SOLICITED)
- info.op_type = I40IW_OP_TYPE_SEND_SOL_INV;
- else
- info.op_type = I40IW_OP_TYPE_SEND_INV;
- }
-
- if (ib_wr->send_flags & IB_SEND_INLINE) {
- info.op.inline_send.data = (void *)(unsigned long)ib_wr->sg_list[0].addr;
- info.op.inline_send.len = ib_wr->sg_list[0].length;
- ret = ukqp->ops.iw_inline_send(ukqp, &info, ib_wr->ex.invalidate_rkey, false);
- } else {
- info.op.send.num_sges = ib_wr->num_sge;
- info.op.send.sg_list = (struct i40iw_sge *)ib_wr->sg_list;
- ret = ukqp->ops.iw_send(ukqp, &info, ib_wr->ex.invalidate_rkey, false);
- }
-
- if (ret) {
- if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
- err = -ENOMEM;
- else
- err = -EINVAL;
- }
- break;
- case IB_WR_RDMA_WRITE:
- info.op_type = I40IW_OP_TYPE_RDMA_WRITE;
-
- if (ib_wr->send_flags & IB_SEND_INLINE) {
- info.op.inline_rdma_write.data = (void *)(unsigned long)ib_wr->sg_list[0].addr;
- info.op.inline_rdma_write.len = ib_wr->sg_list[0].length;
- info.op.inline_rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
- info.op.inline_rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
- ret = ukqp->ops.iw_inline_rdma_write(ukqp, &info, false);
- } else {
- info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
- info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
- info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
- info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
- ret = ukqp->ops.iw_rdma_write(ukqp, &info, false);
- }
-
- if (ret) {
- if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
- err = -ENOMEM;
- else
- err = -EINVAL;
- }
- break;
- case IB_WR_RDMA_READ_WITH_INV:
- inv_stag = true;
- /* fall-through*/
- case IB_WR_RDMA_READ:
- if (ib_wr->num_sge > I40IW_MAX_SGE_RD) {
- err = -EINVAL;
- break;
- }
- info.op_type = I40IW_OP_TYPE_RDMA_READ;
- info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
- info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey;
- info.op.rdma_read.lo_addr.tag_off = ib_wr->sg_list->addr;
- info.op.rdma_read.lo_addr.stag = ib_wr->sg_list->lkey;
- info.op.rdma_read.lo_addr.len = ib_wr->sg_list->length;
- ret = ukqp->ops.iw_rdma_read(ukqp, &info, inv_stag, false);
- if (ret) {
- if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
- err = -ENOMEM;
- else
- err = -EINVAL;
- }
- break;
- case IB_WR_LOCAL_INV:
- info.op_type = I40IW_OP_TYPE_INV_STAG;
- info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey;
- ret = ukqp->ops.iw_stag_local_invalidate(ukqp, &info, true);
- if (ret)
- err = -ENOMEM;
- break;
- case IB_WR_REG_MR:
- {
- struct i40iw_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr);
- int flags = reg_wr(ib_wr)->access;
- struct i40iw_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc;
- struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev;
- struct i40iw_fast_reg_stag_info info;
-
- memset(&info, 0, sizeof(info));
- info.access_rights = I40IW_ACCESS_FLAGS_LOCALREAD;
- info.access_rights |= i40iw_get_user_access(flags);
- info.stag_key = reg_wr(ib_wr)->key & 0xff;
- info.stag_idx = reg_wr(ib_wr)->key >> 8;
- info.page_size = reg_wr(ib_wr)->mr->page_size;
- info.wr_id = ib_wr->wr_id;
-
- info.addr_type = I40IW_ADDR_TYPE_VA_BASED;
- info.va = (void *)(uintptr_t)iwmr->ibmr.iova;
- info.total_len = iwmr->ibmr.length;
- info.reg_addr_pa = *(u64 *)palloc->level1.addr;
- info.first_pm_pbl_index = palloc->level1.idx;
- info.local_fence = ib_wr->send_flags & IB_SEND_FENCE;
- info.signaled = ib_wr->send_flags & IB_SEND_SIGNALED;
-
- if (iwmr->npages > I40IW_MIN_PAGES_PER_FMR)
- info.chunk_size = 1;
-
- ret = dev->iw_priv_qp_ops->iw_mr_fast_register(&iwqp->sc_qp, &info, true);
- if (ret)
- err = -ENOMEM;
- break;
- }
- default:
- err = -EINVAL;
- i40iw_pr_err(" upost_send bad opcode = 0x%x\n",
- ib_wr->opcode);
- break;
- }
-
- if (err)
- break;
- ib_wr = ib_wr->next;
- }
-
-out:
- if (err)
- *bad_wr = ib_wr;
- else
- ukqp->ops.iw_qp_post_wr(ukqp);
- spin_unlock_irqrestore(&iwqp->lock, flags);
-
- return err;
-}
-
-/**
- * i40iw_post_recv - post receive wr for kernel application
- * @ibqp: ib qp pointer
- * @ib_wr: work request for receive
- * @bad_wr: bad wr caused an error
- */
-static int i40iw_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *ib_wr,
- const struct ib_recv_wr **bad_wr)
-{
- struct i40iw_qp *iwqp;
- struct i40iw_qp_uk *ukqp;
- struct i40iw_post_rq_info post_recv;
- struct i40iw_sge sg_list[I40IW_MAX_WQ_FRAGMENT_COUNT];
- enum i40iw_status_code ret = 0;
- unsigned long flags;
- int err = 0;
-
- iwqp = (struct i40iw_qp *)ibqp;
- ukqp = &iwqp->sc_qp.qp_uk;
-
- memset(&post_recv, 0, sizeof(post_recv));
- spin_lock_irqsave(&iwqp->lock, flags);
-
- if (iwqp->flush_issued) {
- err = -EINVAL;
- goto out;
- }
-
- while (ib_wr) {
- post_recv.num_sges = ib_wr->num_sge;
- post_recv.wr_id = ib_wr->wr_id;
- i40iw_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
- post_recv.sg_list = sg_list;
- ret = ukqp->ops.iw_post_receive(ukqp, &post_recv);
- if (ret) {
- i40iw_pr_err(" post_recv err %d\n", ret);
- if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
- err = -ENOMEM;
- else
- err = -EINVAL;
- *bad_wr = ib_wr;
- goto out;
- }
- ib_wr = ib_wr->next;
- }
- out:
- spin_unlock_irqrestore(&iwqp->lock, flags);
- return err;
-}
-
-/**
- * i40iw_poll_cq - poll cq for completion (kernel apps)
- * @ibcq: cq to poll
- * @num_entries: number of entries to poll
- * @entry: wr of entry completed
- */
-static int i40iw_poll_cq(struct ib_cq *ibcq,
- int num_entries,
- struct ib_wc *entry)
-{
- struct i40iw_cq *iwcq;
- int cqe_count = 0;
- struct i40iw_cq_poll_info cq_poll_info;
- enum i40iw_status_code ret;
- struct i40iw_cq_uk *ukcq;
- struct i40iw_sc_qp *qp;
- struct i40iw_qp *iwqp;
- unsigned long flags;
-
- iwcq = (struct i40iw_cq *)ibcq;
- ukcq = &iwcq->sc_cq.cq_uk;
-
- spin_lock_irqsave(&iwcq->lock, flags);
- while (cqe_count < num_entries) {
- ret = ukcq->ops.iw_cq_poll_completion(ukcq, &cq_poll_info);
- if (ret == I40IW_ERR_QUEUE_EMPTY) {
- break;
- } else if (ret == I40IW_ERR_QUEUE_DESTROYED) {
- continue;
- } else if (ret) {
- if (!cqe_count)
- cqe_count = -1;
- break;
- }
- entry->wc_flags = 0;
- entry->wr_id = cq_poll_info.wr_id;
- if (cq_poll_info.error) {
- entry->status = IB_WC_WR_FLUSH_ERR;
- entry->vendor_err = cq_poll_info.major_err << 16 | cq_poll_info.minor_err;
- } else {
- entry->status = IB_WC_SUCCESS;
- }
-
- switch (cq_poll_info.op_type) {
- case I40IW_OP_TYPE_RDMA_WRITE:
- entry->opcode = IB_WC_RDMA_WRITE;
- break;
- case I40IW_OP_TYPE_RDMA_READ_INV_STAG:
- case I40IW_OP_TYPE_RDMA_READ:
- entry->opcode = IB_WC_RDMA_READ;
- break;
- case I40IW_OP_TYPE_SEND_SOL:
- case I40IW_OP_TYPE_SEND_SOL_INV:
- case I40IW_OP_TYPE_SEND_INV:
- case I40IW_OP_TYPE_SEND:
- entry->opcode = IB_WC_SEND;
- break;
- case I40IW_OP_TYPE_REC:
- entry->opcode = IB_WC_RECV;
- break;
- default:
- entry->opcode = IB_WC_RECV;
- break;
- }
-
- entry->ex.imm_data = 0;
- qp = (struct i40iw_sc_qp *)cq_poll_info.qp_handle;
- entry->qp = (struct ib_qp *)qp->back_qp;
- entry->src_qp = cq_poll_info.qp_id;
- iwqp = (struct i40iw_qp *)qp->back_qp;
- if (iwqp->iwarp_state > I40IW_QP_STATE_RTS) {
- if (!I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring))
- complete(&iwqp->sq_drained);
- if (!I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring))
- complete(&iwqp->rq_drained);
- }
- entry->byte_len = cq_poll_info.bytes_xfered;
- entry++;
- cqe_count++;
- }
- spin_unlock_irqrestore(&iwcq->lock, flags);
- return cqe_count;
-}
-
-/**
- * i40iw_req_notify_cq - arm cq kernel application
- * @ibcq: cq to arm
- * @notify_flags: notofication flags
- */
-static int i40iw_req_notify_cq(struct ib_cq *ibcq,
- enum ib_cq_notify_flags notify_flags)
-{
- struct i40iw_cq *iwcq;
- struct i40iw_cq_uk *ukcq;
- unsigned long flags;
- enum i40iw_completion_notify cq_notify = IW_CQ_COMPL_EVENT;
-
- iwcq = (struct i40iw_cq *)ibcq;
- ukcq = &iwcq->sc_cq.cq_uk;
- if (notify_flags == IB_CQ_SOLICITED)
- cq_notify = IW_CQ_COMPL_SOLICITED;
- spin_lock_irqsave(&iwcq->lock, flags);
- ukcq->ops.iw_cq_request_notification(ukcq, cq_notify);
- spin_unlock_irqrestore(&iwcq->lock, flags);
- return 0;
-}
-
-/**
- * i40iw_port_immutable - return port's immutable data
- * @ibdev: ib dev struct
- * @port_num: port number
- * @immutable: immutable data for the port return
- */
-static int i40iw_port_immutable(struct ib_device *ibdev, u8 port_num,
- struct ib_port_immutable *immutable)
-{
- struct ib_port_attr attr;
- int err;
-
- immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
-
- err = ib_query_port(ibdev, port_num, &attr);
-
- if (err)
- return err;
-
- immutable->pkey_tbl_len = attr.pkey_tbl_len;
- immutable->gid_tbl_len = attr.gid_tbl_len;
-
- return 0;
-}
-
-static const char * const i40iw_hw_stat_names[] = {
- // 32bit names
- [I40IW_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards",
- [I40IW_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts",
- [I40IW_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes",
- [I40IW_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards",
- [I40IW_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts",
- [I40IW_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes",
- [I40IW_HW_STAT_INDEX_TCPRTXSEG] = "tcpRetransSegs",
- [I40IW_HW_STAT_INDEX_TCPRXOPTERR] = "tcpInOptErrors",
- [I40IW_HW_STAT_INDEX_TCPRXPROTOERR] = "tcpInProtoErrors",
- // 64bit names
- [I40IW_HW_STAT_INDEX_IP4RXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip4InOctets",
- [I40IW_HW_STAT_INDEX_IP4RXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip4InPkts",
- [I40IW_HW_STAT_INDEX_IP4RXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip4InReasmRqd",
- [I40IW_HW_STAT_INDEX_IP4RXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip4InMcastPkts",
- [I40IW_HW_STAT_INDEX_IP4TXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip4OutOctets",
- [I40IW_HW_STAT_INDEX_IP4TXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip4OutPkts",
- [I40IW_HW_STAT_INDEX_IP4TXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip4OutSegRqd",
- [I40IW_HW_STAT_INDEX_IP4TXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip4OutMcastPkts",
- [I40IW_HW_STAT_INDEX_IP6RXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip6InOctets",
- [I40IW_HW_STAT_INDEX_IP6RXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip6InPkts",
- [I40IW_HW_STAT_INDEX_IP6RXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip6InReasmRqd",
- [I40IW_HW_STAT_INDEX_IP6RXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip6InMcastPkts",
- [I40IW_HW_STAT_INDEX_IP6TXOCTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip6OutOctets",
- [I40IW_HW_STAT_INDEX_IP6TXPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip6OutPkts",
- [I40IW_HW_STAT_INDEX_IP6TXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip6OutSegRqd",
- [I40IW_HW_STAT_INDEX_IP6TXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] =
- "ip6OutMcastPkts",
- [I40IW_HW_STAT_INDEX_TCPRXSEGS + I40IW_HW_STAT_INDEX_MAX_32] =
- "tcpInSegs",
- [I40IW_HW_STAT_INDEX_TCPTXSEG + I40IW_HW_STAT_INDEX_MAX_32] =
- "tcpOutSegs",
- [I40IW_HW_STAT_INDEX_RDMARXRDS + I40IW_HW_STAT_INDEX_MAX_32] =
- "iwInRdmaReads",
- [I40IW_HW_STAT_INDEX_RDMARXSNDS + I40IW_HW_STAT_INDEX_MAX_32] =
- "iwInRdmaSends",
- [I40IW_HW_STAT_INDEX_RDMARXWRS + I40IW_HW_STAT_INDEX_MAX_32] =
- "iwInRdmaWrites",
- [I40IW_HW_STAT_INDEX_RDMATXRDS + I40IW_HW_STAT_INDEX_MAX_32] =
- "iwOutRdmaReads",
- [I40IW_HW_STAT_INDEX_RDMATXSNDS + I40IW_HW_STAT_INDEX_MAX_32] =
- "iwOutRdmaSends",
- [I40IW_HW_STAT_INDEX_RDMATXWRS + I40IW_HW_STAT_INDEX_MAX_32] =
- "iwOutRdmaWrites",
- [I40IW_HW_STAT_INDEX_RDMAVBND + I40IW_HW_STAT_INDEX_MAX_32] =
- "iwRdmaBnd",
- [I40IW_HW_STAT_INDEX_RDMAVINV + I40IW_HW_STAT_INDEX_MAX_32] =
- "iwRdmaInv"
-};
-
-static void i40iw_get_dev_fw_str(struct ib_device *dev, char *str)
-{
- struct i40iw_device *iwdev = to_iwdev(dev);
-
- snprintf(str, IB_FW_VERSION_NAME_MAX, "%llu.%llu",
- i40iw_fw_major_ver(&iwdev->sc_dev),
- i40iw_fw_minor_ver(&iwdev->sc_dev));
-}
-
-/**
- * i40iw_alloc_hw_stats - Allocate a hw stats structure
- * @ibdev: device pointer from stack
- * @port_num: port number
- */
-static struct rdma_hw_stats *i40iw_alloc_hw_stats(struct ib_device *ibdev,
- u8 port_num)
-{
- struct i40iw_device *iwdev = to_iwdev(ibdev);
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- int num_counters = I40IW_HW_STAT_INDEX_MAX_32 +
- I40IW_HW_STAT_INDEX_MAX_64;
- unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN;
-
- BUILD_BUG_ON(ARRAY_SIZE(i40iw_hw_stat_names) !=
- (I40IW_HW_STAT_INDEX_MAX_32 +
- I40IW_HW_STAT_INDEX_MAX_64));
-
- /*
- * PFs get the default update lifespan, but VFs only update once
- * per second
- */
- if (!dev->is_pf)
- lifespan = 1000;
- return rdma_alloc_hw_stats_struct(i40iw_hw_stat_names, num_counters,
- lifespan);
-}
-
-/**
- * i40iw_get_hw_stats - Populates the rdma_hw_stats structure
- * @ibdev: device pointer from stack
- * @stats: stats pointer from stack
- * @port_num: port number
- * @index: which hw counter the stack is requesting we update
- */
-static int i40iw_get_hw_stats(struct ib_device *ibdev,
- struct rdma_hw_stats *stats,
- u8 port_num, int index)
-{
- struct i40iw_device *iwdev = to_iwdev(ibdev);
- struct i40iw_sc_dev *dev = &iwdev->sc_dev;
- struct i40iw_vsi_pestat *devstat = iwdev->vsi.pestat;
- struct i40iw_dev_hw_stats *hw_stats = &devstat->hw_stats;
-
- if (dev->is_pf) {
- i40iw_hw_stats_read_all(devstat, &devstat->hw_stats);
- } else {
- if (i40iw_vchnl_vf_get_pe_stats(dev, &devstat->hw_stats))
- return -ENOSYS;
- }
-
- memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats));
-
- return stats->num_counters;
-}
-
-/**
- * i40iw_query_gid - Query port GID
- * @ibdev: device pointer from stack
- * @port: port number
- * @index: Entry index
- * @gid: Global ID
- */
-static int i40iw_query_gid(struct ib_device *ibdev,
- u8 port,
- int index,
- union ib_gid *gid)
-{
- struct i40iw_device *iwdev = to_iwdev(ibdev);
-
- memset(gid->raw, 0, sizeof(gid->raw));
- ether_addr_copy(gid->raw, iwdev->netdev->dev_addr);
- return 0;
-}
-
-/**
- * i40iw_query_pkey - Query partition key
- * @ibdev: device pointer from stack
- * @port: port number
- * @index: index of pkey
- * @pkey: pointer to store the pkey
- */
-static int i40iw_query_pkey(struct ib_device *ibdev,
- u8 port,
- u16 index,
- u16 *pkey)
-{
- *pkey = 0;
- return 0;
-}
-
-static const struct ib_device_ops i40iw_dev_ops = {
- .owner = THIS_MODULE,
- .driver_id = RDMA_DRIVER_I40IW,
- /* NOTE: Older kernels wrongly use 0 for the uverbs_abi_ver */
- .uverbs_abi_ver = I40IW_ABI_VER,
-
- .alloc_hw_stats = i40iw_alloc_hw_stats,
- .alloc_mr = i40iw_alloc_mr,
- .alloc_pd = i40iw_alloc_pd,
- .alloc_ucontext = i40iw_alloc_ucontext,
- .create_cq = i40iw_create_cq,
- .create_qp = i40iw_create_qp,
- .dealloc_pd = i40iw_dealloc_pd,
- .dealloc_ucontext = i40iw_dealloc_ucontext,
- .dereg_mr = i40iw_dereg_mr,
- .destroy_cq = i40iw_destroy_cq,
- .destroy_qp = i40iw_destroy_qp,
- .drain_rq = i40iw_drain_rq,
- .drain_sq = i40iw_drain_sq,
- .get_dev_fw_str = i40iw_get_dev_fw_str,
- .get_dma_mr = i40iw_get_dma_mr,
- .get_hw_stats = i40iw_get_hw_stats,
- .get_port_immutable = i40iw_port_immutable,
- .iw_accept = i40iw_accept,
- .iw_add_ref = i40iw_add_ref,
- .iw_connect = i40iw_connect,
- .iw_create_listen = i40iw_create_listen,
- .iw_destroy_listen = i40iw_destroy_listen,
- .iw_get_qp = i40iw_get_qp,
- .iw_reject = i40iw_reject,
- .iw_rem_ref = i40iw_rem_ref,
- .map_mr_sg = i40iw_map_mr_sg,
- .mmap = i40iw_mmap,
- .modify_qp = i40iw_modify_qp,
- .poll_cq = i40iw_poll_cq,
- .post_recv = i40iw_post_recv,
- .post_send = i40iw_post_send,
- .query_device = i40iw_query_device,
- .query_gid = i40iw_query_gid,
- .query_pkey = i40iw_query_pkey,
- .query_port = i40iw_query_port,
- .query_qp = i40iw_query_qp,
- .reg_user_mr = i40iw_reg_user_mr,
- .req_notify_cq = i40iw_req_notify_cq,
- INIT_RDMA_OBJ_SIZE(ib_pd, i40iw_pd, ibpd),
- INIT_RDMA_OBJ_SIZE(ib_cq, i40iw_cq, ibcq),
- INIT_RDMA_OBJ_SIZE(ib_ucontext, i40iw_ucontext, ibucontext),
-};
-
-/**
- * i40iw_init_rdma_device - initialization of iwarp device
- * @iwdev: iwarp device
- */
-static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev)
-{
- struct i40iw_ib_device *iwibdev;
- struct net_device *netdev = iwdev->netdev;
- struct pci_dev *pcidev = (struct pci_dev *)iwdev->hw.dev_context;
-
- iwibdev = ib_alloc_device(i40iw_ib_device, ibdev);
- if (!iwibdev) {
- i40iw_pr_err("iwdev == NULL\n");
- return NULL;
- }
- iwdev->iwibdev = iwibdev;
- iwibdev->iwdev = iwdev;
-
- iwibdev->ibdev.node_type = RDMA_NODE_RNIC;
- ether_addr_copy((u8 *)&iwibdev->ibdev.node_guid, netdev->dev_addr);
-
- iwibdev->ibdev.uverbs_cmd_mask =
- (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
- (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
- (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
- (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
- (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
- (1ull << IB_USER_VERBS_CMD_REG_MR) |
- (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
- (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
- (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
- (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
- (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
- (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
- (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
- (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
- (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
- (1ull << IB_USER_VERBS_CMD_POST_RECV) |
- (1ull << IB_USER_VERBS_CMD_POST_SEND);
- iwibdev->ibdev.phys_port_cnt = 1;
- iwibdev->ibdev.num_comp_vectors = iwdev->ceqs_count;
- iwibdev->ibdev.dev.parent = &pcidev->dev;
- memcpy(iwibdev->ibdev.iw_ifname, netdev->name,
- sizeof(iwibdev->ibdev.iw_ifname));
- ib_set_device_ops(&iwibdev->ibdev, &i40iw_dev_ops);
-
- return iwibdev;
-}
-
-/**
- * i40iw_port_ibevent - indicate port event
- * @iwdev: iwarp device
- */
-void i40iw_port_ibevent(struct i40iw_device *iwdev)
-{
- struct i40iw_ib_device *iwibdev = iwdev->iwibdev;
- struct ib_event event;
-
- event.device = &iwibdev->ibdev;
- event.element.port_num = 1;
- event.event = iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
- ib_dispatch_event(&event);
-}
-
-/**
- * i40iw_destroy_rdma_device - destroy rdma device and free resources
- * @iwibdev: IB device ptr
- */
-void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev)
-{
- ib_unregister_device(&iwibdev->ibdev);
- wait_event_timeout(iwibdev->iwdev->close_wq,
- !atomic64_read(&iwibdev->iwdev->use_count),
- I40IW_EVENT_TIMEOUT);
- ib_dealloc_device(&iwibdev->ibdev);
-}
-
-/**
- * i40iw_register_rdma_device - register iwarp device to IB
- * @iwdev: iwarp device
- */
-int i40iw_register_rdma_device(struct i40iw_device *iwdev)
-{
- int ret;
- struct i40iw_ib_device *iwibdev;
-
- iwdev->iwibdev = i40iw_init_rdma_device(iwdev);
- if (!iwdev->iwibdev)
- return -ENOMEM;
- iwibdev = iwdev->iwibdev;
- rdma_set_device_sysfs_group(&iwibdev->ibdev, &i40iw_attr_group);
- ret = ib_device_set_netdev(&iwibdev->ibdev, iwdev->netdev, 1);
- if (ret)
- goto error;
-
- ret = ib_register_device(&iwibdev->ibdev, "i40iw%d");
- if (ret)
- goto error;
-
- return 0;
-error:
- ib_dealloc_device(&iwdev->iwibdev->ibdev);
- return ret;
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.h b/drivers/infiniband/hw/i40iw/i40iw_verbs.h
deleted file mode 100644
index 3a413752ccc3..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.h
+++ /dev/null
@@ -1,179 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_VERBS_H
-#define I40IW_VERBS_H
-
-struct i40iw_ucontext {
- struct ib_ucontext ibucontext;
- struct i40iw_device *iwdev;
- struct list_head cq_reg_mem_list;
- spinlock_t cq_reg_mem_list_lock; /* memory list for cq's */
- struct list_head qp_reg_mem_list;
- spinlock_t qp_reg_mem_list_lock; /* memory list for qp's */
- int abi_ver;
-};
-
-struct i40iw_pd {
- struct ib_pd ibpd;
- struct i40iw_sc_pd sc_pd;
- atomic_t usecount;
-};
-
-struct i40iw_hmc_pble {
- union {
- u32 idx;
- dma_addr_t addr;
- };
-};
-
-struct i40iw_cq_mr {
- struct i40iw_hmc_pble cq_pbl;
- dma_addr_t shadow;
-};
-
-struct i40iw_qp_mr {
- struct i40iw_hmc_pble sq_pbl;
- struct i40iw_hmc_pble rq_pbl;
- dma_addr_t shadow;
- struct page *sq_page;
-};
-
-struct i40iw_pbl {
- struct list_head list;
- union {
- struct i40iw_qp_mr qp_mr;
- struct i40iw_cq_mr cq_mr;
- };
-
- bool pbl_allocated;
- bool on_list;
- u64 user_base;
- struct i40iw_pble_alloc pble_alloc;
- struct i40iw_mr *iwmr;
-};
-
-#define MAX_SAVE_PAGE_ADDRS 4
-struct i40iw_mr {
- union {
- struct ib_mr ibmr;
- struct ib_mw ibmw;
- struct ib_fmr ibfmr;
- };
- struct ib_umem *region;
- u16 type;
- u32 page_cnt;
- u64 page_size;
- u32 npages;
- u32 stag;
- u64 length;
- u64 pgaddrmem[MAX_SAVE_PAGE_ADDRS];
- struct i40iw_pbl iwpbl;
-};
-
-struct i40iw_cq {
- struct ib_cq ibcq;
- struct i40iw_sc_cq sc_cq;
- u16 cq_head;
- u16 cq_size;
- u16 cq_number;
- bool user_mode;
- u32 polled_completions;
- u32 cq_mem_size;
- struct i40iw_dma_mem kmem;
- spinlock_t lock; /* for poll cq */
- struct i40iw_pbl *iwpbl;
-};
-
-struct disconn_work {
- struct work_struct work;
- struct i40iw_qp *iwqp;
-};
-
-struct iw_cm_id;
-struct ietf_mpa_frame;
-struct i40iw_ud_file;
-
-struct i40iw_qp_kmode {
- struct i40iw_dma_mem dma_mem;
- u64 *wrid_mem;
-};
-
-struct i40iw_qp {
- struct ib_qp ibqp;
- struct i40iw_sc_qp sc_qp;
- struct i40iw_device *iwdev;
- struct i40iw_cq *iwscq;
- struct i40iw_cq *iwrcq;
- struct i40iw_pd *iwpd;
- struct i40iw_qp_host_ctx_info ctx_info;
- struct i40iwarp_offload_info iwarp_info;
- void *allocated_buffer;
- atomic_t refcount;
- struct iw_cm_id *cm_id;
- void *cm_node;
- struct ib_mr *lsmm_mr;
- struct work_struct work;
- enum ib_qp_state ibqp_state;
- u32 iwarp_state;
- u32 qp_mem_size;
- u32 last_aeq;
- atomic_t close_timer_started;
- spinlock_t lock; /* for post work requests */
- struct i40iw_qp_context *iwqp_context;
- void *pbl_vbase;
- dma_addr_t pbl_pbase;
- struct page *page;
- u8 active_conn:1;
- u8 user_mode:1;
- u8 hte_added:1;
- u8 flush_issued:1;
- u8 destroyed:1;
- u8 sig_all:1;
- u8 pau_mode:1;
- u8 rsvd:1;
- u16 term_sq_flush_code;
- u16 term_rq_flush_code;
- u8 hw_iwarp_state;
- u8 hw_tcp_state;
- struct i40iw_qp_kmode kqp;
- struct i40iw_dma_mem host_ctx;
- struct timer_list terminate_timer;
- struct i40iw_pbl iwpbl;
- struct i40iw_dma_mem q2_ctx_mem;
- struct i40iw_dma_mem ietf_mem;
- struct completion sq_drained;
- struct completion rq_drained;
-};
-#endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_vf.c b/drivers/infiniband/hw/i40iw/i40iw_vf.c
deleted file mode 100644
index e33d4810965c..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_vf.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include "i40iw_osdep.h"
-#include "i40iw_register.h"
-#include "i40iw_status.h"
-#include "i40iw_hmc.h"
-#include "i40iw_d.h"
-#include "i40iw_type.h"
-#include "i40iw_p.h"
-#include "i40iw_vf.h"
-
-/**
- * i40iw_manage_vf_pble_bp - manage vf pble
- * @cqp: cqp for cqp' sq wqe
- * @info: pble info
- * @scratch: pointer for completion
- * @post_sq: to post and ring
- */
-enum i40iw_status_code i40iw_manage_vf_pble_bp(struct i40iw_sc_cqp *cqp,
- struct i40iw_manage_vf_pble_info *info,
- u64 scratch,
- bool post_sq)
-{
- u64 *wqe;
- u64 temp, header, pd_pl_pba = 0;
-
- wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
- if (!wqe)
- return I40IW_ERR_RING_FULL;
-
- temp = LS_64(info->pd_entry_cnt, I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT) |
- LS_64(info->first_pd_index, I40IW_CQPSQ_MVPBP_FIRST_PD_INX) |
- LS_64(info->sd_index, I40IW_CQPSQ_MVPBP_SD_INX);
- set_64bit_val(wqe, 16, temp);
-
- header = LS_64((info->inv_pd_ent ? 1 : 0), I40IW_CQPSQ_MVPBP_INV_PD_ENT) |
- LS_64(I40IW_CQP_OP_MANAGE_VF_PBLE_BP, I40IW_CQPSQ_OPCODE) |
- LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
- set_64bit_val(wqe, 24, header);
-
- pd_pl_pba = LS_64(info->pd_pl_pba >> 3, I40IW_CQPSQ_MVPBP_PD_PLPBA);
- set_64bit_val(wqe, 32, pd_pl_pba);
-
- i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE VF_PBLE_BP WQE", wqe, I40IW_CQP_WQE_SIZE * 8);
-
- if (post_sq)
- i40iw_sc_cqp_post_sq(cqp);
- return 0;
-}
-
-const struct i40iw_vf_cqp_ops iw_vf_cqp_ops = {
- i40iw_manage_vf_pble_bp
-};
diff --git a/drivers/infiniband/hw/i40iw/i40iw_vf.h b/drivers/infiniband/hw/i40iw/i40iw_vf.h
deleted file mode 100644
index 4359559ece9c..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_vf.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_VF_H
-#define I40IW_VF_H
-
-struct i40iw_sc_cqp;
-
-struct i40iw_manage_vf_pble_info {
- u32 sd_index;
- u16 first_pd_index;
- u16 pd_entry_cnt;
- u8 inv_pd_ent;
- u64 pd_pl_pba;
-};
-
-struct i40iw_vf_cqp_ops {
- enum i40iw_status_code (*manage_vf_pble_bp)(struct i40iw_sc_cqp *,
- struct i40iw_manage_vf_pble_info *,
- u64,
- bool);
-};
-
-enum i40iw_status_code i40iw_manage_vf_pble_bp(struct i40iw_sc_cqp *cqp,
- struct i40iw_manage_vf_pble_info *info,
- u64 scratch,
- bool post_sq);
-
-extern const struct i40iw_vf_cqp_ops iw_vf_cqp_ops;
-
-#endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c
deleted file mode 100644
index 48fd327f876b..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c
+++ /dev/null
@@ -1,756 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include "i40iw_osdep.h"
-#include "i40iw_register.h"
-#include "i40iw_status.h"
-#include "i40iw_hmc.h"
-#include "i40iw_d.h"
-#include "i40iw_type.h"
-#include "i40iw_p.h"
-#include "i40iw_virtchnl.h"
-
-/**
- * vchnl_vf_send_get_ver_req - Request Channel version
- * @dev: IWARP device pointer
- * @vchnl_req: Virtual channel message request pointer
- */
-static enum i40iw_status_code vchnl_vf_send_get_ver_req(struct i40iw_sc_dev *dev,
- struct i40iw_virtchnl_req *vchnl_req)
-{
- enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY;
- struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req->vchnl_msg;
-
- if (!dev->vchnl_up)
- return ret_code;
-
- memset(vchnl_msg, 0, sizeof(*vchnl_msg));
- vchnl_msg->iw_chnl_op_ctx = (uintptr_t)vchnl_req;
- vchnl_msg->iw_chnl_buf_len = sizeof(*vchnl_msg);
- vchnl_msg->iw_op_code = I40IW_VCHNL_OP_GET_VER;
- vchnl_msg->iw_op_ver = I40IW_VCHNL_OP_GET_VER_V0;
- ret_code = dev->vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg->iw_chnl_buf_len);
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: virt channel send failed 0x%x\n", __func__, ret_code);
- return ret_code;
-}
-
-/**
- * vchnl_vf_send_get_hmc_fcn_req - Request HMC Function from VF
- * @dev: IWARP device pointer
- * @vchnl_req: Virtual channel message request pointer
- */
-static enum i40iw_status_code vchnl_vf_send_get_hmc_fcn_req(struct i40iw_sc_dev *dev,
- struct i40iw_virtchnl_req *vchnl_req)
-{
- enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY;
- struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req->vchnl_msg;
-
- if (!dev->vchnl_up)
- return ret_code;
-
- memset(vchnl_msg, 0, sizeof(*vchnl_msg));
- vchnl_msg->iw_chnl_op_ctx = (uintptr_t)vchnl_req;
- vchnl_msg->iw_chnl_buf_len = sizeof(*vchnl_msg);
- vchnl_msg->iw_op_code = I40IW_VCHNL_OP_GET_HMC_FCN;
- vchnl_msg->iw_op_ver = I40IW_VCHNL_OP_GET_HMC_FCN_V0;
- ret_code = dev->vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg->iw_chnl_buf_len);
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: virt channel send failed 0x%x\n", __func__, ret_code);
- return ret_code;
-}
-
-/**
- * vchnl_vf_send_get_pe_stats_req - Request PE stats from VF
- * @dev: IWARP device pointer
- * @vchnl_req: Virtual channel message request pointer
- */
-static enum i40iw_status_code vchnl_vf_send_get_pe_stats_req(struct i40iw_sc_dev *dev,
- struct i40iw_virtchnl_req *vchnl_req)
-{
- enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY;
- struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req->vchnl_msg;
-
- if (!dev->vchnl_up)
- return ret_code;
-
- memset(vchnl_msg, 0, sizeof(*vchnl_msg));
- vchnl_msg->iw_chnl_op_ctx = (uintptr_t)vchnl_req;
- vchnl_msg->iw_chnl_buf_len = sizeof(*vchnl_msg) + sizeof(struct i40iw_dev_hw_stats) - 1;
- vchnl_msg->iw_op_code = I40IW_VCHNL_OP_GET_STATS;
- vchnl_msg->iw_op_ver = I40IW_VCHNL_OP_GET_STATS_V0;
- ret_code = dev->vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg->iw_chnl_buf_len);
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: virt channel send failed 0x%x\n", __func__, ret_code);
- return ret_code;
-}
-
-/**
- * vchnl_vf_send_add_hmc_objs_req - Add HMC objects
- * @dev: IWARP device pointer
- * @vchnl_req: Virtual channel message request pointer
- */
-static enum i40iw_status_code vchnl_vf_send_add_hmc_objs_req(struct i40iw_sc_dev *dev,
- struct i40iw_virtchnl_req *vchnl_req,
- enum i40iw_hmc_rsrc_type rsrc_type,
- u32 start_index,
- u32 rsrc_count)
-{
- enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY;
- struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req->vchnl_msg;
- struct i40iw_virtchnl_hmc_obj_range *add_hmc_obj;
-
- if (!dev->vchnl_up)
- return ret_code;
-
- add_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg->iw_chnl_buf;
- memset(vchnl_msg, 0, sizeof(*vchnl_msg));
- memset(add_hmc_obj, 0, sizeof(*add_hmc_obj));
- vchnl_msg->iw_chnl_op_ctx = (uintptr_t)vchnl_req;
- vchnl_msg->iw_chnl_buf_len = sizeof(*vchnl_msg) + sizeof(struct i40iw_virtchnl_hmc_obj_range) - 1;
- vchnl_msg->iw_op_code = I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE;
- vchnl_msg->iw_op_ver = I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE_V0;
- add_hmc_obj->obj_type = (u16)rsrc_type;
- add_hmc_obj->start_index = start_index;
- add_hmc_obj->obj_count = rsrc_count;
- ret_code = dev->vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg->iw_chnl_buf_len);
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: virt channel send failed 0x%x\n", __func__, ret_code);
- return ret_code;
-}
-
-/**
- * vchnl_vf_send_del_hmc_objs_req - del HMC objects
- * @dev: IWARP device pointer
- * @vchnl_req: Virtual channel message request pointer
- * @ rsrc_type - resource type to delete
- * @ start_index - starting index for resource
- * @ rsrc_count - number of resource type to delete
- */
-static enum i40iw_status_code vchnl_vf_send_del_hmc_objs_req(struct i40iw_sc_dev *dev,
- struct i40iw_virtchnl_req *vchnl_req,
- enum i40iw_hmc_rsrc_type rsrc_type,
- u32 start_index,
- u32 rsrc_count)
-{
- enum i40iw_status_code ret_code = I40IW_ERR_NOT_READY;
- struct i40iw_virtchnl_op_buf *vchnl_msg = vchnl_req->vchnl_msg;
- struct i40iw_virtchnl_hmc_obj_range *add_hmc_obj;
-
- if (!dev->vchnl_up)
- return ret_code;
-
- add_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg->iw_chnl_buf;
- memset(vchnl_msg, 0, sizeof(*vchnl_msg));
- memset(add_hmc_obj, 0, sizeof(*add_hmc_obj));
- vchnl_msg->iw_chnl_op_ctx = (uintptr_t)vchnl_req;
- vchnl_msg->iw_chnl_buf_len = sizeof(*vchnl_msg) + sizeof(struct i40iw_virtchnl_hmc_obj_range) - 1;
- vchnl_msg->iw_op_code = I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE;
- vchnl_msg->iw_op_ver = I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE_V0;
- add_hmc_obj->obj_type = (u16)rsrc_type;
- add_hmc_obj->start_index = start_index;
- add_hmc_obj->obj_count = rsrc_count;
- ret_code = dev->vchnl_if.vchnl_send(dev, 0, (u8 *)vchnl_msg, vchnl_msg->iw_chnl_buf_len);
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: virt channel send failed 0x%x\n", __func__, ret_code);
- return ret_code;
-}
-
-/**
- * vchnl_pf_send_get_ver_resp - Send channel version to VF
- * @dev: IWARP device pointer
- * @vf_id: Virtual function ID associated with the message
- * @vchnl_msg: Virtual channel message buffer pointer
- */
-static void vchnl_pf_send_get_ver_resp(struct i40iw_sc_dev *dev,
- u32 vf_id,
- struct i40iw_virtchnl_op_buf *vchnl_msg)
-{
- enum i40iw_status_code ret_code;
- u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf) + sizeof(u32) - 1];
- struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer;
-
- memset(resp_buffer, 0, sizeof(*resp_buffer));
- vchnl_msg_resp->iw_chnl_op_ctx = vchnl_msg->iw_chnl_op_ctx;
- vchnl_msg_resp->iw_chnl_buf_len = sizeof(resp_buffer);
- vchnl_msg_resp->iw_op_ret_code = I40IW_SUCCESS;
- *((u32 *)vchnl_msg_resp->iw_chnl_buf) = I40IW_VCHNL_CHNL_VER_V0;
- ret_code = dev->vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer));
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: virt channel send failed 0x%x\n", __func__, ret_code);
-}
-
-/**
- * vchnl_pf_send_get_hmc_fcn_resp - Send HMC Function to VF
- * @dev: IWARP device pointer
- * @vf_id: Virtual function ID associated with the message
- * @vchnl_msg: Virtual channel message buffer pointer
- */
-static void vchnl_pf_send_get_hmc_fcn_resp(struct i40iw_sc_dev *dev,
- u32 vf_id,
- struct i40iw_virtchnl_op_buf *vchnl_msg,
- u16 hmc_fcn)
-{
- enum i40iw_status_code ret_code;
- u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf) + sizeof(u16) - 1];
- struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer;
-
- memset(resp_buffer, 0, sizeof(*resp_buffer));
- vchnl_msg_resp->iw_chnl_op_ctx = vchnl_msg->iw_chnl_op_ctx;
- vchnl_msg_resp->iw_chnl_buf_len = sizeof(resp_buffer);
- vchnl_msg_resp->iw_op_ret_code = I40IW_SUCCESS;
- *((u16 *)vchnl_msg_resp->iw_chnl_buf) = hmc_fcn;
- ret_code = dev->vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer));
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: virt channel send failed 0x%x\n", __func__, ret_code);
-}
-
-/**
- * vchnl_pf_send_get_pe_stats_resp - Send PE Stats to VF
- * @dev: IWARP device pointer
- * @vf_id: Virtual function ID associated with the message
- * @vchnl_msg: Virtual channel message buffer pointer
- * @hw_stats: HW Stats struct
- */
-
-static void vchnl_pf_send_get_pe_stats_resp(struct i40iw_sc_dev *dev,
- u32 vf_id,
- struct i40iw_virtchnl_op_buf *vchnl_msg,
- struct i40iw_dev_hw_stats *hw_stats)
-{
- enum i40iw_status_code ret_code;
- u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf) + sizeof(struct i40iw_dev_hw_stats) - 1];
- struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer;
-
- memset(resp_buffer, 0, sizeof(*resp_buffer));
- vchnl_msg_resp->iw_chnl_op_ctx = vchnl_msg->iw_chnl_op_ctx;
- vchnl_msg_resp->iw_chnl_buf_len = sizeof(resp_buffer);
- vchnl_msg_resp->iw_op_ret_code = I40IW_SUCCESS;
- *((struct i40iw_dev_hw_stats *)vchnl_msg_resp->iw_chnl_buf) = *hw_stats;
- ret_code = dev->vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer));
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: virt channel send failed 0x%x\n", __func__, ret_code);
-}
-
-/**
- * vchnl_pf_send_error_resp - Send an error response to VF
- * @dev: IWARP device pointer
- * @vf_id: Virtual function ID associated with the message
- * @vchnl_msg: Virtual channel message buffer pointer
- */
-static void vchnl_pf_send_error_resp(struct i40iw_sc_dev *dev, u32 vf_id,
- struct i40iw_virtchnl_op_buf *vchnl_msg,
- u16 op_ret_code)
-{
- enum i40iw_status_code ret_code;
- u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf)];
- struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)resp_buffer;
-
- memset(resp_buffer, 0, sizeof(resp_buffer));
- vchnl_msg_resp->iw_chnl_op_ctx = vchnl_msg->iw_chnl_op_ctx;
- vchnl_msg_resp->iw_chnl_buf_len = sizeof(resp_buffer);
- vchnl_msg_resp->iw_op_ret_code = (u16)op_ret_code;
- ret_code = dev->vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer));
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: virt channel send failed 0x%x\n", __func__, ret_code);
-}
-
-/**
- * pf_cqp_get_hmc_fcn_callback - Callback for Get HMC Fcn
- * @cqp_req_param: CQP Request param value
- * @not_used: unused CQP callback parameter
- */
-static void pf_cqp_get_hmc_fcn_callback(struct i40iw_sc_dev *dev, void *callback_param,
- struct i40iw_ccq_cqe_info *cqe_info)
-{
- struct i40iw_vfdev *vf_dev = callback_param;
- struct i40iw_virt_mem vf_dev_mem;
-
- if (cqe_info->error) {
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "CQP Completion Error on Get HMC Function. Maj = 0x%04x, Minor = 0x%04x\n",
- cqe_info->maj_err_code, cqe_info->min_err_code);
- dev->vf_dev[vf_dev->iw_vf_idx] = NULL;
- vchnl_pf_send_error_resp(dev, vf_dev->vf_id, &vf_dev->vf_msg_buffer.vchnl_msg,
- (u16)I40IW_ERR_CQP_COMPL_ERROR);
- vf_dev_mem.va = vf_dev;
- vf_dev_mem.size = sizeof(*vf_dev);
- i40iw_free_virt_mem(dev->hw, &vf_dev_mem);
- } else {
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "CQP Completion Operation Return information = 0x%08x\n",
- cqe_info->op_ret_val);
- vf_dev->pmf_index = (u16)cqe_info->op_ret_val;
- vf_dev->msg_count--;
- vchnl_pf_send_get_hmc_fcn_resp(dev,
- vf_dev->vf_id,
- &vf_dev->vf_msg_buffer.vchnl_msg,
- vf_dev->pmf_index);
- }
-}
-
-/**
- * pf_add_hmc_obj - Callback for Add HMC Object
- * @vf_dev: pointer to the VF Device
- */
-static void pf_add_hmc_obj_callback(void *work_vf_dev)
-{
- struct i40iw_vfdev *vf_dev = (struct i40iw_vfdev *)work_vf_dev;
- struct i40iw_hmc_info *hmc_info = &vf_dev->hmc_info;
- struct i40iw_virtchnl_op_buf *vchnl_msg = &vf_dev->vf_msg_buffer.vchnl_msg;
- struct i40iw_hmc_create_obj_info info;
- struct i40iw_virtchnl_hmc_obj_range *add_hmc_obj;
- enum i40iw_status_code ret_code;
-
- if (!vf_dev->pf_hmc_initialized) {
- ret_code = i40iw_pf_init_vfhmc(vf_dev->pf_dev, (u8)vf_dev->pmf_index, NULL);
- if (ret_code)
- goto add_out;
- vf_dev->pf_hmc_initialized = true;
- }
-
- add_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg->iw_chnl_buf;
-
- memset(&info, 0, sizeof(info));
- info.hmc_info = hmc_info;
- info.is_pf = false;
- info.rsrc_type = (u32)add_hmc_obj->obj_type;
- info.entry_type = (info.rsrc_type == I40IW_HMC_IW_PBLE) ? I40IW_SD_TYPE_PAGED : I40IW_SD_TYPE_DIRECT;
- info.start_idx = add_hmc_obj->start_index;
- info.count = add_hmc_obj->obj_count;
- i40iw_debug(vf_dev->pf_dev, I40IW_DEBUG_VIRT,
- "I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE. Add %u type %u objects\n",
- info.count, info.rsrc_type);
- ret_code = i40iw_sc_create_hmc_obj(vf_dev->pf_dev, &info);
- if (!ret_code)
- vf_dev->hmc_info.hmc_obj[add_hmc_obj->obj_type].cnt = add_hmc_obj->obj_count;
-add_out:
- vf_dev->msg_count--;
- vchnl_pf_send_error_resp(vf_dev->pf_dev, vf_dev->vf_id, vchnl_msg, (u16)ret_code);
-}
-
-/**
- * pf_del_hmc_obj_callback - Callback for delete HMC Object
- * @work_vf_dev: pointer to the VF Device
- */
-static void pf_del_hmc_obj_callback(void *work_vf_dev)
-{
- struct i40iw_vfdev *vf_dev = (struct i40iw_vfdev *)work_vf_dev;
- struct i40iw_hmc_info *hmc_info = &vf_dev->hmc_info;
- struct i40iw_virtchnl_op_buf *vchnl_msg = &vf_dev->vf_msg_buffer.vchnl_msg;
- struct i40iw_hmc_del_obj_info info;
- struct i40iw_virtchnl_hmc_obj_range *del_hmc_obj;
- enum i40iw_status_code ret_code = I40IW_SUCCESS;
-
- if (!vf_dev->pf_hmc_initialized)
- goto del_out;
-
- del_hmc_obj = (struct i40iw_virtchnl_hmc_obj_range *)vchnl_msg->iw_chnl_buf;
-
- memset(&info, 0, sizeof(info));
- info.hmc_info = hmc_info;
- info.is_pf = false;
- info.rsrc_type = (u32)del_hmc_obj->obj_type;
- info.start_idx = del_hmc_obj->start_index;
- info.count = del_hmc_obj->obj_count;
- i40iw_debug(vf_dev->pf_dev, I40IW_DEBUG_VIRT,
- "I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE. Delete %u type %u objects\n",
- info.count, info.rsrc_type);
- ret_code = i40iw_sc_del_hmc_obj(vf_dev->pf_dev, &info, false);
-del_out:
- vf_dev->msg_count--;
- vchnl_pf_send_error_resp(vf_dev->pf_dev, vf_dev->vf_id, vchnl_msg, (u16)ret_code);
-}
-
-/**
- * i40iw_vf_init_pestat - Initialize stats for VF
- * @devL pointer to the VF Device
- * @stats: Statistics structure pointer
- * @index: Stats index
- */
-static void i40iw_vf_init_pestat(struct i40iw_sc_dev *dev, struct i40iw_vsi_pestat *stats, u16 index)
-{
- stats->hw = dev->hw;
- i40iw_hw_stats_init(stats, (u8)index, false);
- spin_lock_init(&stats->lock);
-}
-
-/**
- * i40iw_vchnl_recv_pf - Receive PF virtual channel messages
- * @dev: IWARP device pointer
- * @vf_id: Virtual function ID associated with the message
- * @msg: Virtual channel message buffer pointer
- * @len: Length of the virtual channels message
- */
-enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev,
- u32 vf_id,
- u8 *msg,
- u16 len)
-{
- struct i40iw_virtchnl_op_buf *vchnl_msg = (struct i40iw_virtchnl_op_buf *)msg;
- struct i40iw_vfdev *vf_dev = NULL;
- struct i40iw_hmc_fcn_info hmc_fcn_info;
- u16 iw_vf_idx;
- u16 first_avail_iw_vf = I40IW_MAX_PE_ENABLED_VF_COUNT;
- struct i40iw_virt_mem vf_dev_mem;
- struct i40iw_virtchnl_work_info work_info;
- struct i40iw_vsi_pestat *stats;
- enum i40iw_status_code ret_code;
-
- if (!dev || !msg || !len)
- return I40IW_ERR_PARAM;
-
- if (!dev->vchnl_up)
- return I40IW_ERR_NOT_READY;
- if (vchnl_msg->iw_op_code == I40IW_VCHNL_OP_GET_VER) {
- vchnl_pf_send_get_ver_resp(dev, vf_id, vchnl_msg);
- return I40IW_SUCCESS;
- }
- for (iw_vf_idx = 0; iw_vf_idx < I40IW_MAX_PE_ENABLED_VF_COUNT; iw_vf_idx++) {
- if (!dev->vf_dev[iw_vf_idx]) {
- if (first_avail_iw_vf == I40IW_MAX_PE_ENABLED_VF_COUNT)
- first_avail_iw_vf = iw_vf_idx;
- continue;
- }
- if (dev->vf_dev[iw_vf_idx]->vf_id == vf_id) {
- vf_dev = dev->vf_dev[iw_vf_idx];
- break;
- }
- }
- if (vf_dev) {
- if (!vf_dev->msg_count) {
- vf_dev->msg_count++;
- } else {
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "VF%u already has a channel message in progress.\n",
- vf_id);
- return I40IW_SUCCESS;
- }
- }
- switch (vchnl_msg->iw_op_code) {
- case I40IW_VCHNL_OP_GET_HMC_FCN:
- if (!vf_dev &&
- (first_avail_iw_vf != I40IW_MAX_PE_ENABLED_VF_COUNT)) {
- ret_code = i40iw_allocate_virt_mem(dev->hw, &vf_dev_mem, sizeof(struct i40iw_vfdev) +
- (sizeof(struct i40iw_hmc_obj_info) * I40IW_HMC_IW_MAX));
- if (!ret_code) {
- vf_dev = vf_dev_mem.va;
- vf_dev->stats_initialized = false;
- vf_dev->pf_dev = dev;
- vf_dev->msg_count = 1;
- vf_dev->vf_id = vf_id;
- vf_dev->iw_vf_idx = first_avail_iw_vf;
- vf_dev->pf_hmc_initialized = false;
- vf_dev->hmc_info.hmc_obj = (struct i40iw_hmc_obj_info *)(&vf_dev[1]);
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "vf_dev %p, hmc_info %p, hmc_obj %p\n",
- vf_dev, &vf_dev->hmc_info, vf_dev->hmc_info.hmc_obj);
- dev->vf_dev[first_avail_iw_vf] = vf_dev;
- iw_vf_idx = first_avail_iw_vf;
- } else {
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "VF%u Unable to allocate a VF device structure.\n",
- vf_id);
- vchnl_pf_send_error_resp(dev, vf_id, vchnl_msg, (u16)I40IW_ERR_NO_MEMORY);
- return I40IW_SUCCESS;
- }
- memcpy(&vf_dev->vf_msg_buffer.vchnl_msg, vchnl_msg, len);
- hmc_fcn_info.callback_fcn = pf_cqp_get_hmc_fcn_callback;
- hmc_fcn_info.vf_id = vf_id;
- hmc_fcn_info.iw_vf_idx = vf_dev->iw_vf_idx;
- hmc_fcn_info.cqp_callback_param = vf_dev;
- hmc_fcn_info.free_fcn = false;
- ret_code = i40iw_cqp_manage_hmc_fcn_cmd(dev, &hmc_fcn_info);
- if (ret_code)
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "VF%u error CQP HMC Function operation.\n",
- vf_id);
- i40iw_vf_init_pestat(dev, &vf_dev->pestat, vf_dev->pmf_index);
- vf_dev->stats_initialized = true;
- } else {
- if (vf_dev) {
- vf_dev->msg_count--;
- vchnl_pf_send_get_hmc_fcn_resp(dev, vf_id, vchnl_msg, vf_dev->pmf_index);
- } else {
- vchnl_pf_send_error_resp(dev, vf_id, vchnl_msg,
- (u16)I40IW_ERR_NO_MEMORY);
- }
- }
- break;
- case I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE:
- if (!vf_dev)
- return I40IW_ERR_BAD_PTR;
- work_info.worker_vf_dev = vf_dev;
- work_info.callback_fcn = pf_add_hmc_obj_callback;
- memcpy(&vf_dev->vf_msg_buffer.vchnl_msg, vchnl_msg, len);
- i40iw_cqp_spawn_worker(dev, &work_info, vf_dev->iw_vf_idx);
- break;
- case I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE:
- if (!vf_dev)
- return I40IW_ERR_BAD_PTR;
- work_info.worker_vf_dev = vf_dev;
- work_info.callback_fcn = pf_del_hmc_obj_callback;
- memcpy(&vf_dev->vf_msg_buffer.vchnl_msg, vchnl_msg, len);
- i40iw_cqp_spawn_worker(dev, &work_info, vf_dev->iw_vf_idx);
- break;
- case I40IW_VCHNL_OP_GET_STATS:
- if (!vf_dev)
- return I40IW_ERR_BAD_PTR;
- stats = &vf_dev->pestat;
- i40iw_hw_stats_read_all(stats, &stats->hw_stats);
- vf_dev->msg_count--;
- vchnl_pf_send_get_pe_stats_resp(dev, vf_id, vchnl_msg, &stats->hw_stats);
- break;
- default:
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "40iw_vchnl_recv_pf: Invalid OpCode 0x%x\n",
- vchnl_msg->iw_op_code);
- vchnl_pf_send_error_resp(dev, vf_id,
- vchnl_msg, (u16)I40IW_ERR_NOT_IMPLEMENTED);
- }
- return I40IW_SUCCESS;
-}
-
-/**
- * i40iw_vchnl_recv_vf - Receive VF virtual channel messages
- * @dev: IWARP device pointer
- * @vf_id: Virtual function ID associated with the message
- * @msg: Virtual channel message buffer pointer
- * @len: Length of the virtual channels message
- */
-enum i40iw_status_code i40iw_vchnl_recv_vf(struct i40iw_sc_dev *dev,
- u32 vf_id,
- u8 *msg,
- u16 len)
-{
- struct i40iw_virtchnl_resp_buf *vchnl_msg_resp = (struct i40iw_virtchnl_resp_buf *)msg;
- struct i40iw_virtchnl_req *vchnl_req;
-
- vchnl_req = (struct i40iw_virtchnl_req *)(uintptr_t)vchnl_msg_resp->iw_chnl_op_ctx;
- vchnl_req->ret_code = (enum i40iw_status_code)vchnl_msg_resp->iw_op_ret_code;
- if (len == (sizeof(*vchnl_msg_resp) + vchnl_req->parm_len - 1)) {
- if (vchnl_req->parm_len && vchnl_req->parm)
- memcpy(vchnl_req->parm, vchnl_msg_resp->iw_chnl_buf, vchnl_req->parm_len);
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: Got response, data size %u\n", __func__,
- vchnl_req->parm_len);
- } else {
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s: error length on response, Got %u, expected %u\n", __func__,
- len, (u32)(sizeof(*vchnl_msg_resp) + vchnl_req->parm_len - 1));
- }
-
- return I40IW_SUCCESS;
-}
-
-/**
- * i40iw_vchnl_vf_get_ver - Request Channel version
- * @dev: IWARP device pointer
- * @vchnl_ver: Virtual channel message version pointer
- */
-enum i40iw_status_code i40iw_vchnl_vf_get_ver(struct i40iw_sc_dev *dev,
- u32 *vchnl_ver)
-{
- struct i40iw_virtchnl_req vchnl_req;
- enum i40iw_status_code ret_code;
-
- if (!i40iw_vf_clear_to_send(dev))
- return I40IW_ERR_TIMEOUT;
- memset(&vchnl_req, 0, sizeof(vchnl_req));
- vchnl_req.dev = dev;
- vchnl_req.parm = vchnl_ver;
- vchnl_req.parm_len = sizeof(*vchnl_ver);
- vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg;
-
- ret_code = vchnl_vf_send_get_ver_req(dev, &vchnl_req);
- if (ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s Send message failed 0x%0x\n", __func__, ret_code);
- return ret_code;
- }
- ret_code = i40iw_vf_wait_vchnl_resp(dev);
- if (ret_code)
- return ret_code;
- else
- return vchnl_req.ret_code;
-}
-
-/**
- * i40iw_vchnl_vf_get_hmc_fcn - Request HMC Function
- * @dev: IWARP device pointer
- * @hmc_fcn: HMC function index pointer
- */
-enum i40iw_status_code i40iw_vchnl_vf_get_hmc_fcn(struct i40iw_sc_dev *dev,
- u16 *hmc_fcn)
-{
- struct i40iw_virtchnl_req vchnl_req;
- enum i40iw_status_code ret_code;
-
- if (!i40iw_vf_clear_to_send(dev))
- return I40IW_ERR_TIMEOUT;
- memset(&vchnl_req, 0, sizeof(vchnl_req));
- vchnl_req.dev = dev;
- vchnl_req.parm = hmc_fcn;
- vchnl_req.parm_len = sizeof(*hmc_fcn);
- vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg;
-
- ret_code = vchnl_vf_send_get_hmc_fcn_req(dev, &vchnl_req);
- if (ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s Send message failed 0x%0x\n", __func__, ret_code);
- return ret_code;
- }
- ret_code = i40iw_vf_wait_vchnl_resp(dev);
- if (ret_code)
- return ret_code;
- else
- return vchnl_req.ret_code;
-}
-
-/**
- * i40iw_vchnl_vf_add_hmc_objs - Add HMC Object
- * @dev: IWARP device pointer
- * @rsrc_type: HMC Resource type
- * @start_index: Starting index of the objects to be added
- * @rsrc_count: Number of resources to be added
- */
-enum i40iw_status_code i40iw_vchnl_vf_add_hmc_objs(struct i40iw_sc_dev *dev,
- enum i40iw_hmc_rsrc_type rsrc_type,
- u32 start_index,
- u32 rsrc_count)
-{
- struct i40iw_virtchnl_req vchnl_req;
- enum i40iw_status_code ret_code;
-
- if (!i40iw_vf_clear_to_send(dev))
- return I40IW_ERR_TIMEOUT;
- memset(&vchnl_req, 0, sizeof(vchnl_req));
- vchnl_req.dev = dev;
- vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg;
-
- ret_code = vchnl_vf_send_add_hmc_objs_req(dev,
- &vchnl_req,
- rsrc_type,
- start_index,
- rsrc_count);
- if (ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s Send message failed 0x%0x\n", __func__, ret_code);
- return ret_code;
- }
- ret_code = i40iw_vf_wait_vchnl_resp(dev);
- if (ret_code)
- return ret_code;
- else
- return vchnl_req.ret_code;
-}
-
-/**
- * i40iw_vchnl_vf_del_hmc_obj - del HMC obj
- * @dev: IWARP device pointer
- * @rsrc_type: HMC Resource type
- * @start_index: Starting index of the object to delete
- * @rsrc_count: Number of resources to be delete
- */
-enum i40iw_status_code i40iw_vchnl_vf_del_hmc_obj(struct i40iw_sc_dev *dev,
- enum i40iw_hmc_rsrc_type rsrc_type,
- u32 start_index,
- u32 rsrc_count)
-{
- struct i40iw_virtchnl_req vchnl_req;
- enum i40iw_status_code ret_code;
-
- if (!i40iw_vf_clear_to_send(dev))
- return I40IW_ERR_TIMEOUT;
- memset(&vchnl_req, 0, sizeof(vchnl_req));
- vchnl_req.dev = dev;
- vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg;
-
- ret_code = vchnl_vf_send_del_hmc_objs_req(dev,
- &vchnl_req,
- rsrc_type,
- start_index,
- rsrc_count);
- if (ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s Send message failed 0x%0x\n", __func__, ret_code);
- return ret_code;
- }
- ret_code = i40iw_vf_wait_vchnl_resp(dev);
- if (ret_code)
- return ret_code;
- else
- return vchnl_req.ret_code;
-}
-
-/**
- * i40iw_vchnl_vf_get_pe_stats - Get PE stats
- * @dev: IWARP device pointer
- * @hw_stats: HW stats struct
- */
-enum i40iw_status_code i40iw_vchnl_vf_get_pe_stats(struct i40iw_sc_dev *dev,
- struct i40iw_dev_hw_stats *hw_stats)
-{
- struct i40iw_virtchnl_req vchnl_req;
- enum i40iw_status_code ret_code;
-
- if (!i40iw_vf_clear_to_send(dev))
- return I40IW_ERR_TIMEOUT;
- memset(&vchnl_req, 0, sizeof(vchnl_req));
- vchnl_req.dev = dev;
- vchnl_req.parm = hw_stats;
- vchnl_req.parm_len = sizeof(*hw_stats);
- vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg;
-
- ret_code = vchnl_vf_send_get_pe_stats_req(dev, &vchnl_req);
- if (ret_code) {
- i40iw_debug(dev, I40IW_DEBUG_VIRT,
- "%s Send message failed 0x%0x\n", __func__, ret_code);
- return ret_code;
- }
- ret_code = i40iw_vf_wait_vchnl_resp(dev);
- if (ret_code)
- return ret_code;
- else
- return vchnl_req.ret_code;
-}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.h b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.h
deleted file mode 100644
index 24886ef08293..000000000000
--- a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.h
+++ /dev/null
@@ -1,124 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_VIRTCHNL_H
-#define I40IW_VIRTCHNL_H
-
-#include "i40iw_hmc.h"
-
-#pragma pack(push, 1)
-
-struct i40iw_virtchnl_op_buf {
- u16 iw_op_code;
- u16 iw_op_ver;
- u16 iw_chnl_buf_len;
- u16 rsvd;
- u64 iw_chnl_op_ctx;
- /* Member alignment MUST be maintained above this location */
- u8 iw_chnl_buf[1];
-};
-
-struct i40iw_virtchnl_resp_buf {
- u64 iw_chnl_op_ctx;
- u16 iw_chnl_buf_len;
- s16 iw_op_ret_code;
- /* Member alignment MUST be maintained above this location */
- u16 rsvd[2];
- u8 iw_chnl_buf[1];
-};
-
-enum i40iw_virtchnl_ops {
- I40IW_VCHNL_OP_GET_VER = 0,
- I40IW_VCHNL_OP_GET_HMC_FCN,
- I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE,
- I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE,
- I40IW_VCHNL_OP_GET_STATS
-};
-
-#define I40IW_VCHNL_OP_GET_VER_V0 0
-#define I40IW_VCHNL_OP_GET_HMC_FCN_V0 0
-#define I40IW_VCHNL_OP_ADD_HMC_OBJ_RANGE_V0 0
-#define I40IW_VCHNL_OP_DEL_HMC_OBJ_RANGE_V0 0
-#define I40IW_VCHNL_OP_GET_STATS_V0 0
-#define I40IW_VCHNL_CHNL_VER_V0 0
-
-struct i40iw_dev_hw_stats;
-
-struct i40iw_virtchnl_hmc_obj_range {
- u16 obj_type;
- u16 rsvd;
- u32 start_index;
- u32 obj_count;
-};
-
-enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev,
- u32 vf_id,
- u8 *msg,
- u16 len);
-
-enum i40iw_status_code i40iw_vchnl_recv_vf(struct i40iw_sc_dev *dev,
- u32 vf_id,
- u8 *msg,
- u16 len);
-
-struct i40iw_virtchnl_req {
- struct i40iw_sc_dev *dev;
- struct i40iw_virtchnl_op_buf *vchnl_msg;
- void *parm;
- u32 vf_id;
- u16 parm_len;
- s16 ret_code;
-};
-
-#pragma pack(pop)
-
-enum i40iw_status_code i40iw_vchnl_vf_get_ver(struct i40iw_sc_dev *dev,
- u32 *vchnl_ver);
-
-enum i40iw_status_code i40iw_vchnl_vf_get_hmc_fcn(struct i40iw_sc_dev *dev,
- u16 *hmc_fcn);
-
-enum i40iw_status_code i40iw_vchnl_vf_add_hmc_objs(struct i40iw_sc_dev *dev,
- enum i40iw_hmc_rsrc_type rsrc_type,
- u32 start_index,
- u32 rsrc_count);
-
-enum i40iw_status_code i40iw_vchnl_vf_del_hmc_obj(struct i40iw_sc_dev *dev,
- enum i40iw_hmc_rsrc_type rsrc_type,
- u32 start_index,
- u32 rsrc_count);
-
-enum i40iw_status_code i40iw_vchnl_vf_get_pe_stats(struct i40iw_sc_dev *dev,
- struct i40iw_dev_hw_stats *hw_stats);
-#endif
diff --git a/drivers/infiniband/hw/irdma/Kconfig b/drivers/infiniband/hw/irdma/Kconfig
new file mode 100644
index 000000000000..f1524dd44cec
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/Kconfig
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config INFINIBAND_IRDMA
+ tristate "Intel(R) Ethernet Protocol Driver for RDMA"
+ depends on INET
+ depends on IPV6 || !IPV6
+ depends on PCI
+ select GENERIC_ALLOCATOR
+ select CONFIG_VIRTUAL_BUS
+ help
+ This is an Intel(R) Ethernet Protocol Driver for RDMA driver
+ that support E810 (iWARP/RoCE) and X722 (iWARP) network devices.
diff --git a/drivers/infiniband/hw/irdma/Makefile b/drivers/infiniband/hw/irdma/Makefile
new file mode 100644
index 000000000000..160fafe4ff0c
--- /dev/null
+++ b/drivers/infiniband/hw/irdma/Makefile
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+# Copyright (c) 2019, Intel Corporation.
+
+#
+# Makefile for the Intel(R) Ethernet Connection RDMA Linux Driver
+#
+
+obj-$(CONFIG_INFINIBAND_IRDMA) += irdma.o
+
+irdma-objs := cm.o \
+ ctrl.o \
+ hmc.o \
+ hw.o \
+ i40iw_hw.o \
+ i40iw_if.o \
+ icrdma_hw.o \
+ irdma_if.o \
+ main.o \
+ pble.o \
+ puda.o \
+ trace.o \
+ uda.o \
+ uk.o \
+ utils.o \
+ verbs.o \
+ ws.o \
+
+CFLAGS_trace.o = -I$(src)
diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c
index 46f9d94f834f..c0922cceaf13 100644
--- a/drivers/infiniband/hw/irdma/main.c
+++ b/drivers/infiniband/hw/irdma/main.c
@@ -4,6 +4,7 @@
bool irdma_upload_context;
+MODULE_ALIAS("i40iw");
MODULE_AUTHOR("Intel Corporation, <e1000-rdma@...ts.sourceforge.net>");
MODULE_DESCRIPTION("Intel(R) Ethernet Protocol Driver for RDMA");
MODULE_LICENSE("Dual BSD/GPL");
diff --git a/include/uapi/rdma/i40iw-abi.h b/include/uapi/rdma/i40iw-abi.h
deleted file mode 100644
index 79890baa6fdb..000000000000
--- a/include/uapi/rdma/i40iw-abi.h
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 2006 - 2016 Intel Corporation. All rights reserved.
- * Copyright (c) 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2005 Cisco Systems. All rights reserved.
- * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#ifndef I40IW_ABI_H
-#define I40IW_ABI_H
-
-#include <linux/types.h>
-
-#define I40IW_ABI_VER 5
-
-struct i40iw_alloc_ucontext_req {
- __u32 reserved32;
- __u8 userspace_ver;
- __u8 reserved8[3];
-};
-
-struct i40iw_alloc_ucontext_resp {
- __u32 max_pds; /* maximum pds allowed for this user process */
- __u32 max_qps; /* maximum qps allowed for this user process */
- __u32 wq_size; /* size of the WQs (sq+rq) allocated to the mmaped area */
- __u8 kernel_ver;
- __u8 reserved[3];
-};
-
-struct i40iw_alloc_pd_resp {
- __u32 pd_id;
- __u8 reserved[4];
-};
-
-struct i40iw_create_cq_req {
- __aligned_u64 user_cq_buffer;
- __aligned_u64 user_shadow_area;
-};
-
-struct i40iw_create_qp_req {
- __aligned_u64 user_wqe_buffers;
- __aligned_u64 user_compl_ctx;
-
- /* UDA QP PHB */
- __aligned_u64 user_sq_phb; /* place for VA of the sq phb buff */
- __aligned_u64 user_rq_phb; /* place for VA of the rq phb buff */
-};
-
-enum i40iw_memreg_type {
- IW_MEMREG_TYPE_MEM = 0x0000,
- IW_MEMREG_TYPE_QP = 0x0001,
- IW_MEMREG_TYPE_CQ = 0x0002,
-};
-
-struct i40iw_mem_reg_req {
- __u16 reg_type; /* Memory, QP or CQ */
- __u16 cq_pages;
- __u16 rq_pages;
- __u16 sq_pages;
-};
-
-struct i40iw_create_cq_resp {
- __u32 cq_id;
- __u32 cq_size;
- __u32 mmap_db_index;
- __u32 reserved;
-};
-
-struct i40iw_create_qp_resp {
- __u32 qp_id;
- __u32 actual_sq_size;
- __u32 actual_rq_size;
- __u32 i40iw_drv_opt;
- __u16 push_idx;
- __u8 lsmm;
- __u8 rsvd2;
-};
-
-#endif
--
2.26.2
Powered by blists - more mailing lists