| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <40a01a90-b91f-4526-a404-462de3ffa38a@grimberg.me>
Date: Thu, 7 Mar 2024 11:11:29 +0200
From: Sagi Grimberg <sagi@...mberg.me>
To: Aurelien Aptel <aaptel@...dia.com>, linux-nvme@...ts.infradead.org,
netdev@...r.kernel.org, hch@....de, kbusch@...nel.org, axboe@...com,
chaitanyak@...dia.com, davem@...emloft.net, kuba@...nel.org
Cc: Boris Pismenny <borisp@...dia.com>, aurelien.aptel@...il.com,
smalin@...dia.com, malin1024@...il.com, ogerlitz@...dia.com,
yorayz@...dia.com, galshalom@...dia.com, mgurtovoy@...dia.com
Subject: Re: [PATCH v23 06/20] nvme-tcp: Add DDP data-path
On 28/02/2024 14:57, Aurelien Aptel wrote:
> From: Boris Pismenny <borisp@...dia.com>
>
> Introduce the NVMe-TCP DDP data-path offload.
> Using this interface, the NIC hardware will scatter TCP payload directly
> to the BIO pages according to the command_id in the PDU.
> To maintain the correctness of the network stack, the driver is expected
> to construct SKBs that point to the BIO pages.
>
> The data-path interface contains two routines: setup/teardown.
> The setup provides the mapping from command_id to the request buffers,
> while the teardown removes this mapping.
>
> For efficiency, we introduce an asynchronous nvme completion, which is
> split between NVMe-TCP and the NIC driver as follows:
> NVMe-TCP performs the specific completion, while NIC driver performs the
> generic mq_blk completion.
>
> Signed-off-by: Boris Pismenny <borisp@...dia.com>
> Signed-off-by: Ben Ben-Ishay <benishay@...dia.com>
> Signed-off-by: Or Gerlitz <ogerlitz@...dia.com>
> Signed-off-by: Yoray Zack <yorayz@...dia.com>
> Signed-off-by: Shai Malin <smalin@...dia.com>
> Signed-off-by: Aurelien Aptel <aaptel@...dia.com>
> Reviewed-by: Chaitanya Kulkarni <kch@...dia.com>
> Reviewed-by: Max Gurtovoy <mgurtovoy@...dia.com>
> ---
> drivers/nvme/host/tcp.c | 135 ++++++++++++++++++++++++++++++++++++++--
> 1 file changed, 130 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
> index 86a9ad9f679b..b7cfe14661d6 100644
> --- a/drivers/nvme/host/tcp.c
> +++ b/drivers/nvme/host/tcp.c
> @@ -120,6 +120,10 @@ struct nvme_tcp_request {
> struct llist_node lentry;
> __le32 ddgst;
>
> + /* ddp async completion */
> + __le16 nvme_status;
> + union nvme_result result;
> +
> struct bio *curr_bio;
> struct iov_iter iter;
>
> @@ -127,6 +131,11 @@ struct nvme_tcp_request {
> size_t offset;
> size_t data_sent;
> enum nvme_tcp_send_state state;
> +
> +#ifdef CONFIG_ULP_DDP
> + bool offloaded;
> + struct ulp_ddp_io ddp;
> +#endif
> };
>
> enum nvme_tcp_queue_flags {
> @@ -333,6 +342,25 @@ static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
>
> #ifdef CONFIG_ULP_DDP
>
> +static bool nvme_tcp_is_ddp_offloaded(const struct nvme_tcp_request *req)
> +{
> + return req->offloaded;
> +}
> +
> +static int nvme_tcp_ddp_alloc_sgl(struct nvme_tcp_request *req, struct request *rq)
> +{
> + int ret;
> +
> + req->ddp.sg_table.sgl = req->ddp.first_sgl;
> + ret = sg_alloc_table_chained(&req->ddp.sg_table,
> + blk_rq_nr_phys_segments(rq),
> + req->ddp.sg_table.sgl, SG_CHUNK_SIZE);
> + if (ret)
> + return -ENOMEM;
> + req->ddp.nents = blk_rq_map_sg(rq->q, rq, req->ddp.sg_table.sgl);
> + return 0;
> +}
> +
> static struct net_device *
> nvme_tcp_get_ddp_netdev_with_limits(struct nvme_tcp_ctrl *ctrl)
> {
> @@ -366,10 +394,68 @@ nvme_tcp_get_ddp_netdev_with_limits(struct nvme_tcp_ctrl *ctrl)
> }
>
> static bool nvme_tcp_resync_request(struct sock *sk, u32 seq, u32 flags);
> +static void nvme_tcp_ddp_teardown_done(void *ddp_ctx);
> static const struct ulp_ddp_ulp_ops nvme_tcp_ddp_ulp_ops = {
> .resync_request = nvme_tcp_resync_request,
> + .ddp_teardown_done = nvme_tcp_ddp_teardown_done,
> };
>
> +static void nvme_tcp_teardown_ddp(struct nvme_tcp_queue *queue,
> + struct request *rq)
> +{
> + struct net_device *netdev = queue->ctrl->ddp_netdev;
> + struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
> +
> + ulp_ddp_teardown(netdev, queue->sock->sk, &req->ddp, rq);
> + sg_free_table_chained(&req->ddp.sg_table, SG_CHUNK_SIZE);
> +}
> +
> +static void nvme_tcp_ddp_teardown_done(void *ddp_ctx)
> +{
> + struct request *rq = ddp_ctx;
> + struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
> +
> + if (!nvme_try_complete_req(rq, req->nvme_status, req->result))
> + nvme_complete_rq(rq);
> +}
> +
> +static void nvme_tcp_setup_ddp(struct nvme_tcp_queue *queue,
> + struct request *rq)
> +{
> + struct net_device *netdev = queue->ctrl->ddp_netdev;
> + struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
> + int ret;
> +
> + if (rq_data_dir(rq) != READ ||
> + queue->ctrl->ddp_threshold > blk_rq_payload_bytes(rq))
> + return;
> +
> + /*
> + * DDP offload is best-effort, errors are ignored.
> + */
> +
> + req->ddp.command_id = nvme_cid(rq);
> + ret = nvme_tcp_ddp_alloc_sgl(req, rq);
> + if (ret)
> + goto err;
> +
> + ret = ulp_ddp_setup(netdev, queue->sock->sk, &req->ddp);
> + if (ret) {
> + sg_free_table_chained(&req->ddp.sg_table, SG_CHUNK_SIZE);
> + goto err;
> + }
> +
> + /* if successful, sg table is freed in nvme_tcp_teardown_ddp() */
> + req->offloaded = true;
> +
> + return;
> +err:
> + WARN_ONCE(ret, "ddp setup failed (queue 0x%x, cid 0x%x, ret=%d)",
> + nvme_tcp_queue_id(queue),
> + nvme_cid(rq),
> + ret);
> +}
> +
> static int nvme_tcp_offload_socket(struct nvme_tcp_queue *queue)
> {
> struct ulp_ddp_config config = {.type = ULP_DDP_NVME};
> @@ -473,6 +559,11 @@ static bool nvme_tcp_resync_request(struct sock *sk, u32 seq, u32 flags)
>
> #else
>
> +static bool nvme_tcp_is_ddp_offloaded(const struct nvme_tcp_request *req)
> +{
> + return false;
> +}
> +
> static struct net_device *
> nvme_tcp_get_ddp_netdev_with_limits(struct nvme_tcp_ctrl *ctrl)
> {
> @@ -490,6 +581,14 @@ static int nvme_tcp_offload_socket(struct nvme_tcp_queue *queue)
> static void nvme_tcp_unoffload_socket(struct nvme_tcp_queue *queue)
> {}
>
> +static void nvme_tcp_setup_ddp(struct nvme_tcp_queue *queue,
> + struct request *rq)
> +{}
> +
> +static void nvme_tcp_teardown_ddp(struct nvme_tcp_queue *queue,
> + struct request *rq)
> +{}
> +
> static void nvme_tcp_resync_response(struct nvme_tcp_queue *queue,
> struct sk_buff *skb, unsigned int offset)
> {}
> @@ -765,6 +864,24 @@ static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
> queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
> }
>
> +static void nvme_tcp_complete_request(struct request *rq,
> + __le16 status,
> + union nvme_result result,
> + __u16 command_id)
> +{
> + struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
> +
> + if (nvme_tcp_is_ddp_offloaded(req)) {
> + req->nvme_status = status;
this can just be called req->status I think.
> + req->result = result;
I think it will be cleaner to always capture req->result and req->status
regardless of ddp offload.
Powered by blists - more mailing lists