| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20210211211044.32701-4-borisp@mellanox.com>
Date: Thu, 11 Feb 2021 23:10:26 +0200
From: Boris Pismenny <borisp@...lanox.com>
To: <dsahern@...il.com>, <kuba@...nel.org>, <davem@...emloft.net>,
<saeedm@...dia.com>, <hch@....de>, <sagi@...mberg.me>,
<axboe@...com>, <kbusch@...nel.org>, <viro@...iv.linux.org.uk>,
<edumazet@...gle.com>, <smalin@...vell.com>
CC: <boris.pismenny@...il.com>, <linux-nvme@...ts.infradead.org>,
<netdev@...r.kernel.org>, <benishay@...dia.com>,
<ogerlitz@...dia.com>, <yorayz@...dia.com>,
Boris Pismenny <borisp@...lanox.com>,
Ben Ben-Ishay <benishay@...lanox.com>,
Or Gerlitz <ogerlitz@...lanox.com>,
Yoray Zack <yorayz@...lanox.com>
Subject: [PATCH v4 net-next 03/21] iov_iter: DDP copy to iter/pages
When using direct data placement (DDP) the NIC writes some of the payload
directly to the destination buffer, and constructs SKBs such that they
point to this data. To skip copies when SKB data already resides in the
destination we use the newly introduced routines in this commit, which
check if (src == dst), and skip the copy when that's true.
As the current user for these routines is in the block layer (nvme-tcp),
then we only apply the change for bio_vec. Other routines use the normal
methods for copying.
Signed-off-by: Boris Pismenny <borisp@...lanox.com>
Signed-off-by: Ben Ben-Ishay <benishay@...lanox.com>
Signed-off-by: Or Gerlitz <ogerlitz@...lanox.com>
Signed-off-by: Yoray Zack <yorayz@...lanox.com>
---
include/linux/uio.h | 17 +++++++++++++++
lib/iov_iter.c | 53 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 70 insertions(+)
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 72d88566694e..8438e6d5ac25 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -123,6 +123,9 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i);
size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
+#ifdef CONFIG_TCP_DDP
+size_t _ddp_copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
+#endif
size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i);
bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i);
size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i);
@@ -137,6 +140,16 @@ size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
return _copy_to_iter(addr, bytes, i);
}
+#ifdef CONFIG_TCP_DDP
+static __always_inline __must_check
+size_t ddp_copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
+{
+ if (unlikely(!check_copy_size(addr, bytes, true)))
+ return 0;
+ return _ddp_copy_to_iter(addr, bytes, i);
+}
+#endif
+
static __always_inline __must_check
size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
{
@@ -265,6 +278,10 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct io
bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
struct iov_iter *i);
+#ifdef CONFIG_TCP_DDP
+size_t ddp_hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
+ struct iov_iter *i);
+#endif
struct iovec *iovec_from_user(const struct iovec __user *uvector,
unsigned long nr_segs, unsigned long fast_segs,
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index a21e6a5792c5..c5990275e583 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -473,6 +473,18 @@ static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t
kunmap_atomic(from);
}
+#ifdef CONFIG_TCP_DDP
+static void ddp_memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
+{
+ char *to = kmap_atomic(page);
+
+ if (to + offset != from)
+ memcpy(to + offset, from, len);
+
+ kunmap_atomic(to);
+}
+#endif
+
static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
{
char *to = kmap_atomic(page);
@@ -625,6 +637,26 @@ static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
return bytes;
}
+#ifdef CONFIG_TCP_DDP
+size_t _ddp_copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
+{
+ const char *from = addr;
+ if (unlikely(iov_iter_is_pipe(i)))
+ return copy_pipe_to_iter(addr, bytes, i);
+ if (iter_is_iovec(i))
+ might_fault();
+ iterate_and_advance(i, bytes, v,
+ copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
+ ddp_memcpy_to_page(v.bv_page, v.bv_offset,
+ (from += v.bv_len) - v.bv_len, v.bv_len),
+ memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
+ )
+
+ return bytes;
+}
+EXPORT_SYMBOL(_ddp_copy_to_iter);
+#endif
+
size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
{
const char *from = addr;
@@ -1566,6 +1598,27 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump,
}
EXPORT_SYMBOL(csum_and_copy_to_iter);
+#ifdef CONFIG_TCP_DDP
+size_t ddp_hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
+ struct iov_iter *i)
+{
+#ifdef CONFIG_CRYPTO_HASH
+ struct ahash_request *hash = hashp;
+ struct scatterlist sg;
+ size_t copied;
+
+ copied = ddp_copy_to_iter(addr, bytes, i);
+ sg_init_one(&sg, addr, copied);
+ ahash_request_set_crypt(hash, &sg, NULL, copied);
+ crypto_ahash_update(hash);
+ return copied;
+#else
+ return 0;
+#endif
+}
+EXPORT_SYMBOL(ddp_hash_and_copy_to_iter);
+#endif
+
size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
struct iov_iter *i)
{
--
2.24.1
Powered by blists - more mailing lists