| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <b4d2689b2cfd4c457f640743f2495156bd92f050.1669547603.git.leonro@nvidia.com>
Date: Sun, 27 Nov 2022 13:18:16 +0200
From: Leon Romanovsky <leon@...nel.org>
To: Steffen Klassert <steffen.klassert@...unet.com>
Cc: Leon Romanovsky <leonro@...dia.com>,
"David S. Miller" <davem@...emloft.net>,
Eric Dumazet <edumazet@...gle.com>,
Herbert Xu <herbert@...dor.apana.org.au>,
Jakub Kicinski <kuba@...nel.org>, netdev@...r.kernel.org,
Bharat Bhushan <bbhushan2@...vell.com>
Subject: [PATCH xfrm-next v9 6/8] xfrm: speed-up lookup of HW policies
From: Leon Romanovsky <leonro@...dia.com>
Devices that implement IPsec packet offload mode should offload SA and
policies too. In RX path, it causes to the situation that HW will always
have higher priority over any SW policies.
It means that we don't need to perform any search of inexact policies
and/or priority checks if HW policy was discovered. In such situation,
the HW will catch the packets anyway and HW can still implement inexact
lookups.
In case specific policy is not found, we will continue with packet lookup and
check for existence of HW policies in inexact list.
HW policies are added to the head of SPD to ensure fast lookup, as XFRM
iterates over all policies in the loop.
The same solution of adding HW SAs at the begging of the list is applied
to SA database too. However, we don't need to change lookups as they are
sorted by insertion order and not priority.
Signed-off-by: Leon Romanovsky <leonro@...dia.com>
---
net/xfrm/xfrm_policy.c | 16 ++++++----
net/xfrm/xfrm_state.c | 66 ++++++++++++++++++++++++++++++++----------
2 files changed, 62 insertions(+), 20 deletions(-)
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 07f43729ac4e..9d43c0931de4 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -536,7 +536,7 @@ static void xfrm_dst_hash_transfer(struct net *net,
__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
pol->family, nhashmask, dbits, sbits);
- if (!entry0) {
+ if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
hlist_del_rcu(&pol->bydst);
hlist_add_head_rcu(&pol->bydst, ndsttable + h);
h0 = h;
@@ -867,7 +867,7 @@ static void xfrm_policy_inexact_list_reinsert(struct net *net,
break;
}
- if (newpos)
+ if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
hlist_add_behind_rcu(&policy->bydst, newpos);
else
hlist_add_head_rcu(&policy->bydst, &n->hhead);
@@ -1348,7 +1348,7 @@ static void xfrm_hash_rebuild(struct work_struct *work)
else
break;
}
- if (newpos)
+ if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
hlist_add_behind_rcu(&policy->bydst, newpos);
else
hlist_add_head_rcu(&policy->bydst, chain);
@@ -1525,7 +1525,7 @@ static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
break;
}
- if (newpos)
+ if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos);
else
hlist_add_head_rcu(&policy->bydst_inexact_list, chain);
@@ -1562,9 +1562,12 @@ static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain,
break;
}
- if (newpos)
+ if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
hlist_add_behind_rcu(&policy->bydst, &newpos->bydst);
else
+ /* Packet offload policies enter to the head
+ * to speed-up lookups.
+ */
hlist_add_head_rcu(&policy->bydst, chain);
return delpol;
@@ -2181,6 +2184,9 @@ static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
break;
}
}
+ if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET)
+ goto skip_inexact;
+
bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
if (!bin || !xfrm_policy_find_inexact_candidates(&cand, bin, saddr,
daddr))
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 22940bc0d4e4..b4adf4df9d08 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -84,6 +84,25 @@ static unsigned int xfrm_seq_hash(struct net *net, u32 seq)
return __xfrm_seq_hash(seq, net->xfrm.state_hmask);
}
+#define XFRM_STATE_INSERT(by, _n, _h, _type) \
+ { \
+ struct xfrm_state *_x = NULL; \
+ \
+ if (_type != XFRM_DEV_OFFLOAD_PACKET) { \
+ hlist_for_each_entry_rcu(_x, _h, by) { \
+ if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
+ continue; \
+ break; \
+ } \
+ } \
+ \
+ if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
+ /* SAD is empty or consist from HW SAs only */ \
+ hlist_add_head_rcu(_n, _h); \
+ else \
+ hlist_add_before_rcu(_n, &_x->by); \
+ }
+
static void xfrm_hash_transfer(struct hlist_head *list,
struct hlist_head *ndsttable,
struct hlist_head *nsrctable,
@@ -100,23 +119,25 @@ static void xfrm_hash_transfer(struct hlist_head *list,
h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
x->props.reqid, x->props.family,
nhashmask);
- hlist_add_head_rcu(&x->bydst, ndsttable + h);
+ XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type);
h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr,
x->props.family,
nhashmask);
- hlist_add_head_rcu(&x->bysrc, nsrctable + h);
+ XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type);
if (x->id.spi) {
h = __xfrm_spi_hash(&x->id.daddr, x->id.spi,
x->id.proto, x->props.family,
nhashmask);
- hlist_add_head_rcu(&x->byspi, nspitable + h);
+ XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h,
+ x->xso.type);
}
if (x->km.seq) {
h = __xfrm_seq_hash(x->km.seq, nhashmask);
- hlist_add_head_rcu(&x->byseq, nseqtable + h);
+ XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h,
+ x->xso.type);
}
}
}
@@ -1269,16 +1290,24 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
spin_lock_bh(&net->xfrm.xfrm_state_lock);
x->km.state = XFRM_STATE_ACQ;
list_add(&x->km.all, &net->xfrm.state_all);
- hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
+ XFRM_STATE_INSERT(bydst, &x->bydst,
+ net->xfrm.state_bydst + h,
+ x->xso.type);
h = xfrm_src_hash(net, daddr, saddr, encap_family);
- hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
+ XFRM_STATE_INSERT(bysrc, &x->bysrc,
+ net->xfrm.state_bysrc + h,
+ x->xso.type);
if (x->id.spi) {
h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family);
- hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
+ XFRM_STATE_INSERT(byspi, &x->byspi,
+ net->xfrm.state_byspi + h,
+ x->xso.type);
}
if (x->km.seq) {
h = xfrm_seq_hash(net, x->km.seq);
- hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h);
+ XFRM_STATE_INSERT(byseq, &x->byseq,
+ net->xfrm.state_byseq + h,
+ x->xso.type);
}
x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
hrtimer_start(&x->mtimer,
@@ -1394,22 +1423,26 @@ static void __xfrm_state_insert(struct xfrm_state *x)
h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,
x->props.reqid, x->props.family);
- hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
+ XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
+ x->xso.type);
h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family);
- hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
+ XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,
+ x->xso.type);
if (x->id.spi) {
h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto,
x->props.family);
- hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
+ XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
+ x->xso.type);
}
if (x->km.seq) {
h = xfrm_seq_hash(net, x->km.seq);
- hlist_add_head_rcu(&x->byseq, net->xfrm.state_byseq + h);
+ XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h,
+ x->xso.type);
}
hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);
@@ -1523,9 +1556,11 @@ static struct xfrm_state *__find_acq_core(struct net *net,
ktime_set(net->xfrm.sysctl_acq_expires, 0),
HRTIMER_MODE_REL_SOFT);
list_add(&x->km.all, &net->xfrm.state_all);
- hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
+ XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
+ x->xso.type);
h = xfrm_src_hash(net, daddr, saddr, family);
- hlist_add_head_rcu(&x->bysrc, net->xfrm.state_bysrc + h);
+ XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,
+ x->xso.type);
net->xfrm.state_num++;
@@ -2199,7 +2234,8 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high)
spin_lock_bh(&net->xfrm.xfrm_state_lock);
x->id.spi = newspi;
h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family);
- hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
+ XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,
+ x->xso.type);
spin_unlock_bh(&net->xfrm.xfrm_state_lock);
err = 0;
--
2.38.1
Powered by blists - more mailing lists