diff --cc net/netfilter/nft_set_pipapo.c index 4b64c3bd8e70,793790d79d13..a7b8fa8cab7c --- a/net/netfilter/nft_set_pipapo.c +++ b/net/netfilter/nft_set_pipapo.c @@@ -562,7 -539,7 +578,7 @@@ nft_pipapo_lookup(const struct net *net const struct nft_pipapo_elem *e; m = rcu_dereference(priv->match); - e = pipapo_get_slow(m, (const u8 *)key, genmask, get_jiffies_64()); - e = pipapo_get(m, (const u8 *)key, NFT_GENMASK_ANY, get_jiffies_64()); ++ e = pipapo_get_slow(m, (const u8 *)key, NFT_GENMASK_ANY, get_jiffies_64()); return e ? &e->ext : NULL; } diff --cc net/netfilter/nft_set_pipapo_avx2.c index 7559306d0aed,c0884fa68c79..27dab3667548 --- a/net/netfilter/nft_set_pipapo_avx2.c +++ b/net/netfilter/nft_set_pipapo_avx2.c @@@ -1226,75 -1241,28 +1226,74 @@@ next_match #undef NFT_SET_PIPAPO_AVX2_LOOKUP - if (ret < 0) - goto out; - - if (last) { - const struct nft_set_ext *e = &f->mt[ret].e->ext; - - if (unlikely(nft_set_elem_expired(e))) - goto next_match; - - ext = e; - goto out; + if (ret < 0) { + scratch->map_index = map_index; + kernel_fpu_end(); + __local_unlock_nested_bh(&scratch->bh_lock); + return NULL; } + if (last) { + struct nft_pipapo_elem *e; + + e = f->mt[ret].e; + if (unlikely(__nft_set_elem_expired(&e->ext, tstamp) || + !nft_set_elem_active(&e->ext, genmask))) + goto next_match; + + scratch->map_index = map_index; + kernel_fpu_end(); + __local_unlock_nested_bh(&scratch->bh_lock); + return e; + } + + map_index = !map_index; swap(res, fill); - rp += NFT_PIPAPO_GROUPS_PADDED_SIZE(f); + data += NFT_PIPAPO_GROUPS_PADDED_SIZE(f); } -out: - if (i % 2) - scratch->map_index = !map_index; kernel_fpu_end(); + __local_unlock_nested_bh(&scratch->bh_lock); + return NULL; +} + +/** + * nft_pipapo_avx2_lookup() - Dataplane frontend for AVX2 implementation + * @net: Network namespace + * @set: nftables API set representation + * @key: nftables API element representation containing key data + * + * This function is called from the data path. It will search for + * an element matching the given key in the current active copy using + * the AVX2 routines if the FPU is usable or fall back to the generic + * implementation of the algorithm otherwise. + * + * Return: nftables API extension pointer or NULL if no match. + */ +const struct nft_set_ext * +nft_pipapo_avx2_lookup(const struct net *net, const struct nft_set *set, + const u32 *key) +{ + struct nft_pipapo *priv = nft_set_priv(set); - u8 genmask = nft_genmask_cur(net); + const struct nft_pipapo_match *m; + const u8 *rp = (const u8 *)key; + const struct nft_pipapo_elem *e; + + local_bh_disable(); + + if (unlikely(!irq_fpu_usable())) { + const struct nft_set_ext *ext; + + ext = nft_pipapo_lookup(net, set, key); + + local_bh_enable(); + return ext; + } + + m = rcu_dereference(priv->match); + - e = pipapo_get_avx2(m, rp, genmask, get_jiffies_64()); ++ e = pipapo_get_avx2(m, rp, NFT_GENMASK_ANY, get_jiffies_64()); local_bh_enable(); - return ext; + return e ? &e->ext : NULL; }