| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20260119-skb-meta-bpf-emit-call-from-prologue-v1-1-e8b88d6430d8@cloudflare.com>
Date: Mon, 19 Jan 2026 20:53:51 +0100
From: Jakub Sitnicki <jakub@...udflare.com>
To: bpf@...r.kernel.org
Cc: Alexei Starovoitov <ast@...nel.org>,
Daniel Borkmann <daniel@...earbox.net>,
John Fastabend <john.fastabend@...il.com>,
Andrii Nakryiko <andrii@...nel.org>,
Martin KaFai Lau <martin.lau@...ux.dev>,
Eduard Zingerman <eddyz87@...il.com>, Song Liu <song@...nel.org>,
Yonghong Song <yonghong.song@...ux.dev>, KP Singh <kpsingh@...nel.org>,
Stanislav Fomichev <sdf@...ichev.me>, Hao Luo <haoluo@...gle.com>,
Jiri Olsa <jolsa@...nel.org>, Amery Hung <ameryhung@...il.com>,
netdev@...r.kernel.org, kernel-team@...udflare.com
Subject: [PATCH bpf-next 1/4] bpf, verifier: Support direct helper calls
from prologue/epilogue
Prepare to remove support for calling kfuncs from prologue & epilogue.
Instead allow direct helpers calls using BPF_EMIT_CALL. Such calls already
contain helper offset relative to __bpf_call_base and must bypass the
verifier's patch_call_imm fixup, which expects BPF helper IDs rather than a
pre-resolved offsets.
Add a finalized_call flag to bpf_insn_aux_data to mark call instructions
with resolved offsets so the verifier can skip patch_call_imm fixup for
these calls.
Note that the target of BPF_EMIT_CALL should be wrapped with BPF_CALL_x to
prevent an ABI mismatch between BPF and C on 32-bit architectures.
Suggested-by: Alexei Starovoitov <ast@...nel.org>
Signed-off-by: Jakub Sitnicki <jakub@...udflare.com>
---
include/linux/bpf_verifier.h | 1 +
kernel/bpf/verifier.c | 24 ++++++++++++++++++++++++
net/core/filter.c | 3 +--
3 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 130bcbd66f60..e358f01c300d 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -561,6 +561,7 @@ struct bpf_insn_aux_data {
bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */
bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */
bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */
+ bool finalized_call; /* call holds resolved helper offset relative to __bpf_base_call */
u8 alu_state; /* used in combination with alu_limit */
/* true if STX or LDX instruction is a part of a spill/fill
* pattern for a bpf_fastcall call.
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 9de0ec0c3ed9..15694a40ca02 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -21801,6 +21801,19 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,
return 0;
}
+/* Mark helper calls within prog->insns[off ... off+cnt-1] range as resolved,
+ * meaning imm contains the helper offset. Used for prologue & epilogue.
+ */
+static void mark_helper_calls_finalized(struct bpf_verifier_env *env, int off, int cnt)
+{
+ int i;
+
+ for (i = 0; i < cnt; i++) {
+ if (bpf_helper_call(&env->prog->insnsi[i + off]))
+ env->insn_aux_data[i + off].finalized_call = true;
+ }
+}
+
/* convert load instructions that access fields of a context type into a
* sequence of instructions that access fields of the underlying structure:
* struct __sk_buff -> struct sk_buff
@@ -21867,6 +21880,8 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);
if (ret < 0)
return ret;
+
+ mark_helper_calls_finalized(env, 0, cnt - 1);
}
}
@@ -21880,6 +21895,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
for (i = 0; i < insn_cnt; i++, insn++) {
bpf_convert_ctx_access_t convert_ctx_access;
+ bool is_epilogue = false;
u8 mode;
if (env->insn_aux_data[i + delta].nospec) {
@@ -21946,6 +21962,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
* epilogue.
*/
epilogue_idx = i + delta;
+ is_epilogue = true;
}
goto patch_insn_buf;
} else {
@@ -22101,6 +22118,9 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
/* keep walking new program and skip insns we just inserted */
env->prog = new_prog;
insn = new_prog->insnsi + i + delta;
+
+ if (is_epilogue)
+ mark_helper_calls_finalized(env, epilogue_idx, cnt - 1);
}
return 0;
@@ -23477,6 +23497,9 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
goto next_insn;
}
patch_call_imm:
+ if (env->insn_aux_data[i + delta].finalized_call)
+ goto next_insn;
+
fn = env->ops->get_func_proto(insn->imm, env->prog);
/* all functions that have prototype and verifier allowed
* programs to call them, must be real in-kernel functions
@@ -23488,6 +23511,7 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
return -EFAULT;
}
insn->imm = fn->func - __bpf_call_base;
+ env->insn_aux_data[i + delta].finalized_call = true;
next_insn:
if (subprogs[cur_subprog + 1].start == i + delta + 1) {
subprogs[cur_subprog].stack_depth += stack_depth_extra;
diff --git a/net/core/filter.c b/net/core/filter.c
index d43df98e1ded..aa0fabcd21d1 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -9082,8 +9082,7 @@ static int bpf_unclone_prologue(struct bpf_insn *insn_buf, bool direct_write,
/* ret = bpf_skb_pull_data(skb, 0); */
*insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
*insn++ = BPF_ALU64_REG(BPF_XOR, BPF_REG_2, BPF_REG_2);
- *insn++ = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
- BPF_FUNC_skb_pull_data);
+ *insn++ = BPF_EMIT_CALL(bpf_skb_pull_data);
/* if (!ret)
* goto restore;
* return TC_ACT_SHOT;
--
2.43.0
Powered by blists - more mailing lists