| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <1400035847-7730-2-git-send-email-ast@plumgrid.com>
Date: Tue, 13 May 2014 19:50:45 -0700
From: Alexei Starovoitov <ast@...mgrid.com>
To: "David S. Miller" <davem@...emloft.net>
Cc: Eric Dumazet <edumazet@...gle.com>,
Daniel Borkmann <dborkman@...hat.com>,
"H. Peter Anvin" <hpa@...or.com>,
Thomas Gleixner <tglx@...utronix.de>,
Ingo Molnar <mingo@...nel.org>,
Chema Gonzalez <chema@...gle.com>,
Kees Cook <keescook@...omium.org>, netdev@...r.kernel.org
Subject: [PATCH net-next 1/3] net: filter: x86: split bpf_jit_compile()
Split bpf_jit_compile() into two functions to improve readability
of for(pass++) loop. The change follows similar style of JIT compilers
for arm, powerpc, s390
The body of new do_jit() was not reformatted to reduce noise
in this patch, since the following patch replaces most of it.
Tested with BPF testsuite.
Signed-off-by: Alexei Starovoitov <ast@...mgrid.com>
---
arch/x86/net/bpf_jit_comp.c | 157 +++++++++++++++++++++++++------------------
1 file changed, 92 insertions(+), 65 deletions(-)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index dc017735bb91..c5fa7c9cb665 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -178,41 +178,26 @@ static struct bpf_binary_header *bpf_alloc_binary(unsigned int proglen,
return header;
}
-void bpf_jit_compile(struct sk_filter *fp)
+struct jit_context {
+ unsigned int cleanup_addr; /* epilogue code offset */
+ int pc_ret0; /* bpf index of first RET #0 instruction (if any) */
+ u8 seen;
+};
+
+static int do_jit(struct sk_filter *bpf_prog, int *addrs, u8 *image,
+ int oldproglen, struct jit_context *ctx)
{
+ const struct sock_filter *filter = bpf_prog->insns;
+ int flen = bpf_prog->len;
u8 temp[64];
u8 *prog;
- unsigned int proglen, oldproglen = 0;
- int ilen, i;
+ int ilen, i, proglen;
int t_offset, f_offset;
- u8 t_op, f_op, seen = 0, pass;
- u8 *image = NULL;
- struct bpf_binary_header *header = NULL;
+ u8 t_op, f_op, seen = 0;
u8 *func;
- int pc_ret0 = -1; /* bpf index of first RET #0 instruction (if any) */
- unsigned int cleanup_addr; /* epilogue code offset */
- unsigned int *addrs;
- const struct sock_filter *filter = fp->insns;
- int flen = fp->len;
-
- if (!bpf_jit_enable)
- return;
-
- addrs = kmalloc(flen * sizeof(*addrs), GFP_KERNEL);
- if (addrs == NULL)
- return;
-
- /* Before first pass, make a rough estimation of addrs[]
- * each bpf instruction is translated to less than 64 bytes
- */
- for (proglen = 0, i = 0; i < flen; i++) {
- proglen += 64;
- addrs[i] = proglen;
- }
- cleanup_addr = proglen; /* epilogue address */
+ unsigned int cleanup_addr = ctx->cleanup_addr;
+ u8 seen_or_pass0 = ctx->seen;
- for (pass = 0; pass < 10; pass++) {
- u8 seen_or_pass0 = (pass == 0) ? (SEEN_XREG | SEEN_DATAREF | SEEN_MEM) : seen;
/* no prologue/epilogue for trivial filters (RET something) */
proglen = 0;
prog = temp;
@@ -325,12 +310,12 @@ void bpf_jit_compile(struct sk_filter *fp)
case BPF_S_ALU_DIV_X: /* A /= X; */
seen |= SEEN_XREG;
EMIT2(0x85, 0xdb); /* test %ebx,%ebx */
- if (pc_ret0 > 0) {
+ if (ctx->pc_ret0 > 0) {
/* addrs[pc_ret0 - 1] is start address of target
* (addrs[i] - 4) is the address following this jmp
* ("xor %edx,%edx; div %ebx" being 4 bytes long)
*/
- EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] -
+ EMIT_COND_JMP(X86_JE, addrs[ctx->pc_ret0 - 1] -
(addrs[i] - 4));
} else {
EMIT_COND_JMP(X86_JNE, 2 + 5);
@@ -342,12 +327,12 @@ void bpf_jit_compile(struct sk_filter *fp)
case BPF_S_ALU_MOD_X: /* A %= X; */
seen |= SEEN_XREG;
EMIT2(0x85, 0xdb); /* test %ebx,%ebx */
- if (pc_ret0 > 0) {
+ if (ctx->pc_ret0 > 0) {
/* addrs[pc_ret0 - 1] is start address of target
* (addrs[i] - 6) is the address following this jmp
* ("xor %edx,%edx; div %ebx;mov %edx,%eax" being 6 bytes long)
*/
- EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] -
+ EMIT_COND_JMP(X86_JE, addrs[ctx->pc_ret0 - 1] -
(addrs[i] - 6));
} else {
EMIT_COND_JMP(X86_JNE, 2 + 5);
@@ -441,8 +426,8 @@ void bpf_jit_compile(struct sk_filter *fp)
break;
case BPF_S_RET_K:
if (!K) {
- if (pc_ret0 == -1)
- pc_ret0 = i;
+ if (ctx->pc_ret0 == -1)
+ ctx->pc_ret0 = i;
CLEAR_A();
} else {
EMIT1_off32(0xb8, K); /* mov $imm32,%eax */
@@ -603,7 +588,7 @@ void bpf_jit_compile(struct sk_filter *fp)
int off = pkt_type_offset();
if (off < 0)
- goto out;
+ return -EINVAL;
if (is_imm8(off)) {
/* movzbl off8(%rdi),%eax */
EMIT4(0x0f, 0xb6, 0x47, off);
@@ -725,36 +710,79 @@ cond_branch: f_offset = addrs[i + filter[i].jf] - addrs[i];
}
EMIT_COND_JMP(f_op, f_offset);
break;
- default:
- /* hmm, too complex filter, give up with jit compiler */
- goto out;
- }
- ilen = prog - temp;
- if (image) {
- if (unlikely(proglen + ilen > oldproglen)) {
- pr_err("bpb_jit_compile fatal error\n");
- kfree(addrs);
- module_free(NULL, header);
- return;
- }
- memcpy(image + proglen, temp, ilen);
+ default:
+ /* hmm, too complex filter, give up with jit compiler */
+ return -EINVAL;
+ }
+ ilen = prog - temp;
+ if (image) {
+ if (unlikely(proglen + ilen > oldproglen)) {
+ pr_err("bpb_jit_compile fatal error\n");
+ return -EFAULT;
}
- proglen += ilen;
- addrs[i] = proglen;
- prog = temp;
+ memcpy(image + proglen, temp, ilen);
}
- /* last bpf instruction is always a RET :
- * use it to give the cleanup instruction(s) addr
- */
- cleanup_addr = proglen - 1; /* ret */
- if (seen_or_pass0)
- cleanup_addr -= 1; /* leaveq */
- if (seen_or_pass0 & SEEN_XREG)
- cleanup_addr -= 4; /* mov -8(%rbp),%rbx */
+ proglen += ilen;
+ addrs[i] = proglen;
+ prog = temp;
+ }
+ /* last bpf instruction is always a RET :
+ * use it to give the cleanup instruction(s) addr
+ */
+ ctx->cleanup_addr = proglen - 1; /* ret */
+ if (seen_or_pass0)
+ ctx->cleanup_addr -= 1; /* leaveq */
+ if (seen_or_pass0 & SEEN_XREG)
+ ctx->cleanup_addr -= 4; /* mov -8(%rbp),%rbx */
+
+ ctx->seen = seen;
+
+ return proglen;
+}
+
+void bpf_jit_compile(struct sk_filter *prog)
+{
+ struct bpf_binary_header *header = NULL;
+ int proglen, oldproglen = 0;
+ struct jit_context ctx = {};
+ u8 *image = NULL;
+ int *addrs;
+ int pass;
+ int i;
+
+ if (!bpf_jit_enable)
+ return;
+ if (!prog || !prog->len)
+ return;
+
+ addrs = kmalloc(prog->len * sizeof(*addrs), GFP_KERNEL);
+ if (!addrs)
+ return;
+
+ /* Before first pass, make a rough estimation of addrs[]
+ * each bpf instruction is translated to less than 64 bytes
+ */
+ for (proglen = 0, i = 0; i < prog->len; i++) {
+ proglen += 64;
+ addrs[i] = proglen;
+ }
+ ctx.cleanup_addr = proglen;
+ ctx.seen = SEEN_XREG | SEEN_DATAREF | SEEN_MEM;
+ ctx.pc_ret0 = -1;
+
+ for (pass = 0; pass < 10; pass++) {
+ proglen = do_jit(prog, addrs, image, oldproglen, &ctx);
+ if (proglen <= 0) {
+ image = NULL;
+ if (header)
+ module_free(NULL, header);
+ goto out;
+ }
if (image) {
if (proglen != oldproglen)
- pr_err("bpb_jit_compile proglen=%u != oldproglen=%u\n", proglen, oldproglen);
+ pr_err("bpf_jit: proglen=%d != oldproglen=%d\n",
+ proglen, oldproglen);
break;
}
if (proglen == oldproglen) {
@@ -766,17 +794,16 @@ cond_branch: f_offset = addrs[i + filter[i].jf] - addrs[i];
}
if (bpf_jit_enable > 1)
- bpf_jit_dump(flen, proglen, pass, image);
+ bpf_jit_dump(prog->len, proglen, 0, image);
if (image) {
bpf_flush_icache(header, image + proglen);
set_memory_ro((unsigned long)header, header->pages);
- fp->bpf_func = (void *)image;
- fp->jited = 1;
+ prog->bpf_func = (void *)image;
+ prog->jited = 1;
}
out:
kfree(addrs);
- return;
}
static void bpf_jit_free_deferred(struct work_struct *work)
--
1.7.9.5
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Powered by blists - more mailing lists