diff --git a/arch/arm64/net/bpf_timed_may_goto.S b/arch/arm64/net/bpf_timed_may_goto.S index a9a802711a7f9d..53148c9a1c5845 100644 --- a/arch/arm64/net/bpf_timed_may_goto.S +++ b/arch/arm64/net/bpf_timed_may_goto.S @@ -14,16 +14,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) stp x3, x4, [sp, #48] /* - * Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP - * (x25) to get the pointer to count and timestamp and pass it as the - * first argument in x0. - * - * Before generating the call to arch_bpf_timed_may_goto, the verifier - * generates a load instruction using FP, i.e. REG_AX = *(u64 *)(FP - - * stack_off_cnt), so BPF_REG_FP (x25) is always set up by the arm64 - * jit in this case. + * BPF_REG_AX (x9) contains the count and timestamp pointer; pass it as + * the first argument in x0. */ - add x0, x9, x25 + mov x0, x9 bl bpf_check_timed_may_goto /* BPF_REG_AX(x9) will be stored into count, so move return value to it. */ mov x9, x0 diff --git a/arch/powerpc/net/bpf_timed_may_goto.S b/arch/powerpc/net/bpf_timed_may_goto.S index 84ecf6fa7f5dcf..d88825dd34fea1 100644 --- a/arch/powerpc/net/bpf_timed_may_goto.S +++ b/arch/powerpc/net/bpf_timed_may_goto.S @@ -8,7 +8,7 @@ * arch_bpf_timed_may_goto() trampoline for powerpc64 * * Custom BPF convention (verifier/JIT): - * - input: stack offset in BPF_REG_AX (r12) + * - input: count and timestamp pointer in BPF_REG_AX (r12) * - output: updated count in BPF_REG_AX (r12) * * Call bpf_check_timed_may_goto(ptr) with normal powerpc64 ABI: @@ -32,10 +32,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) std r8, 72(r1) /* - * r3 = BPF_REG_FP + BPF_REG_AX - * BPF_REG_FP is r31; BPF_REG_AX is r12 (stack offset in bytes). + * BPF_REG_AX (r12) contains the count and timestamp pointer; pass it as + * the first argument in r3. */ - add r3, r31, r12 + mr r3, r12 bl CFUNC(bpf_check_timed_may_goto) /* Put return value back into AX */ diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index 02c637d874205a..8c03e272e0bf53 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -6,12 +6,11 @@ /* * Trampoline for the BPF timed may_goto loop bound. Custom calling convention: - * - input: stack offset in BPF_REG_AX (t0) + * - input: count and timestamp pointer in BPF_REG_AX (t0) * - output: updated count in BPF_REG_AX (t0) * - * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (a5, a0-a4) are saved - * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving. + * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI. + * BPF R0-R5 (a5, a0-a4) are saved across the call. */ SYM_FUNC_START(arch_bpf_timed_may_goto) @@ -28,7 +27,11 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_S a3, 1*SZREG(sp) REG_S a4, 0*SZREG(sp) - add a0, t0, s5 + /* + * BPF_REG_AX (t0) contains the count and timestamp pointer; pass it as + * the first argument in a0. + */ + mv a0, t0 call bpf_check_timed_may_goto mv t0, a0 diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index c4b47070bb590c..f825514fd2c967 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1888,9 +1888,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if ((void *)func == arch_bpf_timed_may_goto) { /* * arch_bpf_timed_may_goto() has a special ABI: the - * parameters are in BPF_REG_AX and BPF_REG_10; the - * return value is in BPF_REG_AX; and all GPRs except - * REG_W0, REG_W1, and BPF_REG_AX are callee-saved. + * parameter and return value are in BPF_REG_AX; all + * GPRs except REG_W0, REG_W1, and BPF_REG_AX are + * callee-saved. */ /* brasl %r0,func */ diff --git a/arch/s390/net/bpf_timed_may_goto.S b/arch/s390/net/bpf_timed_may_goto.S index 06f567a460d7bd..e34c99982c4a25 100644 --- a/arch/s390/net/bpf_timed_may_goto.S +++ b/arch/s390/net/bpf_timed_may_goto.S @@ -21,9 +21,9 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* - * This function has a special ABI: the parameters are in %r12 and - * %r13; the return value is in %r12; all GPRs except %r0, %r1, and - * %r12 are callee-saved; and the return address is in %r0. + * This function has a special ABI: the parameter and return value are + * in %r12; all GPRs except %r0, %r1, and %r12 are callee-saved; and + * the return address is in %r0. */ stmg %r2,%r5,FRAME_OFF+R2_OFF(%r15) stg %r14,FRAME_OFF+R14_OFF(%r15) @@ -33,7 +33,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) lay %r15,-FRAME_SIZE(%r15) stg %r1,__SF_BACKCHAIN(%r15) - lay %r2,0(%r12,%r13) + lgr %r2,%r12 brasl %r14,bpf_check_timed_may_goto lgr %r12,%r2 diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S index 54c690cae19087..6e9d9e7c2b1d62 100644 --- a/arch/x86/net/bpf_timed_may_goto.S +++ b/arch/x86/net/bpf_timed_may_goto.S @@ -11,12 +11,6 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) ANNOTATE_NOENDBR - /* - * r10 passes us stack depth, load the pointer to count and timestamp - * into r10 by adding it to BPF frame pointer. - */ - leaq (%rbp, %r10, 1), %r10 - /* Setup frame. */ pushq %rbp movq %rsp, %rbp diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 65b441e4a35177..dc59501a32bbd3 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1797,20 +1797,20 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) stack_depth_extra = 16; insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt); if (insn->off >= 0) - insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5); + insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 6); else insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1); insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); - insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); + insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 3); /* - * AX is used as an argument to pass in stack_off_cnt - * (to add to r10/fp), and also as the return value of - * the call to arch_bpf_timed_may_goto. + * AX is used to pass FP + stack_off_cnt as the argument to + * arch_bpf_timed_may_goto(), and also holds its return value. */ - insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt); - insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); - insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); - cnt = 7; + insn_buf[4] = BPF_MOV64_REG(BPF_REG_AX, BPF_REG_FP); + insn_buf[5] = BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, stack_off_cnt); + insn_buf[6] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); + insn_buf[7] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt); + cnt = 8; new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -2661,4 +2661,3 @@ int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env) return 0; } - diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index 328cf630210a23..c5996d9dc8cc97 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c @@ -635,15 +635,16 @@ __xlated("4: r0 = &(void __percpu *)(r0)") __xlated("...") /* may_goto expansion starts */ __xlated("6: r12 = *(u64 *)(r10 -24)") -__xlated("7: if r12 == 0x0 goto pc+6") +__xlated("7: if r12 == 0x0 goto pc+7") __xlated("8: r12 -= 1") -__xlated("9: if r12 != 0x0 goto pc+2") -__xlated("10: r12 = -24") -__xlated("11: call unknown") -__xlated("12: *(u64 *)(r10 -24) = r12") +__xlated("9: if r12 != 0x0 goto pc+3") +__xlated("10: r12 = r10") +__xlated("11: r12 += -24") +__xlated("12: call unknown") +__xlated("13: *(u64 *)(r10 -24) = r12") /* may_goto expansion ends */ -__xlated("13: *(u64 *)(r10 -8) = r1") -__xlated("14: exit") +__xlated("14: *(u64 *)(r10 -8) = r1") +__xlated("15: exit") __success __naked void may_goto_interaction_x86_64(void) { @@ -676,15 +677,16 @@ __xlated("2: r1 = 1") __xlated("3: call bpf_get_smp_processor_id") /* may_goto expansion starts */ __xlated("4: r12 = *(u64 *)(r10 -24)") -__xlated("5: if r12 == 0x0 goto pc+6") +__xlated("5: if r12 == 0x0 goto pc+7") __xlated("6: r12 -= 1") -__xlated("7: if r12 != 0x0 goto pc+2") -__xlated("8: r12 = -24") -__xlated("9: call unknown") -__xlated("10: *(u64 *)(r10 -24) = r12") +__xlated("7: if r12 != 0x0 goto pc+3") +__xlated("8: r12 = r10") +__xlated("9: r12 += -24") +__xlated("10: call unknown") +__xlated("11: *(u64 *)(r10 -24) = r12") /* may_goto expansion ends */ -__xlated("11: *(u64 *)(r10 -8) = r1") -__xlated("12: exit") +__xlated("12: *(u64 *)(r10 -8) = r1") +__xlated("13: exit") __success __naked void may_goto_interaction(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c index db7e30da234f1f..1721897160dc99 100644 --- a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +++ b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c @@ -90,15 +90,16 @@ __arch_loongarch __xlated("0: *(u64 *)(r10 -16) = 65535") __xlated("1: *(u64 *)(r10 -8) = 0") __xlated("2: r12 = *(u64 *)(r10 -16)") -__xlated("3: if r12 == 0x0 goto pc+6") +__xlated("3: if r12 == 0x0 goto pc+7") __xlated("4: r12 -= 1") -__xlated("5: if r12 != 0x0 goto pc+2") -__xlated("6: r12 = -16") -__xlated("7: call unknown") -__xlated("8: *(u64 *)(r10 -16) = r12") -__xlated("9: r0 = 1") -__xlated("10: r0 = 2") -__xlated("11: exit") +__xlated("5: if r12 != 0x0 goto pc+3") +__xlated("6: r12 = r10") +__xlated("7: r12 += -16") +__xlated("8: call unknown") +__xlated("9: *(u64 *)(r10 -16) = r12") +__xlated("10: r0 = 1") +__xlated("11: r0 = 2") +__xlated("12: exit") __success __naked void may_goto_batch_2(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index ea0a7e73331dad..b1400bc86c9863 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -70,6 +70,25 @@ __naked void no_private_stack_nested(void) " ::: __clobber_all); } +SEC("fentry/bpf_fentry_test9") +__description("Private stack, timed may_goto") +__success __retval(0) +__arch_x86_64 +__jited(" movabsq $0x{{.*}}, %r9") +__jited("...") +__jited(" movq %r9, %r10") +__arch_arm64 +__jited(" mov x9, x25") +int private_stack_timed_may_goto(void) +{ + volatile __u64 stack_space[16] = {}; + + while (can_loop) + ; + + return stack_space[0]; +} + __used __naked static void cumulative_stack_depth_subprog(void) {