diff options
Diffstat (limited to 'tools/testing/selftests/bpf/prog_tests')
150 files changed, 13795 insertions, 1863 deletions
diff --git a/tools/testing/selftests/bpf/prog_tests/access_variable_array.c b/tools/testing/selftests/bpf/prog_tests/access_variable_array.c deleted file mode 100644 index 08131782437c..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/access_variable_array.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2022 Bytedance */ - -#include <test_progs.h> -#include "test_access_variable_array.skel.h" - -void test_access_variable_array(void) -{ - struct test_access_variable_array *skel; - - skel = test_access_variable_array__open_and_load(); - if (!ASSERT_OK_PTR(skel, "test_access_variable_array__open_and_load")) - return; - - test_access_variable_array__destroy(skel); -} diff --git a/tools/testing/selftests/bpf/prog_tests/align.c b/tools/testing/selftests/bpf/prog_tests/align.c deleted file mode 100644 index 24c509ce4e5b..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/align.c +++ /dev/null @@ -1,712 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include <test_progs.h> - -#define MAX_INSNS 512 -#define MAX_MATCHES 24 - -struct bpf_reg_match { - unsigned int line; - const char *reg; - const char *match; -}; - -struct bpf_align_test { - const char *descr; - struct bpf_insn insns[MAX_INSNS]; - enum { - UNDEF, - ACCEPT, - REJECT - } result; - enum bpf_prog_type prog_type; - /* Matches must be in order of increasing line */ - struct bpf_reg_match matches[MAX_MATCHES]; -}; - -static struct bpf_align_test tests[] = { - /* Four tests of known constants. These aren't staggeringly - * interesting since we track exact values now. - */ - { - .descr = "mov", - .insns = { - BPF_MOV64_IMM(BPF_REG_3, 2), - BPF_MOV64_IMM(BPF_REG_3, 4), - BPF_MOV64_IMM(BPF_REG_3, 8), - BPF_MOV64_IMM(BPF_REG_3, 16), - BPF_MOV64_IMM(BPF_REG_3, 32), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {0, "R1", "ctx()"}, - {0, "R10", "fp0"}, - {0, "R3", "2"}, - {1, "R3", "4"}, - {2, "R3", "8"}, - {3, "R3", "16"}, - {4, "R3", "32"}, - }, - }, - { - .descr = "shift", - .insns = { - BPF_MOV64_IMM(BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_3, 4), - BPF_MOV64_IMM(BPF_REG_4, 32), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {0, "R1", "ctx()"}, - {0, "R10", "fp0"}, - {0, "R3", "1"}, - {1, "R3", "2"}, - {2, "R3", "4"}, - {3, "R3", "8"}, - {4, "R3", "16"}, - {5, "R3", "1"}, - {6, "R4", "32"}, - {7, "R4", "16"}, - {8, "R4", "8"}, - {9, "R4", "4"}, - {10, "R4", "2"}, - }, - }, - { - .descr = "addsub", - .insns = { - BPF_MOV64_IMM(BPF_REG_3, 4), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, 4), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, 2), - BPF_MOV64_IMM(BPF_REG_4, 8), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 2), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {0, "R1", "ctx()"}, - {0, "R10", "fp0"}, - {0, "R3", "4"}, - {1, "R3", "8"}, - {2, "R3", "10"}, - {3, "R4", "8"}, - {4, "R4", "12"}, - {5, "R4", "14"}, - }, - }, - { - .descr = "mul", - .insns = { - BPF_MOV64_IMM(BPF_REG_3, 7), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 2), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 4), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {0, "R1", "ctx()"}, - {0, "R10", "fp0"}, - {0, "R3", "7"}, - {1, "R3", "7"}, - {2, "R3", "14"}, - {3, "R3", "56"}, - }, - }, - - /* Tests using unknown values */ -#define PREP_PKT_POINTERS \ - BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, \ - offsetof(struct __sk_buff, data)), \ - BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, \ - offsetof(struct __sk_buff, data_end)) - -#define LOAD_UNKNOWN(DST_REG) \ - PREP_PKT_POINTERS, \ - BPF_MOV64_REG(BPF_REG_0, BPF_REG_2), \ - BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8), \ - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_0, 1), \ - BPF_EXIT_INSN(), \ - BPF_LDX_MEM(BPF_B, DST_REG, BPF_REG_2, 0) - - { - .descr = "unknown shift", - .insns = { - LOAD_UNKNOWN(BPF_REG_3), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1), - LOAD_UNKNOWN(BPF_REG_4), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_4, 5), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {6, "R0", "pkt(off=8,r=8)"}, - {6, "R3", "var_off=(0x0; 0xff)"}, - {7, "R3", "var_off=(0x0; 0x1fe)"}, - {8, "R3", "var_off=(0x0; 0x3fc)"}, - {9, "R3", "var_off=(0x0; 0x7f8)"}, - {10, "R3", "var_off=(0x0; 0xff0)"}, - {12, "R3", "pkt_end()"}, - {17, "R4", "var_off=(0x0; 0xff)"}, - {18, "R4", "var_off=(0x0; 0x1fe0)"}, - {19, "R4", "var_off=(0x0; 0xff0)"}, - {20, "R4", "var_off=(0x0; 0x7f8)"}, - {21, "R4", "var_off=(0x0; 0x3fc)"}, - {22, "R4", "var_off=(0x0; 0x1fe)"}, - }, - }, - { - .descr = "unknown mul", - .insns = { - LOAD_UNKNOWN(BPF_REG_3), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_3), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 1), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_3), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 2), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_3), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 4), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_3), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 8), - BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 2), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {6, "R3", "var_off=(0x0; 0xff)"}, - {7, "R4", "var_off=(0x0; 0xff)"}, - {8, "R4", "var_off=(0x0; 0xff)"}, - {9, "R4", "var_off=(0x0; 0xff)"}, - {10, "R4", "var_off=(0x0; 0x1fe)"}, - {11, "R4", "var_off=(0x0; 0xff)"}, - {12, "R4", "var_off=(0x0; 0x3fc)"}, - {13, "R4", "var_off=(0x0; 0xff)"}, - {14, "R4", "var_off=(0x0; 0x7f8)"}, - {15, "R4", "var_off=(0x0; 0xff0)"}, - }, - }, - { - .descr = "packet const offset", - .insns = { - PREP_PKT_POINTERS, - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - - BPF_MOV64_IMM(BPF_REG_0, 0), - - /* Skip over ethernet header. */ - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - - BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 0), - BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 1), - BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 2), - BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 3), - BPF_LDX_MEM(BPF_H, BPF_REG_4, BPF_REG_5, 0), - BPF_LDX_MEM(BPF_H, BPF_REG_4, BPF_REG_5, 2), - BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0), - - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - {2, "R5", "pkt(r=0)"}, - {4, "R5", "pkt(off=14,r=0)"}, - {5, "R4", "pkt(off=14,r=0)"}, - {9, "R2", "pkt(r=18)"}, - {10, "R5", "pkt(off=14,r=18)"}, - {10, "R4", "var_off=(0x0; 0xff)"}, - {13, "R4", "var_off=(0x0; 0xffff)"}, - {14, "R4", "var_off=(0x0; 0xffff)"}, - }, - }, - { - .descr = "packet variable offset", - .insns = { - LOAD_UNKNOWN(BPF_REG_6), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2), - - /* First, add a constant to the R5 packet pointer, - * then a variable with a known alignment. - */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0), - - /* Now, test in the other direction. Adding first - * the variable offset to R5, then the constant. - */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0), - - /* Test multiple accumulations of unknown values - * into a packet pointer. - */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 4), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0), - - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - /* Calculated offset in R6 has unknown value, but known - * alignment of 4. - */ - {6, "R2", "pkt(r=8)"}, - {7, "R6", "var_off=(0x0; 0x3fc)"}, - /* Offset is added to packet pointer R5, resulting in - * known fixed offset, and variable offset from R6. - */ - {11, "R5", "pkt(id=1,off=14,"}, - /* At the time the word size load is performed from R5, - * it's total offset is NET_IP_ALIGN + reg->off (0) + - * reg->aux_off (14) which is 16. Then the variable - * offset is considered using reg->aux_off_align which - * is 4 and meets the load's requirements. - */ - {15, "R4", "var_off=(0x0; 0x3fc)"}, - {15, "R5", "var_off=(0x0; 0x3fc)"}, - /* Variable offset is added to R5 packet pointer, - * resulting in auxiliary alignment of 4. To avoid BPF - * verifier's precision backtracking logging - * interfering we also have a no-op R4 = R5 - * instruction to validate R5 state. We also check - * that R4 is what it should be in such case. - */ - {18, "R4", "var_off=(0x0; 0x3fc)"}, - {18, "R5", "var_off=(0x0; 0x3fc)"}, - /* Constant offset is added to R5, resulting in - * reg->off of 14. - */ - {19, "R5", "pkt(id=2,off=14,"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off - * (14) which is 16. Then the variable offset is 4-byte - * aligned, so the total offset is 4-byte aligned and - * meets the load's requirements. - */ - {24, "R4", "var_off=(0x0; 0x3fc)"}, - {24, "R5", "var_off=(0x0; 0x3fc)"}, - /* Constant offset is added to R5 packet pointer, - * resulting in reg->off value of 14. - */ - {26, "R5", "pkt(off=14,r=8)"}, - /* Variable offset is added to R5, resulting in a - * variable offset of (4n). See comment for insn #18 - * for R4 = R5 trick. - */ - {28, "R4", "var_off=(0x0; 0x3fc)"}, - {28, "R5", "var_off=(0x0; 0x3fc)"}, - /* Constant is added to R5 again, setting reg->off to 18. */ - {29, "R5", "pkt(id=3,off=18,"}, - /* And once more we add a variable; resulting var_off - * is still (4n), fixed offset is not changed. - * Also, we create a new reg->id. - */ - {31, "R4", "var_off=(0x0; 0x7fc)"}, - {31, "R5", "var_off=(0x0; 0x7fc)"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off (18) - * which is 20. Then the variable offset is (4n), so - * the total offset is 4-byte aligned and meets the - * load's requirements. - */ - {35, "R4", "var_off=(0x0; 0x7fc)"}, - {35, "R5", "var_off=(0x0; 0x7fc)"}, - }, - }, - { - .descr = "packet variable offset 2", - .insns = { - /* Create an unknown offset, (4n+2)-aligned */ - LOAD_UNKNOWN(BPF_REG_6), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 14), - /* Add it to the packet pointer */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - /* Check bounds and perform a read */ - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0), - /* Make a (4n) offset from the value we just read */ - BPF_ALU64_IMM(BPF_AND, BPF_REG_6, 0xff), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2), - /* Add it to the packet pointer */ - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - /* Check bounds and perform a read */ - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0), - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - /* Calculated offset in R6 has unknown value, but known - * alignment of 4. - */ - {6, "R2", "pkt(r=8)"}, - {7, "R6", "var_off=(0x0; 0x3fc)"}, - /* Adding 14 makes R6 be (4n+2) */ - {8, "R6", "var_off=(0x2; 0x7fc)"}, - /* Packet pointer has (4n+2) offset */ - {11, "R5", "var_off=(0x2; 0x7fc)"}, - {12, "R4", "var_off=(0x2; 0x7fc)"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off (0) - * which is 2. Then the variable offset is (4n+2), so - * the total offset is 4-byte aligned and meets the - * load's requirements. - */ - {15, "R5", "var_off=(0x2; 0x7fc)"}, - /* Newly read value in R6 was shifted left by 2, so has - * known alignment of 4. - */ - {17, "R6", "var_off=(0x0; 0x3fc)"}, - /* Added (4n) to packet pointer's (4n+2) var_off, giving - * another (4n+2). - */ - {19, "R5", "var_off=(0x2; 0xffc)"}, - {20, "R4", "var_off=(0x2; 0xffc)"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off (0) - * which is 2. Then the variable offset is (4n+2), so - * the total offset is 4-byte aligned and meets the - * load's requirements. - */ - {23, "R5", "var_off=(0x2; 0xffc)"}, - }, - }, - { - .descr = "dubious pointer arithmetic", - .insns = { - PREP_PKT_POINTERS, - BPF_MOV64_IMM(BPF_REG_0, 0), - /* (ptr - ptr) << 2 */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_3), - BPF_ALU64_REG(BPF_SUB, BPF_REG_5, BPF_REG_2), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_5, 2), - /* We have a (4n) value. Let's make a packet offset - * out of it. First add 14, to make it a (4n+2) - */ - BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14), - /* Then make sure it's nonnegative */ - BPF_JMP_IMM(BPF_JSGE, BPF_REG_5, 0, 1), - BPF_EXIT_INSN(), - /* Add it to packet pointer */ - BPF_MOV64_REG(BPF_REG_6, BPF_REG_2), - BPF_ALU64_REG(BPF_ADD, BPF_REG_6, BPF_REG_5), - /* Check bounds and perform a read */ - BPF_MOV64_REG(BPF_REG_4, BPF_REG_6), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_6, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .result = REJECT, - .matches = { - {3, "R5", "pkt_end()"}, - /* (ptr - ptr) << 2 == unknown, (4n) */ - {5, "R5", "var_off=(0x0; 0xfffffffffffffffc)"}, - /* (4n) + 14 == (4n+2). We blow our bounds, because - * the add could overflow. - */ - {6, "R5", "var_off=(0x2; 0xfffffffffffffffc)"}, - /* Checked s>=0 */ - {9, "R5", "var_off=(0x2; 0x7ffffffffffffffc)"}, - /* packet pointer + nonnegative (4n+2) */ - {11, "R6", "var_off=(0x2; 0x7ffffffffffffffc)"}, - {12, "R4", "var_off=(0x2; 0x7ffffffffffffffc)"}, - /* NET_IP_ALIGN + (4n+2) == (4n), alignment is fine. - * We checked the bounds, but it might have been able - * to overflow if the packet pointer started in the - * upper half of the address space. - * So we did not get a 'range' on R6, and the access - * attempt will fail. - */ - {15, "R6", "var_off=(0x2; 0x7ffffffffffffffc)"}, - } - }, - { - .descr = "variable subtraction", - .insns = { - /* Create an unknown offset, (4n+2)-aligned */ - LOAD_UNKNOWN(BPF_REG_6), - BPF_MOV64_REG(BPF_REG_7, BPF_REG_6), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 14), - /* Create another unknown, (4n)-aligned, and subtract - * it from the first one - */ - BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 2), - BPF_ALU64_REG(BPF_SUB, BPF_REG_6, BPF_REG_7), - /* Bounds-check the result */ - BPF_JMP_IMM(BPF_JSGE, BPF_REG_6, 0, 1), - BPF_EXIT_INSN(), - /* Add it to the packet pointer */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6), - /* Check bounds and perform a read */ - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - /* Calculated offset in R6 has unknown value, but known - * alignment of 4. - */ - {6, "R2", "pkt(r=8)"}, - {8, "R6", "var_off=(0x0; 0x3fc)"}, - /* Adding 14 makes R6 be (4n+2) */ - {9, "R6", "var_off=(0x2; 0x7fc)"}, - /* New unknown value in R7 is (4n) */ - {10, "R7", "var_off=(0x0; 0x3fc)"}, - /* Subtracting it from R6 blows our unsigned bounds */ - {11, "R6", "var_off=(0x2; 0xfffffffffffffffc)"}, - /* Checked s>= 0 */ - {14, "R6", "var_off=(0x2; 0x7fc)"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off (0) - * which is 2. Then the variable offset is (4n+2), so - * the total offset is 4-byte aligned and meets the - * load's requirements. - */ - {20, "R5", "var_off=(0x2; 0x7fc)"}, - }, - }, - { - .descr = "pointer variable subtraction", - .insns = { - /* Create an unknown offset, (4n+2)-aligned and bounded - * to [14,74] - */ - LOAD_UNKNOWN(BPF_REG_6), - BPF_MOV64_REG(BPF_REG_7, BPF_REG_6), - BPF_ALU64_IMM(BPF_AND, BPF_REG_6, 0xf), - BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 14), - /* Subtract it from the packet pointer */ - BPF_MOV64_REG(BPF_REG_5, BPF_REG_2), - BPF_ALU64_REG(BPF_SUB, BPF_REG_5, BPF_REG_6), - /* Create another unknown, (4n)-aligned and >= 74. - * That in fact means >= 76, since 74 % 4 == 2 - */ - BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 2), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, 76), - /* Add it to the packet pointer */ - BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_7), - /* Check bounds and perform a read */ - BPF_MOV64_REG(BPF_REG_4, BPF_REG_5), - BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4), - BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1), - BPF_EXIT_INSN(), - BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0), - BPF_EXIT_INSN(), - }, - .prog_type = BPF_PROG_TYPE_SCHED_CLS, - .matches = { - /* Calculated offset in R6 has unknown value, but known - * alignment of 4. - */ - {6, "R2", "pkt(r=8)"}, - {9, "R6", "var_off=(0x0; 0x3c)"}, - /* Adding 14 makes R6 be (4n+2) */ - {10, "R6", "var_off=(0x2; 0x7c)"}, - /* Subtracting from packet pointer overflows ubounds */ - {13, "R5", "var_off=(0xffffffffffffff82; 0x7c)"}, - /* New unknown value in R7 is (4n), >= 76 */ - {14, "R7", "var_off=(0x0; 0x7fc)"}, - /* Adding it to packet pointer gives nice bounds again */ - {16, "R5", "var_off=(0x2; 0x7fc)"}, - /* At the time the word size load is performed from R5, - * its total fixed offset is NET_IP_ALIGN + reg->off (0) - * which is 2. Then the variable offset is (4n+2), so - * the total offset is 4-byte aligned and meets the - * load's requirements. - */ - {20, "R5", "var_off=(0x2; 0x7fc)"}, - }, - }, -}; - -static int probe_filter_length(const struct bpf_insn *fp) -{ - int len; - - for (len = MAX_INSNS - 1; len > 0; --len) - if (fp[len].code != 0 || fp[len].imm != 0) - break; - return len + 1; -} - -static char bpf_vlog[32768]; - -static int do_test_single(struct bpf_align_test *test) -{ - struct bpf_insn *prog = test->insns; - int prog_type = test->prog_type; - char bpf_vlog_copy[32768]; - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .prog_flags = BPF_F_STRICT_ALIGNMENT, - .log_buf = bpf_vlog, - .log_size = sizeof(bpf_vlog), - .log_level = 2, - ); - const char *main_pass_start = "0: R1=ctx() R10=fp0"; - const char *line_ptr; - int cur_line = -1; - int prog_len, i; - char *start; - int fd_prog; - int ret; - - prog_len = probe_filter_length(prog); - fd_prog = bpf_prog_load(prog_type ? : BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", - prog, prog_len, &opts); - if (fd_prog < 0 && test->result != REJECT) { - printf("Failed to load program.\n"); - printf("%s", bpf_vlog); - ret = 1; - } else if (fd_prog >= 0 && test->result == REJECT) { - printf("Unexpected success to load!\n"); - printf("%s", bpf_vlog); - ret = 1; - close(fd_prog); - } else { - ret = 0; - /* We make a local copy so that we can strtok() it */ - strncpy(bpf_vlog_copy, bpf_vlog, sizeof(bpf_vlog_copy)); - start = strstr(bpf_vlog_copy, main_pass_start); - if (!start) { - ret = 1; - printf("Can't find initial line '%s'\n", main_pass_start); - goto out; - } - line_ptr = strtok(start, "\n"); - for (i = 0; i < MAX_MATCHES; i++) { - struct bpf_reg_match m = test->matches[i]; - const char *p; - int tmp; - - if (!m.match) - break; - while (line_ptr) { - cur_line = -1; - sscanf(line_ptr, "%u: ", &cur_line); - if (cur_line == -1) - sscanf(line_ptr, "from %u to %u: ", &tmp, &cur_line); - if (cur_line == m.line) - break; - line_ptr = strtok(NULL, "\n"); - } - if (!line_ptr) { - printf("Failed to find line %u for match: %s=%s\n", - m.line, m.reg, m.match); - ret = 1; - printf("%s", bpf_vlog); - break; - } - /* Check the next line as well in case the previous line - * did not have a corresponding bpf insn. Example: - * func#0 @0 - * 0: R1=ctx() R10=fp0 - * 0: (b7) r3 = 2 ; R3_w=2 - * - * Sometimes it's actually two lines below, e.g. when - * searching for "6: R3_w=scalar(umax=255,var_off=(0x0; 0xff))": - * from 4 to 6: R0_w=pkt(off=8,r=8) R1=ctx() R2_w=pkt(r=8) R3_w=pkt_end() R10=fp0 - * 6: R0_w=pkt(off=8,r=8) R1=ctx() R2_w=pkt(r=8) R3_w=pkt_end() R10=fp0 - * 6: (71) r3 = *(u8 *)(r2 +0) ; R2_w=pkt(r=8) R3_w=scalar(umax=255,var_off=(0x0; 0xff)) - */ - while (!(p = strstr(line_ptr, m.reg)) || !strstr(p, m.match)) { - cur_line = -1; - line_ptr = strtok(NULL, "\n"); - sscanf(line_ptr ?: "", "%u: ", &cur_line); - if (!line_ptr || cur_line != m.line) - break; - } - if (cur_line != m.line || !line_ptr || !(p = strstr(line_ptr, m.reg)) || !strstr(p, m.match)) { - printf("Failed to find match %u: %s=%s\n", m.line, m.reg, m.match); - ret = 1; - printf("%s", bpf_vlog); - break; - } - } -out: - if (fd_prog >= 0) - close(fd_prog); - } - return ret; -} - -void test_align(void) -{ - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - struct bpf_align_test *test = &tests[i]; - - if (!test__start_subtest(test->descr)) - continue; - - ASSERT_OK(do_test_single(test), test->descr); - } -} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c index d98577a6babc..1ad5d03d07ad 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c @@ -222,7 +222,7 @@ static void test_store_release(struct arena_atomics *skel) "store_release64_result"); } -void test_arena_atomics(void) +void serial_test_arena_atomics(void) { struct arena_atomics *skel; int err; diff --git a/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c b/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c new file mode 100644 index 000000000000..01fcf4965ea4 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <bpf/bpf.h> +#include <errno.h> +#include <sys/mman.h> +#include <unistd.h> + +#define ARENA_PAGES 32 + +static char log_buf[16384]; + +static void test_arena_direct_value_one_past_end(void) +{ + char expected[128]; + __u32 arena_sz = ARENA_PAGES * getpagesize(); + struct bpf_insn insns[] = { + BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_VALUE, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); + void *arena; + int map_fd, prog_fd; + + map_opts.map_flags = BPF_F_MMAPABLE; + prog_opts.log_buf = log_buf; + prog_opts.log_size = sizeof(log_buf); + prog_opts.log_level = 1; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena_direct_value", + 0, 0, ARENA_PAGES, &map_opts); + if (map_fd < 0) { + if (errno == EOPNOTSUPP) { + test__skip(); + return; + } + ASSERT_GE(map_fd, 0, "bpf_map_create"); + return; + } + + arena = mmap(NULL, arena_sz, PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, 0); + if (!ASSERT_NEQ(arena, MAP_FAILED, "arena_mmap")) + goto cleanup; + + insns[0].imm = map_fd; + insns[1].imm = arena_sz; + + prog_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT, + "arena_direct_value", "GPL", insns, + ARRAY_SIZE(insns), &prog_opts); + if (!ASSERT_LT(prog_fd, 0, "prog_load")) { + close(prog_fd); + goto cleanup; + } + + snprintf(expected, sizeof(expected), + "invalid access to map value pointer, value_size=0 off=%u", + arena_sz); + ASSERT_HAS_SUBSTR(log_buf, expected, "verifier_log"); + +cleanup: + if (arena != MAP_FAILED) + munmap(arena, arena_sz); + close(map_fd); +} + +void serial_test_arena_direct_value(void) +{ + if (test__start_subtest("one_past_end")) + test_arena_direct_value_one_past_end(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_htab.c b/tools/testing/selftests/bpf/prog_tests/arena_htab.c index d69fd2465f53..91ccf0402980 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_htab.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_htab.c @@ -81,7 +81,7 @@ static void test_arena_htab_asm(void) arena_htab_asm__destroy(skel); } -void test_arena_htab(void) +void serial_test_arena_htab(void) { if (test__start_subtest("arena_htab_llvm")) test_arena_htab_llvm(); diff --git a/tools/testing/selftests/bpf/prog_tests/arena_list.c b/tools/testing/selftests/bpf/prog_tests/arena_list.c index 4f2866a615ce..2648e06f53d0 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_list.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_list.c @@ -68,7 +68,7 @@ out: arena_list__destroy(skel); } -void test_arena_list(void) +void serial_test_arena_list(void) { if (test__start_subtest("arena_list_1")) test_arena_list_add_del(1, false); diff --git a/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c b/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file mode 100644 index 000000000000..14c2d1a1d673 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <sys/user.h> +#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */ +#include <unistd.h> +#define PAGE_SIZE getpagesize() +#endif + +#include "arena_mem_usage.skel.h" + +/* + * arena_map_mem_usage() is surfaced to user space through the map's + * /proc/<pid>/fdinfo/<fd> "memlock:" line (the same value bpftool map show + * prints). Read it directly so the test has no external dependency. + */ +static long map_memlock(int map_fd) +{ + char path[64], line[128]; + long memlock = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", map_fd); + f = fopen(path, "r"); + if (!ASSERT_OK_PTR(f, "open_fdinfo")) + return -1; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "memlock:\t%ld", &memlock) == 1) + break; + } + fclose(f); + ASSERT_NEQ(memlock, -1, "parse_memlock"); + return memlock; +} + +static int run(struct bpf_program *prog, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + + if (!ASSERT_OK(err, name)) + return -1; + if (!ASSERT_OK(opts.retval, name)) + return -1; + return 0; +} + +void serial_test_arena_mem_usage(void) +{ + struct arena_mem_usage *skel; + const long ps = PAGE_SIZE; + char *base; + size_t sz; + int fd, i; + + skel = arena_mem_usage__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_load")) + return; + fd = bpf_map__fd(skel->maps.arena); + + /* Fresh arena: no data pages, and the scratch page is not counted. */ + ASSERT_EQ(map_memlock(fd), 0, "initial"); + + /* BPF-side allocation of 17 pages. */ + skel->bss->alloc_cnt = 17; + if (run(skel->progs.alloc, "alloc")) + goto out; + /* + * A NULL ptr means bpf_arena_alloc_pages() itself failed (e.g. the host + * is under memory pressure), not a miscount -- flag it distinctly so a + * red CI run is not mistaken for a counting bug. + */ + if (!ASSERT_OK_PTR(skel->bss->ptr, "arena_alloc_pages")) + goto out; + ASSERT_EQ(map_memlock(fd), 17 * ps, "after_alloc"); + + /* Free a single page (arena_free_pages page_cnt==1 path). */ + skel->bss->free_byte_off = 0; + skel->bss->free_cnt = 1; + if (run(skel->progs.free_pages, "free_one")) + goto out; + ASSERT_EQ(map_memlock(fd), 16 * ps, "after_free_one"); + + /* Free ten pages in one call (bulk path); only the freed pages count. */ + skel->bss->free_byte_off = 1 * ps; + skel->bss->free_cnt = 10; + if (run(skel->progs.free_pages, "free_bulk")) + goto out; + ASSERT_EQ(map_memlock(fd), 6 * ps, "after_free_bulk"); + + /* Free the remaining six -> arena empty again. */ + skel->bss->free_byte_off = 11 * ps; + skel->bss->free_cnt = 6; + if (run(skel->progs.free_pages, "free_rest")) + goto out; + ASSERT_EQ(map_memlock(fd), 0, "after_free_rest"); + + /* + * User-space fault-in: touching unallocated arena pages allocates them + * through arena_vm_fault(). libbpf mmap()s the arena at map_extra during + * load, so bpf_map__initial_value() hands back that base. + */ + base = bpf_map__initial_value(skel->maps.arena, &sz); + if (!ASSERT_OK_PTR(base, "arena_base")) + goto out; + for (i = 0; i < 8; i++) + base[i * ps] = 1; + ASSERT_EQ(map_memlock(fd), 8 * ps, "after_faultin"); + + /* + * Free the faulted-in pages from BPF. They are mapped into the user vma + * (elevated refcount), so this also exercises the zap path. + */ + skel->bss->ptr = base; + skel->bss->free_byte_off = 0; + skel->bss->free_cnt = 8; + if (run(skel->progs.free_pages, "free_faulted")) + goto out; + ASSERT_EQ(map_memlock(fd), 0, "after_free_faulted"); +out: + arena_mem_usage__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c b/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c index 693fd86fbde6..545b05d7a0aa 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c @@ -5,13 +5,6 @@ #include <sys/sysinfo.h> struct __qspinlock { int val; }; -typedef struct __qspinlock arena_spinlock_t; - -struct arena_qnode { - unsigned long next; - int count; - int locked; -}; #include "arena_spin_lock.skel.h" @@ -108,7 +101,7 @@ end: return; } -void test_arena_spin_lock(void) +void serial_test_arena_spin_lock(void) { repeat = 1000; if (test__start_subtest("arena_spin_lock_1")) diff --git a/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c b/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c index f81a0c066505..0c1c6cbfa0f1 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c @@ -23,7 +23,7 @@ static void test_arena_str(void) arena_strsearch__destroy(skel); } -void test_arena_strsearch(void) +void serial_test_arena_strsearch(void) { if (test__start_subtest("arena_strsearch")) test_arena_str(); diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c index 9e77e5da7097..e8c1a619e330 100644 --- a/tools/testing/selftests/bpf/prog_tests/attach_probe.c +++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c @@ -123,6 +123,140 @@ cleanup: test_attach_probe_manual__destroy(skel); } +/* manual attach address-based kprobe/kretprobe testings */ +static void test_attach_kprobe_by_addr(enum probe_attach_mode attach_mode) +{ + LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); + struct test_attach_probe_manual *skel; + unsigned long func_addr; + + if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) + return; + + func_addr = ksym_get_addr(SYS_NANOSLEEP_KPROBE_NAME); + if (!ASSERT_NEQ(func_addr, 0UL, "func_addr")) + return; + + skel = test_attach_probe_manual__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_kprobe_manual_open_and_load")) + return; + + kprobe_opts.attach_mode = attach_mode; + kprobe_opts.retprobe = false; + kprobe_opts.offset = func_addr; + skel->links.handle_kprobe = + bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe, + NULL, &kprobe_opts); + if (!ASSERT_OK_PTR(skel->links.handle_kprobe, "attach_kprobe_by_addr")) + goto cleanup; + + kprobe_opts.retprobe = true; + skel->links.handle_kretprobe = + bpf_program__attach_kprobe_opts(skel->progs.handle_kretprobe, + NULL, &kprobe_opts); + if (!ASSERT_OK_PTR(skel->links.handle_kretprobe, "attach_kretprobe_by_addr")) + goto cleanup; + + /* trigger & validate kprobe && kretprobe */ + usleep(1); + + ASSERT_EQ(skel->bss->kprobe_res, 1, "check_kprobe_res"); + ASSERT_EQ(skel->bss->kretprobe_res, 2, "check_kretprobe_res"); + +cleanup: + test_attach_probe_manual__destroy(skel); +} + +/* reject legacy address-based kprobe attach */ +static void test_attach_kprobe_legacy_by_addr_reject(void) +{ + LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); + struct test_attach_probe_manual *skel; + unsigned long func_addr; + + if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) + return; + + func_addr = ksym_get_addr(SYS_NANOSLEEP_KPROBE_NAME); + if (!ASSERT_NEQ(func_addr, 0UL, "func_addr")) + return; + + skel = test_attach_probe_manual__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_kprobe_manual_open_and_load")) + return; + + kprobe_opts.attach_mode = PROBE_ATTACH_MODE_LEGACY; + kprobe_opts.offset = func_addr; + skel->links.handle_kprobe = + bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe, + NULL, &kprobe_opts); + ASSERT_ERR_PTR(skel->links.handle_kprobe, "attach_kprobe_legacy_by_addr"); + ASSERT_EQ(libbpf_get_error(skel->links.handle_kprobe), + -EOPNOTSUPP, "attach_kprobe_legacy_by_addr_err"); + + test_attach_probe_manual__destroy(skel); +} + +/* + * bpf_fentry_shadow_test exists in both vmlinux (net/bpf/test_run.c) and + * bpf_testmod (bpf_testmod.c). When bpf_testmod is loaded the symbol is + * duplicated. Test that kprobe attachment handles this correctly: + * - Unqualified name ("bpf_fentry_shadow_test") attaches to vmlinux. + * - MOD:SYM name ("bpf_testmod:bpf_fentry_shadow_test") attaches to module. + * + * Note: bpf_fentry_shadow_test is not invoked via test_run, so we only + * verify that attach and detach succeed without triggering the probe. + */ +static void test_attach_probe_dup_sym(enum probe_attach_mode attach_mode) +{ + DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); + struct bpf_link *kprobe_link, *kretprobe_link; + struct test_attach_probe_manual *skel; + + skel = test_attach_probe_manual__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_dup_sym_open_and_load")) + return; + + kprobe_opts.attach_mode = attach_mode; + + /* Unqualified: should attach to vmlinux symbol */ + kprobe_opts.retprobe = false; + kprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe, + "bpf_fentry_shadow_test", + &kprobe_opts); + if (!ASSERT_OK_PTR(kprobe_link, "attach_kprobe_vmlinux")) + goto cleanup; + bpf_link__destroy(kprobe_link); + + kprobe_opts.retprobe = true; + kretprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kretprobe, + "bpf_fentry_shadow_test", + &kprobe_opts); + if (!ASSERT_OK_PTR(kretprobe_link, "attach_kretprobe_vmlinux")) + goto cleanup; + bpf_link__destroy(kretprobe_link); + + /* MOD:SYM qualified: should attach to module symbol */ + kprobe_opts.retprobe = false; + kprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe, + "bpf_testmod:bpf_fentry_shadow_test", + &kprobe_opts); + if (!ASSERT_OK_PTR(kprobe_link, "attach_kprobe_module")) + goto cleanup; + bpf_link__destroy(kprobe_link); + + kprobe_opts.retprobe = true; + kretprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kretprobe, + "bpf_testmod:bpf_fentry_shadow_test", + &kprobe_opts); + if (!ASSERT_OK_PTR(kretprobe_link, "attach_kretprobe_module")) + goto cleanup; + bpf_link__destroy(kretprobe_link); + +cleanup: + test_attach_probe_manual__destroy(skel); +} + /* attach uprobe/uretprobe long event name testings */ static void test_attach_uprobe_long_event_name(void) { @@ -220,11 +354,73 @@ static void test_attach_kprobe_write_ctx(void) kprobe_write_ctx__destroy(skel); } + +static void test_freplace_kprobe_write_ctx(void) +{ + struct bpf_program *prog_kprobe, *prog_ext, *prog_fentry; + struct kprobe_write_ctx *skel_kprobe, *skel_ext = NULL; + struct bpf_link *link_kprobe = NULL, *link_ext = NULL; + int err, prog_fd; + LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); + + skel_kprobe = kprobe_write_ctx__open(); + if (!ASSERT_OK_PTR(skel_kprobe, "kprobe_write_ctx__open kprobe")) + return; + + prog_kprobe = skel_kprobe->progs.kprobe_dummy; + bpf_program__set_autoload(prog_kprobe, true); + + prog_fentry = skel_kprobe->progs.fentry; + bpf_program__set_autoload(prog_fentry, true); + + err = kprobe_write_ctx__load(skel_kprobe); + if (!ASSERT_OK(err, "kprobe_write_ctx__load kprobe")) + goto out; + + skel_ext = kprobe_write_ctx__open(); + if (!ASSERT_OK_PTR(skel_ext, "kprobe_write_ctx__open ext")) + goto out; + + prog_ext = skel_ext->progs.freplace_kprobe; + bpf_program__set_autoload(prog_ext, true); + + prog_fd = bpf_program__fd(skel_kprobe->progs.kprobe_write_ctx); + bpf_program__set_attach_target(prog_ext, prog_fd, "kprobe_write_ctx"); + + err = kprobe_write_ctx__load(skel_ext); + if (!ASSERT_OK(err, "kprobe_write_ctx__load ext")) + goto out; + + prog_fd = bpf_program__fd(prog_kprobe); + link_ext = bpf_program__attach_freplace(prog_ext, prog_fd, "kprobe_dummy"); + ASSERT_ERR_PTR(link_ext, "bpf_program__attach_freplace link"); + ASSERT_EQ(libbpf_get_error(link_ext), -EINVAL, "bpf_program__attach_freplace error"); + + link_kprobe = bpf_program__attach_kprobe_opts(prog_kprobe, "bpf_fentry_test1", + &kprobe_opts); + if (!ASSERT_OK_PTR(link_kprobe, "bpf_program__attach_kprobe_opts")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog_fentry), &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + +out: + bpf_link__destroy(link_ext); + bpf_link__destroy(link_kprobe); + kprobe_write_ctx__destroy(skel_ext); + kprobe_write_ctx__destroy(skel_kprobe); +} #else static void test_attach_kprobe_write_ctx(void) { test__skip(); } + +static void test_freplace_kprobe_write_ctx(void) +{ + test__skip(); +} #endif static void test_attach_probe_auto(struct test_attach_probe *skel) @@ -416,6 +612,21 @@ void test_attach_probe(void) test_attach_probe_manual(PROBE_ATTACH_MODE_PERF); if (test__start_subtest("manual-link")) test_attach_probe_manual(PROBE_ATTACH_MODE_LINK); + if (test__start_subtest("kprobe-perf-by-addr")) + test_attach_kprobe_by_addr(PROBE_ATTACH_MODE_PERF); + if (test__start_subtest("kprobe-link-by-addr")) + test_attach_kprobe_by_addr(PROBE_ATTACH_MODE_LINK); + if (test__start_subtest("kprobe-legacy-by-addr-reject")) + test_attach_kprobe_legacy_by_addr_reject(); + + if (test__start_subtest("dup-sym-default")) + test_attach_probe_dup_sym(PROBE_ATTACH_MODE_DEFAULT); + if (test__start_subtest("dup-sym-legacy")) + test_attach_probe_dup_sym(PROBE_ATTACH_MODE_LEGACY); + if (test__start_subtest("dup-sym-perf")) + test_attach_probe_dup_sym(PROBE_ATTACH_MODE_PERF); + if (test__start_subtest("dup-sym-link")) + test_attach_probe_dup_sym(PROBE_ATTACH_MODE_LINK); if (test__start_subtest("auto")) test_attach_probe_auto(skel); @@ -434,6 +645,8 @@ void test_attach_probe(void) test_attach_kprobe_long_event_name(); if (test__start_subtest("kprobe-write-ctx")) test_attach_kprobe_write_ctx(); + if (test__start_subtest("freplace-kprobe-write-ctx")) + test_freplace_kprobe_write_ctx(); cleanup: test_attach_probe__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_attr_size.c b/tools/testing/selftests/bpf/prog_tests/bpf_attr_size.c new file mode 100644 index 000000000000..87842c4347a6 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_attr_size.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Google LLC */ +#include <linux/bpf.h> +#include <unistd.h> +#include <sys/syscall.h> +#include <test_progs.h> +#include <cgroup_helpers.h> +#include "cgroup_skb_direct_packet_access.skel.h" + +#define OLD_QUERY_SIZE offsetofend(union bpf_attr, query.prog_cnt) +#define FULL_QUERY_SIZE offsetofend(union bpf_attr, query.revision) + +static void test_query_size_boundaries(void) +{ + struct cgroup_skb_direct_packet_access *skel; + struct bpf_link *link = NULL; + union bpf_attr attr; + int cg_fd = -1; + int err; + + skel = cgroup_skb_direct_packet_access__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_load")) + return; + + cg_fd = test__join_cgroup("/attr_size_cg"); + if (!ASSERT_GE(cg_fd, 0, "join_cgroup")) + goto cleanup; + + link = bpf_program__attach_cgroup(skel->progs.direct_packet_access, + cg_fd); + if (!ASSERT_OK_PTR(link, "cg_attach")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.query.target_fd = cg_fd; + attr.query.attach_type = BPF_CGROUP_INET_INGRESS; + attr.query.revision = 0xdeadbeefdeadbeefULL; + + err = syscall(__NR_bpf, BPF_PROG_QUERY, &attr, OLD_QUERY_SIZE); + if (ASSERT_OK(err, "query_old_size")) { + ASSERT_EQ(attr.query.prog_cnt, 1, "prog_cnt_written_old"); + ASSERT_EQ(attr.query.revision, 0xdeadbeefdeadbeefULL, + "revision_not_written_old"); + } + + memset(&attr, 0, sizeof(attr)); + attr.query.target_fd = cg_fd; + attr.query.attach_type = BPF_CGROUP_INET_INGRESS; + + err = syscall(__NR_bpf, BPF_PROG_QUERY, &attr, FULL_QUERY_SIZE); + if (!ASSERT_OK(err, "query_full_size")) + goto cleanup; + + ASSERT_EQ(attr.query.prog_cnt, 1, "prog_cnt_written"); + ASSERT_GT(attr.query.revision, 0, "revision_written"); + +cleanup: + if (link) + bpf_link__destroy(link); + if (cg_fd >= 0) + close(cg_fd); + cgroup_skb_direct_packet_access__destroy(skel); +} + +static void test_map_info_tail_zero(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, map_opts); + struct bpf_map_info_fake { + __u8 info[offsetofend(struct bpf_map_info, hash_size)]; + __u32 pad; + } info = { + .pad = 1, + }; + int map_fd, err; + __u32 info_len; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr", sizeof(int), 1, 1, &map_opts); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) + return; + + info_len = sizeof(info); + err = bpf_obj_get_info_by_fd(map_fd, &info, &info_len); + ASSERT_EQ(err, -E2BIG, "bpf_obj_get_info_by_fd"); + + close(map_fd); +} + +static void test_prog_info_tail_zero(void) +{ + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_prog_info_fake { + __u8 info[offsetofend(struct bpf_prog_info, attach_btf_id)]; + __u32 pad; + } info = { + .pad = 1, + }; + int prog_fd, err; + __u32 info_len; + + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "test_prog", "GPL", insns, + ARRAY_SIZE(insns), &prog_opts); + if (!ASSERT_GE(prog_fd, 0, "bpf_prog_load")) + return; + + info_len = sizeof(info); + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); + ASSERT_EQ(err, -E2BIG, "bpf_obj_get_info_by_fd"); + + close(prog_fd); +} + +void test_bpf_attr_size(void) +{ + if (test__start_subtest("query_size_boundaries")) + test_query_size_boundaries(); + if (test__start_subtest("map_info_tail_zero")) + test_map_info_tail_zero(); + if (test__start_subtest("prog_info_tail_zero")) + test_prog_info_tail_zero(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c b/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c index 75f4dff7d042..fa484d00a7a5 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c @@ -6,6 +6,7 @@ #include <sys/syscall.h> #include <sys/mman.h> #include <unistd.h> +#include <linux/compiler.h> #include <test_progs.h> #include <network_helpers.h> #include <bpf/btf.h> @@ -105,6 +106,11 @@ static void kprobe_multi_link_api_subtest(void) unsigned long long addrs[8]; __u64 cookies[8]; + if (!env.has_testmod) { + test__skip(); + return; + } + if (!ASSERT_OK(load_kallsyms(), "load_kallsyms")) goto cleanup; @@ -192,6 +198,11 @@ static void kprobe_multi_attach_api_subtest(void) }; __u64 cookies[8]; + if (!env.has_testmod) { + test__skip(); + return; + } + skel = kprobe_multi__open_and_load(); if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load")) goto cleanup; @@ -241,10 +252,17 @@ cleanup: kprobe_multi__destroy(skel); } -/* defined in prog_tests/uprobe_multi_test.c */ -void uprobe_multi_func_1(void); -void uprobe_multi_func_2(void); -void uprobe_multi_func_3(void); +/* + * Weak uprobe target stubs. noinline is required because + * uprobe_multi_test_run() takes their addresses to configure the BPF + * program's attachment points; an inlined function has no stable + * address in the binary to probe. The strong definitions in + * uprobe_multi_test.c take precedence when that translation unit is + * linked. + */ +noinline __weak void uprobe_multi_func_1(void) { asm volatile (""); } +noinline __weak void uprobe_multi_func_2(void) { asm volatile (""); } +noinline __weak void uprobe_multi_func_3(void) { asm volatile (""); } static void uprobe_multi_test_run(struct uprobe_multi *skel) { @@ -421,11 +439,12 @@ cleanup: bpf_link__destroy(link3); } -static void burn_cpu(void) +static void burn_cpu(long loops) { - volatile int j = 0; + long j = 0; cpu_set_t cpu_set; - int i, err; + long i; + int err; /* generate some branches on cpu 0 */ CPU_ZERO(&cpu_set); @@ -433,9 +452,10 @@ static void burn_cpu(void) err = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set), &cpu_set); ASSERT_OK(err, "set_thread_affinity"); - /* spin the loop for a while (random high number) */ - for (i = 0; i < 1000000; ++i) + for (i = 0; i < loops; ++i) { ++j; + barrier(); + } } static void pe_subtest(struct test_bpf_cookie *skel) @@ -451,7 +471,7 @@ static void pe_subtest(struct test_bpf_cookie *skel) attr.type = PERF_TYPE_SOFTWARE; attr.config = PERF_COUNT_SW_CPU_CLOCK; attr.sample_period = 100000; - pfd = syscall(__NR_perf_event_open, &attr, -1, 0, -1, PERF_FLAG_FD_CLOEXEC); + pfd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, PERF_FLAG_FD_CLOEXEC); if (!ASSERT_GE(pfd, 0, "perf_fd")) goto cleanup; @@ -460,7 +480,7 @@ static void pe_subtest(struct test_bpf_cookie *skel) if (!ASSERT_OK_PTR(link, "link1")) goto cleanup; - burn_cpu(); /* trigger BPF prog */ + burn_cpu(100000000L); /* trigger BPF prog */ ASSERT_EQ(skel->bss->pe_res, 0x100000, "pe_res1"); @@ -479,7 +499,7 @@ static void pe_subtest(struct test_bpf_cookie *skel) if (!ASSERT_OK_PTR(link, "link2")) goto cleanup; - burn_cpu(); /* trigger BPF prog */ + burn_cpu(100000000L); /* trigger BPF prog */ ASSERT_EQ(skel->bss->pe_res, 0x200000, "pe_res2"); @@ -561,8 +581,6 @@ cleanup: close(fmod_ret_fd); } -int stack_mprotect(void); - static void lsm_subtest(struct test_bpf_cookie *skel) { __u64 cookie; diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_gotox.c b/tools/testing/selftests/bpf/prog_tests/bpf_gotox.c index 75b0cf2467ab..73dc63882b7d 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_gotox.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_gotox.c @@ -317,7 +317,7 @@ static void check_ldimm64_off_load(struct bpf_gotox *skel __always_unused) static int __check_ldimm64_gotox_prog_load(struct bpf_insn *insns, __u32 insn_cnt, - __u32 off1, __u32 off2) + int off1, int off2, int off3) { const __u32 values[] = {5, 7, 9, 11, 13, 15}; const __u32 max_entries = ARRAY_SIZE(values); @@ -349,16 +349,46 @@ static int __check_ldimm64_gotox_prog_load(struct bpf_insn *insns, /* r1 += off2 */ insns[2].imm = off2; + /* r1 = *(r1 + off3) */ + insns[3].off = off3; + ret = prog_load(insns, insn_cnt); close(map_fd); return ret; } -static void reject_offsets(struct bpf_insn *insns, __u32 insn_cnt, __u32 off1, __u32 off2) +static void +allow_offsets(struct bpf_insn *insns, __u32 insn_cnt, int off1, int off2, int off3) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int prog_fd, err; + char s[128] = ""; + + prog_fd = __check_ldimm64_gotox_prog_load(insns, insn_cnt, off1, off2, off3); + snprintf(s, sizeof(s), "__check_ldimm64_gotox_prog_load(%d,%d,%d)", off1, off2, off3); + if (!ASSERT_GE(prog_fd, 0, s)) + return; + + err = bpf_prog_test_run_opts(prog_fd, &topts); + if (!ASSERT_OK(err, "test_run_opts err")) { + close(prog_fd); + return; + } + + if (!ASSERT_EQ(topts.retval, (off1 + off2 + off3) / 8, "test_run_opts retval")) { + close(prog_fd); + return; + } + + close(prog_fd); +} + +static void +reject_offsets(struct bpf_insn *insns, __u32 insn_cnt, int off1, int off2, int off3) { int prog_fd; - prog_fd = __check_ldimm64_gotox_prog_load(insns, insn_cnt, off1, off2); + prog_fd = __check_ldimm64_gotox_prog_load(insns, insn_cnt, off1, off2, off3); if (!ASSERT_EQ(prog_fd, -EACCES, "__check_ldimm64_gotox_prog_load")) close(prog_fd); } @@ -376,7 +406,7 @@ static void check_ldimm64_off_gotox(struct bpf_gotox *skel __always_unused) * The program rewrites the offsets in the instructions below: * r1 = &map + offset1 * r1 += offset2 - * r1 = *r1 + * r1 = *(r1 + offset3) * gotox r1 */ BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_VALUE, 0), @@ -403,43 +433,55 @@ static void check_ldimm64_off_gotox(struct bpf_gotox *skel __always_unused) BPF_MOV64_IMM(BPF_REG_0, 5), BPF_EXIT_INSN(), }; - int prog_fd, err; - __u32 off1, off2; - - /* allow all combinations off1 + off2 < 6 */ - for (off1 = 0; off1 < 6; off1++) { - for (off2 = 0; off1 + off2 < 6; off2++) { - LIBBPF_OPTS(bpf_test_run_opts, topts); - - prog_fd = __check_ldimm64_gotox_prog_load(insns, ARRAY_SIZE(insns), - off1 * 8, off2 * 8); - if (!ASSERT_GE(prog_fd, 0, "__check_ldimm64_gotox_prog_load")) - return; - - err = bpf_prog_test_run_opts(prog_fd, &topts); - if (!ASSERT_OK(err, "test_run_opts err")) { - close(prog_fd); - return; - } - - if (!ASSERT_EQ(topts.retval, off1 + off2, "test_run_opts retval")) { - close(prog_fd); - return; - } - - close(prog_fd); - } - } + int off1, off2, off3; + + /* allow all combinations off1 + off2 + off3 < 6 */ + for (off1 = 0; off1 < 6; off1++) + for (off2 = 0; off1 + off2 < 6; off2++) + for (off3 = 0; off1 + off2 + off3 < 6; off3++) + allow_offsets(insns, ARRAY_SIZE(insns), + off1 * 8, off2 * 8, off3 * 8); + + /* allow for some offsets to be negative */ + allow_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 0, -(8 * 3)); + allow_offsets(insns, ARRAY_SIZE(insns), 8 * 3, -(8 * 3), 0); + allow_offsets(insns, ARRAY_SIZE(insns), 0, 8 * 3, -(8 * 3)); + allow_offsets(insns, ARRAY_SIZE(insns), 8 * 4, 0, -(8 * 2)); + allow_offsets(insns, ARRAY_SIZE(insns), 8 * 4, -(8 * 2), 0); + allow_offsets(insns, ARRAY_SIZE(insns), 0, 8 * 4, -(8 * 2)); + + /* disallow negative sums of offsets */ + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 0, -(8 * 4)); + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, -(8 * 4), 0); + reject_offsets(insns, ARRAY_SIZE(insns), 0, 8 * 3, -(8 * 4)); + + /* disallow the off1 to be negative in any case */ + reject_offsets(insns, ARRAY_SIZE(insns), -8 * 1, 0, 0); + reject_offsets(insns, ARRAY_SIZE(insns), -8 * 1, 8 * 1, 0); + reject_offsets(insns, ARRAY_SIZE(insns), -8 * 1, 8 * 1, 8 * 1); + + /* reject off1 + off2 + off3 >= 6 */ + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 8 * 3, 8 * 0); + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 7, 8 * 0, 8 * 0); + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 0, 8 * 7, 8 * 0); + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 8 * 0, 8 * 3); + reject_offsets(insns, ARRAY_SIZE(insns), 8 * 0, 8 * 3, 8 * 3); + + /* reject (off1 + off2) % 8 != 0, off3 % 8 != 0 */ + reject_offsets(insns, ARRAY_SIZE(insns), 3, 3, 0); + reject_offsets(insns, ARRAY_SIZE(insns), 7, 0, 0); + reject_offsets(insns, ARRAY_SIZE(insns), 0, 7, 0); + reject_offsets(insns, ARRAY_SIZE(insns), 0, 0, 7); +} - /* reject off1 + off2 >= 6 */ - reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 8 * 3); - reject_offsets(insns, ARRAY_SIZE(insns), 8 * 7, 8 * 0); - reject_offsets(insns, ARRAY_SIZE(insns), 8 * 0, 8 * 7); +static void check_ldimm64_off_gotox_llvm(struct bpf_gotox *skel) +{ + __u64 in[] = {0, 1, 2, 3, 4}; + __u64 out[] = {1, 1, 5, 1, 1}; + int i; - /* reject (off1 + off2) % 8 != 0 */ - reject_offsets(insns, ARRAY_SIZE(insns), 3, 3); - reject_offsets(insns, ARRAY_SIZE(insns), 7, 0); - reject_offsets(insns, ARRAY_SIZE(insns), 0, 7); + for (i = 0; i < ARRAY_SIZE(in); i++) + check_simple(skel, skel->progs.load_with_nonzero_offset, in[i], out[i]); } void test_bpf_gotox(void) @@ -496,5 +538,8 @@ void test_bpf_gotox(void) if (test__start_subtest("check-ldimm64-off-gotox")) __subtest(skel, check_ldimm64_off_gotox); + if (test__start_subtest("check-ldimm64-off-gotox-llvm")) + __subtest(skel, check_ldimm64_off_gotox_llvm); + bpf_gotox__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c index 269870bec941..0222a9a5d076 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c @@ -3,7 +3,7 @@ #include <bpf/bpf.h> #include <test_progs.h> -#ifdef __x86_64__ +#if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__) static int map_create(__u32 map_type, __u32 max_entries) { const char *map_name = "insn_array"; diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index 215878ea04de..14d4c1793aed 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -5,24 +5,26 @@ #include "test_bpf_nf.skel.h" #include "test_bpf_nf_fail.skel.h" +#define CT_OPTS_ERROR_GUARD 0x12345678 + static char log_buf[1024 * 1024]; struct { const char *prog_name; const char *err_msg; } test_bpf_nf_fail_tests[] = { - { "alloc_release", "kernel function bpf_ct_release args#0 expected pointer to STRUCT nf_conn but" }, - { "insert_insert", "kernel function bpf_ct_insert_entry args#0 expected pointer to STRUCT nf_conn___init but" }, - { "lookup_insert", "kernel function bpf_ct_insert_entry args#0 expected pointer to STRUCT nf_conn___init but" }, - { "set_timeout_after_insert", "kernel function bpf_ct_set_timeout args#0 expected pointer to STRUCT nf_conn___init but" }, - { "set_status_after_insert", "kernel function bpf_ct_set_status args#0 expected pointer to STRUCT nf_conn___init but" }, - { "change_timeout_after_alloc", "kernel function bpf_ct_change_timeout args#0 expected pointer to STRUCT nf_conn but" }, - { "change_status_after_alloc", "kernel function bpf_ct_change_status args#0 expected pointer to STRUCT nf_conn but" }, + { "alloc_release", "kernel function bpf_ct_release R1 expected pointer to STRUCT nf_conn but" }, + { "insert_insert", "kernel function bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, + { "lookup_insert", "kernel function bpf_ct_insert_entry R1 expected pointer to STRUCT nf_conn___init but" }, + { "set_timeout_after_insert", "kernel function bpf_ct_set_timeout R1 expected pointer to STRUCT nf_conn___init but" }, + { "set_status_after_insert", "kernel function bpf_ct_set_status R1 expected pointer to STRUCT nf_conn___init but" }, + { "change_timeout_after_alloc", "kernel function bpf_ct_change_timeout R1 expected pointer to STRUCT nf_conn but" }, + { "change_status_after_alloc", "kernel function bpf_ct_change_status R1 expected pointer to STRUCT nf_conn but" }, { "write_not_allowlisted_field", "no write support to nf_conn at off" }, - { "lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted arg1" }, - { "lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted arg3" }, - { "xdp_lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted arg1" }, - { "xdp_lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted arg3" }, + { "lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted R2" }, + { "lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted R4" }, + { "xdp_lookup_null_bpf_tuple", "Possibly NULL pointer passed to trusted R2" }, + { "xdp_lookup_null_bpf_opts", "Possibly NULL pointer passed to trusted R4" }, }; enum { @@ -119,6 +121,10 @@ static void test_bpf_nf_ct(int mode) ASSERT_EQ(skel->bss->test_einval_reserved_new, -EINVAL, "Test EINVAL for reserved in new struct not set to 0"); ASSERT_EQ(skel->bss->test_einval_netns_id, -EINVAL, "Test EINVAL for netns_id < -1"); ASSERT_EQ(skel->bss->test_einval_len_opts, -EINVAL, "Test EINVAL for len__opts != NF_BPF_CT_OPTS_SZ"); + ASSERT_EQ(skel->bss->test_einval_len_opts_small_lookup, CT_OPTS_ERROR_GUARD, + "Test no error write for lookup opts__sz before error field"); + ASSERT_EQ(skel->bss->test_einval_len_opts_small_alloc, CT_OPTS_ERROR_GUARD, + "Test no error write for alloc opts__sz before error field"); ASSERT_EQ(skel->bss->test_eproto_l4proto, -EPROTO, "Test EPROTO for l4proto != TCP or UDP"); ASSERT_EQ(skel->bss->test_enonet_netns_id, -ENONET, "Test ENONET for bad but valid netns_id"); ASSERT_EQ(skel->bss->test_enoent_lookup, -ENOENT, "Test ENOENT for failed lookup"); diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c index 730357cd0c9a..6dbd1487343c 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c @@ -8,6 +8,11 @@ #include "bpf_qdisc_fifo.skel.h" #include "bpf_qdisc_fq.skel.h" #include "bpf_qdisc_fail__incompl_ops.skel.h" +#include "bpf_qdisc_fail__invalid_dynptr.skel.h" +#include "bpf_qdisc_fail__invalid_dynptr_slice.skel.h" +#include "bpf_qdisc_fail__invalid_dynptr_cross_frame.skel.h" +#include "bpf_qdisc_fail__untrusted_write.skel.h" +#include "bpf_qdisc_dynptr_use_after_invalidate_clone.skel.h" #define LO_IFINDEX 1 @@ -223,6 +228,11 @@ void test_ns_bpf_qdisc(void) test_qdisc_attach_to_non_root(); if (test__start_subtest("incompl_ops")) test_incompl_ops(); + RUN_TESTS(bpf_qdisc_fail__invalid_dynptr); + RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_cross_frame); + RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_slice); + RUN_TESTS(bpf_qdisc_fail__untrusted_write); + RUN_TESTS(bpf_qdisc_dynptr_use_after_invalidate_clone); } void serial_test_bpf_qdisc_default(void) diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c index f829b6f09bc9..eb05fc82f81b 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c @@ -14,6 +14,7 @@ #include "tcp_ca_incompl_cong_ops.skel.h" #include "tcp_ca_unsupp_cong_op.skel.h" #include "tcp_ca_kfunc.skel.h" +#include "tcp_ca_untrusted_btf_write.skel.h" #include "bpf_cc_cubic.skel.h" static const unsigned int total_bytes = 10 * 1024 * 1024; @@ -112,6 +113,10 @@ static void test_cubic(void) ASSERT_EQ(cubic_skel->bss->bpf_cubic_acked_called, 1, "pkts_acked called"); + ASSERT_TRUE(cubic_skel->bss->nodelay_init_reject, "init reject nodelay option"); + ASSERT_TRUE(cubic_skel->bss->nodelay_cwnd_event_tx_start_reject, + "cwnd_event_tx_start reject nodelay option"); + bpf_link__destroy(link); bpf_cubic__destroy(cubic_skel); } @@ -575,6 +580,15 @@ static void test_tcp_ca_kfunc(void) tcp_ca_kfunc__destroy(skel); } +static void test_untrusted_btf_write(void) +{ + struct tcp_ca_untrusted_btf_write *skel; + + skel = tcp_ca_untrusted_btf_write__open_and_load(); + ASSERT_ERR_PTR(skel, "tcp_ca_untrusted_btf_write__open_and_load"); + tcp_ca_untrusted_btf_write__destroy(skel); +} + static void test_cc_cubic(void) { struct cb_opts cb_opts = { @@ -633,6 +647,8 @@ void test_bpf_tcp_ca(void) test_link_replace(); if (test__start_subtest("tcp_ca_kfunc")) test_tcp_ca_kfunc(); + if (test__start_subtest("untrusted_btf_write")) + test_untrusted_btf_write(); if (test__start_subtest("cc_cubic")) test_cc_cubic(); if (test__start_subtest("dctcp_autoattach_map")) diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 054ecb6b1e9f..67b9015cbd98 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -1924,11 +1924,11 @@ static struct btf_raw_test raw_tests[] = { }, { - .descr = "invalid BTF_INFO", + .descr = "invalid BTF kind", .raw_types = { /* int */ /* [1] */ BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), - BTF_TYPE_ENC(0, 0x20000000, 4), + BTF_TYPE_ENC(0, 0x7f000000, 4), BTF_END_RAW, }, .str_sec = "", @@ -1941,7 +1941,7 @@ static struct btf_raw_test raw_tests[] = { .value_type_id = 1, .max_entries = 4, .btf_load_err = true, - .err_str = "Invalid btf_info", + .err_str = "Invalid kind", }, { @@ -4121,8 +4121,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 1, .value_type_id = 1, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #3, type tag order", @@ -4141,8 +4139,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 1, .value_type_id = 1, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #4, type tag order", @@ -4161,8 +4157,6 @@ static struct btf_raw_test raw_tests[] = { .key_type_id = 1, .value_type_id = 1, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #5, type tag order", @@ -4198,11 +4192,9 @@ static struct btf_raw_test raw_tests[] = { .map_name = "tag_type_check_btf", .key_size = sizeof(int), .value_size = 4, - .key_type_id = 1, - .value_type_id = 1, + .key_type_id = 4, + .value_type_id = 4, .max_entries = 1, - .btf_load_err = true, - .err_str = "Type tags don't precede modifiers", }, { .descr = "type_tag test #7, tag with kflag", @@ -4258,6 +4250,91 @@ static struct btf_raw_test raw_tests[] = { .max_entries = 1, }, +/* + * struct inner { + * struct bpf_spin_lock lock; + * }; + * + * struct value { + * struct bpf_spin_lock lock; + * struct inner nested; + * }; + */ +{ + .descr = "struct test duplicate nested unique fields", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_TBD, 1, 0), + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [3] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_STRUCT_ENC(NAME_TBD, 2, 8), /* [4] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_MEMBER_ENC(NAME_TBD, 3, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0bpf_spin_lock\0val\0inner\0lock\0value\0lock\0nested"), + .btf_load_err = true, +}, + +/* + * struct value { + * struct bpf_refcount a; + * struct bpf_refcount b; + * }; + */ +{ + .descr = "struct test duplicate bpf_refcount fields", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_TBD, 1, 0), + BTF_STRUCT_ENC(NAME_TBD, 2, 8), /* [3] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_MEMBER_ENC(NAME_TBD, 2, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0bpf_refcount\0refs\0value\0a\0b"), + .btf_load_err = true, +}, + +{ + .descr = "struct test repeated fields count overflow", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_TBD, 0, 0), /* [2] */ + BTF_TYPE_TAG_ENC(NAME_TBD, 2), /* [3] */ + BTF_PTR_ENC(3), /* [4] */ + BTF_TYPE_ARRAY_ENC(4, 1, 1), /* [5] */ + BTF_STRUCT_ENC(NAME_TBD, 10, 8), /* [6] */ + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_MEMBER_ENC(NAME_TBD, 5, 0), + BTF_TYPE_ARRAY_ENC(6, 1, 0x1999999aU), /* [7] */ + BTF_STRUCT_ENC(NAME_TBD, 2, 8 + 8 * 0x1999999aU), /* [8] */ + BTF_MEMBER_ENC(NAME_TBD, 4, 0), + BTF_MEMBER_ENC(NAME_TBD, 7, 64), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0prog_test_ref_kfunc\0kptr_untrusted\0elem" + "\0p0\0p1\0p2\0p3\0p4\0p5\0p6\0p7\0p8\0p9" + "\0outer\0trigger\0elems"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "repeat_fields", + .key_size = sizeof(int), + .value_size = 8 + 8 * 0x1999999aU, + .key_type_id = 1, + .value_type_id = 8, + .max_entries = 1, + .btf_load_err = true, +}, }; /* struct btf_raw_test raw_tests[] */ static const char *get_next_str(const char *start, const char *end) @@ -8092,7 +8169,7 @@ static struct btf_dedup_test dedup_tests[] = { static int btf_type_size(const struct btf_type *t) { int base_size = sizeof(struct btf_type); - __u16 vlen = BTF_INFO_VLEN(t->info); + __u32 vlen = BTF_INFO_VLEN(t->info); __u16 kind = BTF_INFO_KIND(t->info); switch (kind) { diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c index 5bc15bb6b7ce..9d6161151593 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c @@ -20,18 +20,22 @@ static void test_split_simple() { btf__add_struct(btf1, "s1", 4); /* [3] struct s1 { */ btf__add_field(btf1, "f1", 1, 0, 0); /* int f1; */ /* } */ + btf__add_typedef(btf1, "t1", 1); /* [4] typedef int */ VALIDATE_RAW_BTF( btf1, "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", "[2] PTR '(anon)' type_id=1", "[3] STRUCT 's1' size=4 vlen=1\n" - "\t'f1' type_id=1 bits_offset=0"); + "\t'f1' type_id=1 bits_offset=0", + "[4] TYPEDEF 't1' type_id=1"); ASSERT_STREQ(btf_type_c_dump(btf1), "\ struct s1 {\n\ int f1;\n\ -};\n\n", "c_dump"); +};\n\ +\n\ +typedef int t1;\n\n", "c_dump"); btf2 = btf__new_empty_split(btf1); if (!ASSERT_OK_PTR(btf2, "empty_split_btf")) @@ -49,39 +53,46 @@ struct s1 {\n\ ASSERT_EQ(btf_is_int(t), true, "int_kind"); ASSERT_STREQ(btf__str_by_offset(btf2, t->name_off), "int", "int_name"); - btf__add_struct(btf2, "s2", 16); /* [4] struct s2 { */ - btf__add_field(btf2, "f1", 6, 0, 0); /* struct s1 f1; */ - btf__add_field(btf2, "f2", 5, 32, 0); /* int f2; */ + btf__add_struct(btf2, "s2", 16); /* [5] struct s2 { */ + btf__add_field(btf2, "f1", 7, 0, 0); /* struct s1 f1; */ + btf__add_field(btf2, "f2", 6, 32, 0); /* int f2; */ btf__add_field(btf2, "f3", 2, 64, 0); /* int *f3; */ /* } */ /* duplicated int */ - btf__add_int(btf2, "int", 4, BTF_INT_SIGNED); /* [5] int */ + btf__add_int(btf2, "int", 4, BTF_INT_SIGNED); /* [6] int */ /* duplicated struct s1 */ - btf__add_struct(btf2, "s1", 4); /* [6] struct s1 { */ - btf__add_field(btf2, "f1", 5, 0, 0); /* int f1; */ + btf__add_struct(btf2, "s1", 4); /* [7] struct s1 { */ + btf__add_field(btf2, "f1", 6, 0, 0); /* int f1; */ /* } */ + /* duplicated typedef t1 */ + btf__add_typedef(btf2, "t1", 6); /* [8] typedef int */ + VALIDATE_RAW_BTF( btf2, "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", "[2] PTR '(anon)' type_id=1", "[3] STRUCT 's1' size=4 vlen=1\n" "\t'f1' type_id=1 bits_offset=0", - "[4] STRUCT 's2' size=16 vlen=3\n" - "\t'f1' type_id=6 bits_offset=0\n" - "\t'f2' type_id=5 bits_offset=32\n" + "[4] TYPEDEF 't1' type_id=1", + "[5] STRUCT 's2' size=16 vlen=3\n" + "\t'f1' type_id=7 bits_offset=0\n" + "\t'f2' type_id=6 bits_offset=32\n" "\t'f3' type_id=2 bits_offset=64", - "[5] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", - "[6] STRUCT 's1' size=4 vlen=1\n" - "\t'f1' type_id=5 bits_offset=0"); + "[6] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[7] STRUCT 's1' size=4 vlen=1\n" + "\t'f1' type_id=6 bits_offset=0", + "[8] TYPEDEF 't1' type_id=6"); ASSERT_STREQ(btf_type_c_dump(btf2), "\ struct s1 {\n\ int f1;\n\ };\n\ \n\ +typedef int t1;\n\ +\n\ struct s1___2 {\n\ int f1;\n\ };\n\ @@ -90,7 +101,9 @@ struct s2 {\n\ struct s1___2 f1;\n\ int f2;\n\ int *f3;\n\ -};\n\n", "c_dump"); +};\n\ +\n\ +typedef int t1___2;\n\n", "c_dump"); err = btf__dedup(btf2, NULL); if (!ASSERT_OK(err, "btf_dedup")) @@ -102,7 +115,8 @@ struct s2 {\n\ "[2] PTR '(anon)' type_id=1", "[3] STRUCT 's1' size=4 vlen=1\n" "\t'f1' type_id=1 bits_offset=0", - "[4] STRUCT 's2' size=16 vlen=3\n" + "[4] TYPEDEF 't1' type_id=1", + "[5] STRUCT 's2' size=16 vlen=3\n" "\t'f1' type_id=3 bits_offset=0\n" "\t'f2' type_id=1 bits_offset=32\n" "\t'f3' type_id=2 bits_offset=64"); @@ -112,6 +126,8 @@ struct s1 {\n\ int f1;\n\ };\n\ \n\ +typedef int t1;\n\ +\n\ struct s2 {\n\ struct s1 f1;\n\ int f2;\n\ @@ -487,9 +503,8 @@ static void test_split_module(void) for (i = 0; i < ARRAY_SIZE(mod_funcs); i++) { const struct btf_param *p; const struct btf_type *t; - __u16 vlen; + __u32 vlen, j; __u32 id; - int j; id = btf__find_by_name_kind(btf1, mod_funcs[i], BTF_KIND_FUNC); if (!ASSERT_GE(id, nr_base_types, "func_id")) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index f1642794f70e..9f1b50e07a29 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -1027,8 +1027,8 @@ static void test_btf_dump_datasec_data(char *str) char license[4] = "GPL"; struct btf_dump *d; - btf = btf__parse("xdping_kern.bpf.o", NULL); - if (!ASSERT_OK_PTR(btf, "xdping_kern.bpf.o BTF not found")) + btf = btf__parse("xdp_dummy.bpf.o", NULL); + if (!ASSERT_OK_PTR(btf, "xdp_dummy.bpf.o BTF not found")) return; d = btf_dump__new(btf, btf_dump_snprintf, str, NULL); diff --git a/tools/testing/selftests/bpf/prog_tests/btf_kind.c b/tools/testing/selftests/bpf/prog_tests/btf_kind.c new file mode 100644 index 000000000000..f61afe6a79a5 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_kind.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ + +#include <test_progs.h> +#include <bpf/btf.h> +#include <bpf/libbpf.h> + +/* Verify kind encoding exists for each kind */ +static void test_btf_kind_encoding(void) +{ + LIBBPF_OPTS(btf_new_opts, opts); + const struct btf_header *hdr; + const void *raw_btf; + struct btf *btf; + __u32 raw_size; + + opts.add_layout = true; + btf = btf__new_empty_opts(&opts); + if (!ASSERT_OK_PTR(btf, "btf_new")) + return; + + raw_btf = btf__raw_data(btf, &raw_size); + if (!ASSERT_OK_PTR(raw_btf, "btf__raw_data")) + return; + + hdr = raw_btf; + + ASSERT_EQ(hdr->layout_off % 4, 0, "layout_aligned"); + ASSERT_EQ(hdr->layout_len, sizeof(struct btf_layout) * NR_BTF_KINDS, + "layout_len"); + ASSERT_EQ(hdr->str_off, hdr->layout_off + hdr->layout_len, "str_after_layout"); + btf__free(btf); + + opts.add_layout = false; + btf = btf__new_empty_opts(&opts); + if (!ASSERT_OK_PTR(btf, "btf_new")) + return; + + raw_btf = btf__raw_data(btf, &raw_size); + if (!ASSERT_OK_PTR(raw_btf, "btf__raw_data")) + return; + + hdr = raw_btf; + + ASSERT_EQ(hdr->layout_off, 0, "no_layout_off"); + ASSERT_EQ(hdr->layout_len, 0, "no_layout_len"); + ASSERT_EQ(hdr->str_off, hdr->type_off + hdr->type_len, "strs_after_types"); + btf__free(btf); +} + +static int write_raw_btf(void *raw_btf, size_t raw_size, char *file) +{ + int fd = mkstemp(file); + ssize_t n; + + if (!ASSERT_OK_FD(fd, "open_raw_btf")) + return -1; + n = write(fd, raw_btf, raw_size); + close(fd); + if (!ASSERT_EQ(n, (ssize_t)raw_size, "write_raw_btf")) + return -1; + return 0; +} + +/* + * Fabricate an unrecognized kind at BTF_KIND_MAX + 1, and after adding + * the appropriate struct/typedefs to the BTF such that it recognizes + * this kind, ensure that parsing of BTF containing the unrecognized kind + * can succeed. + */ +void test_btf_kind_decoding(void) +{ + char btf_kind_file1[] = "/tmp/test_btf_kind.XXXXXX"; + char btf_kind_file2[] = "/tmp/test_btf_kind.XXXXXX"; + char btf_kind_file3[] = "/tmp/test_btf_kind.XXXXXX"; + struct btf *btf = NULL, *new_btf = NULL; + __s32 int_id, unrec_id, id, id2; + LIBBPF_OPTS(btf_new_opts, opts); + struct btf_layout *l; + struct btf_header *hdr; + const void *raw_btf; + struct btf_type *t; + void *new_raw_btf; + void *str_data; + __u32 raw_size; + + opts.add_layout = true; + btf = btf__new_empty_opts(&opts); + if (!ASSERT_OK_PTR(btf, "btf_new")) + return; + + int_id = btf__add_int(btf, "test_char", 1, BTF_INT_CHAR); + if (!ASSERT_GT(int_id, 0, "add_int_id")) + return; + + /* + * Create our type with unrecognized kind by adding a typedef kind + * we will overwrite it with our unrecognized kind value. + */ + unrec_id = btf__add_typedef(btf, "unrec_kind", int_id); + if (!ASSERT_GT(unrec_id, 0, "add_unrec_id")) + return; + + /* + * Add an id after it that we will look up to verify we can parse + * beyond unrecognized kinds. + */ + id = btf__add_typedef(btf, "test_lookup", int_id); + if (!ASSERT_GT(id, 0, "add_test_lookup_id")) + return; + id2 = btf__add_typedef(btf, "test_lookup2", int_id); + if (!ASSERT_GT(id2, 0, "add_test_lookup_id2")) + return; + + raw_btf = (void *)btf__raw_data(btf, &raw_size); + if (!ASSERT_OK_PTR(raw_btf, "btf__raw_data")) + return; + + new_raw_btf = calloc(1, raw_size + sizeof(*l)); + if (!ASSERT_OK_PTR(new_raw_btf, "calloc_raw_btf")) + return; + memcpy(new_raw_btf, raw_btf, raw_size); + + hdr = new_raw_btf; + + /* Move strings to make space for one new layout description */ + raw_size += sizeof(*l); + str_data = new_raw_btf + hdr->hdr_len + hdr->str_off; + memmove(str_data + sizeof(*l), str_data, hdr->str_len); + hdr->str_off += sizeof(*l); + + /* Add new layout description */ + hdr->layout_len += sizeof(*l); + l = new_raw_btf + hdr->hdr_len + hdr->layout_off; + l[NR_BTF_KINDS].info_sz = 0; + l[NR_BTF_KINDS].elem_sz = 0; + l[NR_BTF_KINDS].flags = 0; + + /* Now modify typedef added above to be an unrecognized kind. */ + t = (void *)hdr + hdr->hdr_len + hdr->type_off + sizeof(struct btf_type) + + sizeof(__u32); + t->info = (NR_BTF_KINDS << 24); + + /* Write BTF to a raw file, ready for parsing. */ + if (write_raw_btf(new_raw_btf, raw_size, btf_kind_file1)) + goto out; + + /* + * Verify parsing succeeds, and that we can read type info past + * the unrecognized kind. + */ + new_btf = btf__parse_raw(btf_kind_file1); + if (ASSERT_OK_PTR(new_btf, "btf__parse_raw")) { + ASSERT_EQ(btf__find_by_name(new_btf, "unrec_kind"), unrec_id, + "unrec_kind_found"); + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup", + BTF_KIND_TYPEDEF), id, + "verify_id_lookup"); + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup2", + BTF_KIND_TYPEDEF), id2, + "verify_id2_lookup"); + } + btf__free(new_btf); + new_btf = NULL; + + /* + * Next, change info_sz to equal sizeof(struct btf_type); this means the + * "test_lookup" kind will be reinterpreted as a singular info element + * following the unrecognized kind. + */ + l[NR_BTF_KINDS].info_sz = sizeof(struct btf_type); + if (write_raw_btf(new_raw_btf, raw_size, btf_kind_file2)) + goto out; + + new_btf = btf__parse_raw(btf_kind_file2); + if (ASSERT_OK_PTR(new_btf, "btf__parse_raw")) { + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup", + BTF_KIND_TYPEDEF), -ENOENT, + "verify_id_not_found"); + /* id of "test_lookup2" will be id2 -1 as we have removed one type */ + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup2", + BTF_KIND_TYPEDEF), id2 - 1, + "verify_id_lookup2"); + + } + btf__free(new_btf); + new_btf = NULL; + + /* + * Change elem_sz to equal sizeof(struct btf_type) and set vlen + * associated with unrecognized type to 1; this allows us to verify + * vlen-specified BTF can still be parsed. + */ + l[NR_BTF_KINDS].info_sz = 0; + l[NR_BTF_KINDS].elem_sz = sizeof(struct btf_type); + t->info |= 1; + if (write_raw_btf(new_raw_btf, raw_size, btf_kind_file3)) + goto out; + + new_btf = btf__parse_raw(btf_kind_file3); + if (ASSERT_OK_PTR(new_btf, "btf__parse_raw")) { + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup", + BTF_KIND_TYPEDEF), -ENOENT, + "verify_id_not_found"); + /* id of "test_lookup2" will be id2 -1 as we have removed one type */ + ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup2", + BTF_KIND_TYPEDEF), id2 - 1, + "verify_id_lookup2"); + + } +out: + btf__free(new_btf); + free(new_raw_btf); + unlink(btf_kind_file1); + unlink(btf_kind_file2); + unlink(btf_kind_file3); + btf__free(btf); +} + +void test_btf_kind(void) +{ + if (test__start_subtest("btf_kind_encoding")) + test_btf_kind_encoding(); + if (test__start_subtest("btf_kind_decoding")) + test_btf_kind_decoding(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c b/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c new file mode 100644 index 000000000000..3248bccc3557 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpf/btf.h> + +/* + * A hash map with a key-less BTF (btf_key_type_id == 0) used to be accepted + * and then NULL-deref in btf_type_show() when dumped through bpffs. A fixed + * kernel rejects it at creation; verify that rejection, with a keyed positive + * control so the -EINVAL is about the missing key type and not some unrelated + * failure. + */ +static void check_keyless(int map_type, __u32 map_flags, int btf_fd, int val_id) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + int map_fd; + + opts.map_flags = map_flags; + opts.btf_fd = btf_fd; + opts.btf_value_type_id = val_id; + + /* Positive control: the same map with a real key type is accepted. */ + opts.btf_key_type_id = val_id; + map_fd = bpf_map_create(map_type, "keyed_map", 4, 4, 8, &opts); + if (!ASSERT_GE(map_fd, 0, "keyed create is accepted")) + return; + close(map_fd); + + /* A key-less BTF must be rejected. */ + opts.btf_key_type_id = 0; + map_fd = bpf_map_create(map_type, "keyless_map", 4, 4, 8, &opts); + ASSERT_EQ(map_fd, -EINVAL, "key-less create is rejected"); + if (map_fd >= 0) + close(map_fd); +} + +void test_btf_map_keyless(void) +{ + int btf_fd, val_id; + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return; + + val_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_GT(val_id, 0, "btf__add_int")) + goto out; + + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) + goto out; + btf_fd = btf__fd(btf); + + if (test__start_subtest("hash")) + check_keyless(BPF_MAP_TYPE_HASH, 0, btf_fd, val_id); + if (test__start_subtest("rhash")) + check_keyless(BPF_MAP_TYPE_RHASH, BPF_F_NO_PREALLOC, btf_fd, val_id); +out: + btf__free(btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_sanitize.c b/tools/testing/selftests/bpf/prog_tests/btf_sanitize.c new file mode 100644 index 000000000000..652b51efafc2 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_sanitize.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026, Oracle and/or its affiliates. */ +#include <test_progs.h> +#include <linux/btf.h> +#include "bpf/libbpf_internal.h" +#include "../test_btf.h" +#include "kfree_skb.skel.h" + +#define TYPE_LEN (sizeof(struct btf_type) + sizeof(__u32)) +#define MAX_NR_LAYOUT 2 +#define LAYOUT_LEN (sizeof(struct btf_layout) * MAX_NR_LAYOUT) +#define STR_LEN sizeof("\0int") + +struct layout_btf { + struct btf_header hdr; + __u32 types[TYPE_LEN/sizeof(__u32)]; + struct btf_layout layout[MAX_NR_LAYOUT]; + char strs[STR_LEN]; +}; + +static const struct layout_btf layout_btf = { + .hdr = { + .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(struct btf_header), + .type_off = 0, + .type_len = TYPE_LEN, + .str_off = TYPE_LEN + LAYOUT_LEN, + .str_len = STR_LEN, + .layout_off = TYPE_LEN, + .layout_len = LAYOUT_LEN, + }, + .types = { + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), + }, + .layout = { + { .info_sz = 0, .elem_sz = 0, .flags = 0 }, + { .info_sz = sizeof(__u32), .elem_sz = 0, .flags = 0 }, + }, + .strs = "\0int", +}; + +void test_btf_sanitize_layout(void) +{ + struct btf *orig = NULL, *sanitized = NULL; + struct kern_feature_cache *cache = NULL; + struct kfree_skb *skel = NULL; + const struct btf_header *hdr; + const void *raw; + __u32 raw_sz; + + skel = kfree_skb__open(); + if (!ASSERT_OK_PTR(skel, "kfree_skb_skel")) + return; + orig = btf__new(&layout_btf, sizeof(layout_btf)); + if (!ASSERT_OK_PTR(orig, "btf_new_layout")) + goto out; + raw = btf__raw_data(orig, &raw_sz); + if (!ASSERT_OK_PTR(raw, "btf__raw_data_orig")) + goto out; + hdr = (struct btf_header *)raw; + ASSERT_EQ(hdr->layout_off, TYPE_LEN, "layout_off_nonzero"); + ASSERT_EQ(hdr->layout_len, LAYOUT_LEN, "layout_len_nonzero"); + + cache = calloc(1, sizeof(*cache)); + if (!ASSERT_OK_PTR(cache, "alloc_feat_cache")) + goto out; + for (int i = 0; i < __FEAT_CNT; i++) + cache->res[i] = FEAT_SUPPORTED; + cache->res[FEAT_BTF_LAYOUT] = FEAT_MISSING; + + bpf_object_set_feat_cache(skel->obj, cache); + + if (!ASSERT_FALSE(kernel_supports(skel->obj, FEAT_BTF_LAYOUT), "layout_feature_missing")) + goto out; + if (!ASSERT_TRUE(kernel_supports(skel->obj, FEAT_BTF_FUNC), "other_feature_allowed")) + goto out; + + sanitized = bpf_object__sanitize_btf(skel->obj, orig); + if (!ASSERT_OK_PTR(sanitized, "bpf_object__sanitize_btf")) + goto out; + + raw = btf__raw_data(sanitized, &raw_sz); + if (!ASSERT_OK_PTR(raw, "btf__raw_data_sanitized")) + goto out; + hdr = (struct btf_header *)raw; + ASSERT_EQ(hdr->layout_off, 0, "layout_off_zero"); + ASSERT_EQ(hdr->layout_len, 0, "layout_len_zero"); + ASSERT_EQ(hdr->str_off, TYPE_LEN, "strs_after_types"); + ASSERT_EQ(hdr->str_len, STR_LEN, "strs_len_unchanged"); + ASSERT_EQ(raw_sz, hdr->hdr_len + hdr->type_len + hdr->str_len, "btf_raw_sz_reduced"); +out: + /* This will free the cache we allocated above */ + kfree_skb__destroy(skel); + btf__free(sanitized); + btf__free(orig); +} diff --git a/tools/testing/selftests/bpf/prog_tests/btf_write.c b/tools/testing/selftests/bpf/prog_tests/btf_write.c index 6e36de1302fc..5c84723cf254 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_write.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_write.c @@ -497,10 +497,121 @@ cleanup: btf__free(btf2); } +static void test_btf_add_btf_split() +{ + struct btf *base = NULL, *split1 = NULL, *split2 = NULL; + struct btf *combined = NULL; + int id, err; + + /* Create a base BTF with an INT and a PTR to it */ + base = btf__new_empty(); + if (!ASSERT_OK_PTR(base, "base")) + return; + + id = btf__add_int(base, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(id, 1, "base_int_id"); + id = btf__add_ptr(base, 1); + ASSERT_EQ(id, 2, "base_ptr_id"); + + /* base has 2 types, type IDs 1..2 */ + ASSERT_EQ(btf__type_cnt(base), 3, "base_type_cnt"); + + /* Create split1 on base: a STRUCT referencing base's int (ID 1) */ + split1 = btf__new_empty_split(base); + if (!ASSERT_OK_PTR(split1, "split1")) + goto cleanup; + + id = btf__add_struct(split1, "s1", 4); + /* split types start at base_type_cnt = 3 */ + ASSERT_EQ(id, 3, "split1_struct_id"); + btf__add_field(split1, "x", 1, 0, 0); /* refers to base int */ + + id = btf__add_ptr(split1, 3); + ASSERT_EQ(id, 4, "split1_ptr_id"); /* ptr to the struct (split self-ref) */ + + /* Add a typedef "int_alias" -> base int in split1, which will be + * duplicated in split2 to test that btf__dedup() merges them. + */ + id = btf__add_typedef(split1, "int_alias", 1); + ASSERT_EQ(id, 5, "split1_typedef_id"); + + /* Create split2 on base: a TYPEDEF referencing base's ptr (ID 2) */ + split2 = btf__new_empty_split(base); + if (!ASSERT_OK_PTR(split2, "split2")) + goto cleanup; + + id = btf__add_typedef(split2, "int_ptr", 2); /* refers to base ptr */ + ASSERT_EQ(id, 3, "split2_typedef_id"); + + id = btf__add_struct(split2, "s2", 8); + ASSERT_EQ(id, 4, "split2_struct_id"); + btf__add_field(split2, "p", 3, 0, 0); /* refers to split2's own typedef */ + + /* Same "int_alias" typedef as split1 - should be deduped away */ + id = btf__add_typedef(split2, "int_alias", 1); + ASSERT_EQ(id, 5, "split2_dup_typedef_id"); + + /* Create combined split BTF on same base and merge both */ + combined = btf__new_empty_split(base); + if (!ASSERT_OK_PTR(combined, "combined")) + goto cleanup; + + /* Merge split1: its types (3,4,5) should land at IDs 3,4,5 */ + id = btf__add_btf(combined, split1); + if (!ASSERT_GE(id, 0, "add_split1")) + goto cleanup; + ASSERT_EQ(id, 3, "split1_first_id"); + + /* Merge split2: its types (3,4,5) should be remapped to 6,7,8 */ + id = btf__add_btf(combined, split2); + if (!ASSERT_GE(id, 0, "add_split2")) + goto cleanup; + ASSERT_EQ(id, 6, "split2_first_id"); + + /* Before dedup: base (2) + split1 (3) + split2 (3) = 8 types + void */ + ASSERT_EQ(btf__type_cnt(combined), 9, "pre_dedup_type_cnt"); + + VALIDATE_RAW_BTF( + combined, + /* base types (IDs 1-2) */ + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] PTR '(anon)' type_id=1", + + /* split1 types (IDs 3-5): base refs unchanged */ + "[3] STRUCT 's1' size=4 vlen=1\n" + "\t'x' type_id=1 bits_offset=0", /* refers to base int=1 */ + "[4] PTR '(anon)' type_id=3", /* refers to split1's struct=3 */ + "[5] TYPEDEF 'int_alias' type_id=1", /* refers to base int=1 */ + + /* split2 types (IDs 6-8): remapped from 3,4,5 to 6,7,8 */ + "[6] TYPEDEF 'int_ptr' type_id=2", /* base ptr=2, unchanged */ + "[7] STRUCT 's2' size=8 vlen=1\n" + "\t'p' type_id=6 bits_offset=0", /* split2 typedef: 3->6 */ + "[8] TYPEDEF 'int_alias' type_id=1"); /* dup of [5] */ + + /* Dedup to mirror the bpftool merge flow; should remove the + * duplicate "int_alias" typedef. + */ + err = btf__dedup(combined, NULL); + if (!ASSERT_OK(err, "dedup")) + goto cleanup; + + /* After dedup: one int_alias removed, so 7 types + void */ + ASSERT_EQ(btf__type_cnt(combined), 8, "dedup_type_cnt"); + +cleanup: + btf__free(combined); + btf__free(split2); + btf__free(split1); + btf__free(base); +} + void test_btf_write() { if (test__start_subtest("btf_add")) test_btf_add(); if (test__start_subtest("btf_add_btf")) test_btf_add_btf(); + if (test__start_subtest("btf_add_btf_split")) + test_btf_add_btf_split(); } diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c index c40df623a8f7..490e15e7126d 100644 --- a/tools/testing/selftests/bpf/prog_tests/cb_refs.c +++ b/tools/testing/selftests/bpf/prog_tests/cb_refs.c @@ -11,9 +11,9 @@ struct { const char *prog_name; const char *err_msg; } cb_refs_tests[] = { - { "underflow_prog", "must point to scalar, or struct with scalar" }, - { "leak_prog", "Possibly NULL pointer passed to helper arg2" }, - { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */ + { "underflow_prog", "release kfunc bpf_kfunc_call_test_release expects referenced PTR_TO_BTF_ID passed to R1" }, + { "leak_prog", "Possibly NULL pointer passed to helper R2" }, + { "nested_cb", "Unreleased reference id=5 alloc_insn=2" }, /* alloc_insn=2{4,5} */ { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */ }; diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c index a5afd16705f0..5a1e08d39a06 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c @@ -10,6 +10,17 @@ #include "cgroup_iter_memcg.h" #include "cgroup_iter_memcg.skel.h" +/* + * memcg stats are cached per-cpu and only become visible once the periodic + * flusher runs (FLUSH_TIME, 2s), or once pending updates cross + * MEMCG_CHARGE_BATCH * num_online_cpus(). That threshold grows with the CPU + * count, so on a large machine a single pass does not reach it and + * bpf_mem_cgroup_flush_stats() returns without flushing anything. Retry for + * long enough to cover a flusher cycle. + */ +#define MEMCG_STAT_RETRIES 16 +#define MEMCG_STAT_RETRY_DELAY_US (250 * 1000) + static int read_stats(struct bpf_link *link) { int fd, ret = 0; @@ -35,11 +46,13 @@ static int read_stats(struct bpf_link *link) static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* * Increase memcg anon usage by mapping and writing * to a new anon region. @@ -53,6 +66,12 @@ static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->nr_anon_mapped && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_anon_mapped, 0, "final anon mapped val"); cleanup: @@ -61,6 +80,7 @@ cleanup: static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; char *path; @@ -76,6 +96,7 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) fd = open(path, O_CREAT | O_RDWR, 0644); if (!ASSERT_OK_FD(fd, "open fd")) return; +retry: if (!ASSERT_OK(ftruncate(fd, len), "ftruncate")) goto cleanup_fd; @@ -88,6 +109,13 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup_map; + if ((!memcg_query->nr_file_pages || !memcg_query->nr_file_mapped) && + ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_file_pages, 0, "final file value"); ASSERT_GT(memcg_query->nr_file_mapped, 0, "final file mapped value"); @@ -100,6 +128,7 @@ cleanup_fd: static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; size_t len; int fd; @@ -107,58 +136,39 @@ static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) /* * Increase memcg shmem usage by creating and writing - * to a shmem object. + * to a memfd backed by shmem/tmpfs. */ - fd = shm_open("/tmp_shmem", O_CREAT | O_RDWR, 0644); - if (!ASSERT_OK_FD(fd, "shm_open")) + fd = memfd_create("tmp_shmem", 0); + if (!ASSERT_OK_FD(fd, "memfd_create")) return; +retry: if (!ASSERT_OK(fallocate(fd, 0, 0, len), "fallocate")) goto cleanup; if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; - ASSERT_GT(memcg_query->nr_shmem, 0, "final shmem value"); - -cleanup: - close(fd); - shm_unlink("/tmp_shmem"); -} - -#define NR_PIPES 64 -static void test_kmem(struct bpf_link *link, struct memcg_query *memcg_query) -{ - int fds[NR_PIPES][2], i; - - /* - * Increase kmem value by creating pipes which will allocate some - * kernel buffers. - */ - for (i = 0; i < NR_PIPES; i++) { - if (!ASSERT_OK(pipe(fds[i]), "pipe")) - goto cleanup; + if (!memcg_query->nr_shmem && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + goto retry; } - if (!ASSERT_OK(read_stats(link), "read stats")) - goto cleanup; - - ASSERT_GT(memcg_query->memcg_kmem, 0, "kmem value"); + ASSERT_GT(memcg_query->nr_shmem, 0, "final shmem value"); cleanup: - for (i = i - 1; i >= 0; i--) { - close(fds[i][0]); - close(fds[i][1]); - } + close(fd); } static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* Create region to use for triggering a page fault. */ map = mmap(NULL, len, PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (!ASSERT_NEQ(map, MAP_FAILED, "mmap anon")) @@ -170,6 +180,12 @@ static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->pgfault && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->pgfault, 0, "final pgfault val"); cleanup: @@ -209,8 +225,6 @@ void test_cgroup_iter_memcg(void) test_shmem(link, &skel->data_query->memcg_query); if (test__start_subtest("cgroup_iter_memcg__file")) test_file(link, &skel->data_query->memcg_query); - if (test__start_subtest("cgroup_iter_memcg__kmem")) - test_kmem(link, &skel->data_query->memcg_query); if (test__start_subtest("cgroup_iter_memcg__pgfault")) test_pgfault(link, &skel->data_query->memcg_query); diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c b/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c index d4d583872fa2..d2ccf409dfe3 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c @@ -102,6 +102,82 @@ close_skel: return err; } +/* + * Replacing a link's program (bpf_link_update) must target the correct slot in + * the effective array even when a BPF_F_PREORDER program is attached to the + * same cgroup. All programs here are attached to a single cgroup; "parent" is + * reused only as a third distinct program. + * + * Attach child(1) normally and child_2(2) with BPF_F_PREORDER, so the effective + * order is [2, 1]. Then replace child(1)'s program with parent(3): only the + * non-preorder slot changes, giving [2, 3]. + */ +static int run_link_replace_test(int cgroup_fd, int sock_fd) +{ + LIBBPF_OPTS(bpf_link_create_opts, create_opts); + int err = 0, normal_link = -1, preorder_link = -1; + struct cgroup_preorder *skel = NULL; + enum bpf_attach_type atype; + __u8 *result, buf = 0x00; + socklen_t optlen = 1; + + skel = cgroup_preorder__open_and_load(); + if (!ASSERT_OK_PTR(skel, "cgroup_preorder__open_and_load")) + return -1; + + err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1); + if (!ASSERT_OK(err, "setsockopt")) + goto close_skel; + + atype = bpf_program__expected_attach_type(skel->progs.child); + + create_opts.flags = 0; + normal_link = bpf_link_create(bpf_program__fd(skel->progs.child), + cgroup_fd, atype, &create_opts); + if (!ASSERT_GE(normal_link, 0, "create_normal_link")) { + err = normal_link; + goto close_skel; + } + + create_opts.flags = BPF_F_PREORDER; + preorder_link = bpf_link_create(bpf_program__fd(skel->progs.child_2), + cgroup_fd, atype, &create_opts); + if (!ASSERT_GE(preorder_link, 0, "create_preorder_link")) { + err = preorder_link; + goto close_links; + } + + result = skel->bss->result; + skel->bss->idx = 0; + memset(result, 0, 4); + + err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen); + if (!ASSERT_OK(err, "getsockopt-before")) + goto close_links; + ASSERT_TRUE(result[0] == 2 && result[1] == 1, "order before update"); + + /* Replace the normal link's program child(1) -> parent(3). */ + err = bpf_link_update(normal_link, bpf_program__fd(skel->progs.parent), NULL); + if (!ASSERT_OK(err, "bpf_link_update")) + goto close_links; + + skel->bss->idx = 0; + memset(result, 0, 4); + + err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen); + if (!ASSERT_OK(err, "getsockopt-after")) + goto close_links; + ASSERT_TRUE(result[0] == 2 && result[1] == 3, "order after update"); + +close_links: + if (preorder_link >= 0) + close(preorder_link); + close(normal_link); +close_skel: + cgroup_preorder__destroy(skel); + return err; +} + void test_cgroup_preorder(void) { int cg_parent = -1, cg_child = -1, sock_fd = -1; @@ -120,6 +196,7 @@ void test_cgroup_preorder(void) ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, false), "getsockopt_test_1"); ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, true), "getsockopt_test_2"); + ASSERT_OK(run_link_replace_test(cg_child, sock_fd), "link_replace_test"); out: close(sock_fd); diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_storage.c b/tools/testing/selftests/bpf/prog_tests/cgroup_storage.c index cf395715ced4..8dab655db342 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_storage.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_storage.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 +#include <unistd.h> +#include <sys/socket.h> #include <test_progs.h> #include "cgroup_helpers.h" #include "network_helpers.h" @@ -86,6 +88,11 @@ void test_cgroup_storage(void) err = SYS_NOFAIL(PING_CMD); ASSERT_OK(err, "sixth ping"); + err = bpf_map__get_next_key(skel->maps.cgroup_storage, &key, &key, + sizeof(key)); + ASSERT_ERR(err, "bpf_map__get_next_key should fail"); + ASSERT_EQ(errno, ENOENT, "no second key"); + cleanup_progs: cgroup_storage__destroy(skel); cleanup_network: @@ -94,3 +101,43 @@ cleanup_cgroup: close(cgroup_fd); cleanup_cgroup_environment(); } + +void test_cgroup_storage_oob(void) +{ + struct cgroup_storage *skel; + int cgroup_fd, sock_fd; + + cgroup_fd = cgroup_setup_and_join(TEST_CGROUP); + if (!ASSERT_OK_FD(cgroup_fd, "create cgroup")) + return; + + /* Load and attach BPF program */ + skel = cgroup_storage__open_and_load(); + if (!ASSERT_OK_PTR(skel, "cgroup_storage__open_and_load")) + goto cleanup_cgroup; + + skel->links.trigger_oob = bpf_program__attach_cgroup(skel->progs.trigger_oob, + cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.trigger_oob, "attach_cgroup")) + goto cleanup_skel; + + /* Create a socket to trigger cgroup/sock_create hook. + * This will execute our BPF program and trigger the OOB read + * if the bug is present (before the fix). + */ + sock_fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!ASSERT_OK_FD(sock_fd, "create socket")) + goto cleanup_skel; + + close(sock_fd); + + /* If we reach here without a kernel panic or KASAN report, + * the test passes (the fix is working). + */ + +cleanup_skel: + cgroup_storage__destroy(skel); +cleanup_cgroup: + close(cgroup_fd); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c b/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c index 478a77cb67e6..c4398ccf3493 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c +++ b/tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c @@ -176,7 +176,7 @@ static void test_cgroup_iter_sleepable(int cgroup_fd, __u64 cgroup_id) DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); union bpf_iter_link_info linfo; struct cgrp_ls_sleepable *skel; - struct bpf_link *link; + struct bpf_link *link, *fexit_link; int err, iter_fd; char buf[16]; @@ -200,16 +200,27 @@ static void test_cgroup_iter_sleepable(int cgroup_fd, __u64 cgroup_id) if (!ASSERT_OK_PTR(link, "attach_iter")) goto out; + fexit_link = bpf_program__attach(skel->progs.fexit_update); + if (!ASSERT_OK_PTR(fexit_link, "attach_fexit")) + goto out_link; + iter_fd = bpf_iter_create(bpf_link__fd(link)); if (!ASSERT_GE(iter_fd, 0, "iter_create")) - goto out_link; + goto out_fexit_link; + + skel->bss->target_pid = sys_gettid(); /* trigger the program run */ (void)read(iter_fd, buf, sizeof(buf)); + skel->bss->target_pid = 0; + + ASSERT_EQ(skel->bss->update_err, 0, "update_err"); ASSERT_EQ(skel->bss->cgroup_id, cgroup_id, "cgroup_id"); close(iter_fd); +out_fexit_link: + bpf_link__destroy(fexit_link); out_link: bpf_link__destroy(link); out: diff --git a/tools/testing/selftests/bpf/prog_tests/clone_attach_btf_id.c b/tools/testing/selftests/bpf/prog_tests/clone_attach_btf_id.c new file mode 100644 index 000000000000..1c3e28e74606 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/clone_attach_btf_id.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Meta */ +#include <test_progs.h> +#include "clone_attach_btf_id.skel.h" + +/* + * Test that bpf_program__clone() respects caller-provided attach_btf_id + * override via bpf_prog_load_opts. + * + * The BPF program has SEC("fentry/bpf_fentry_test1"). Clone it twice + * from the same prepared object: first with no opts (callback resolves + * attach_btf_id from sec_name), then with attach_btf_id overridden to + * bpf_fentry_test2. Verify each loaded program's attach_btf_id via + * bpf_prog_get_info_by_fd(). + */ + +static int get_prog_attach_btf_id(int prog_fd) +{ + struct bpf_prog_info info = {}; + __u32 info_len = sizeof(info); + int err; + + err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); + if (err) + return err; + return info.attach_btf_id; +} + +void test_clone_attach_btf_id(void) +{ + struct clone_attach_btf_id *skel; + int fd1 = -1, fd2 = -1, err; + int btf_id_test1, btf_id_test2; + + btf_id_test1 = libbpf_find_vmlinux_btf_id("bpf_fentry_test1", BPF_TRACE_FENTRY); + if (!ASSERT_GT(btf_id_test1, 0, "find_btf_id_test1")) + return; + + btf_id_test2 = libbpf_find_vmlinux_btf_id("bpf_fentry_test2", BPF_TRACE_FENTRY); + if (!ASSERT_GT(btf_id_test2, 0, "find_btf_id_test2")) + return; + + skel = clone_attach_btf_id__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "obj_prepare")) + goto out; + + /* Clone with no opts — callback resolves BTF from sec_name */ + fd1 = bpf_program__clone(skel->progs.fentry_handler, NULL); + if (!ASSERT_GE(fd1, 0, "clone_default")) + goto out; + ASSERT_EQ(get_prog_attach_btf_id(fd1), btf_id_test1, + "attach_btf_id_default"); + + /* + * Clone with attach_btf_id override pointing to a different + * function. The BPF program never accesses arguments, so the + * load succeeds regardless of signature mismatch. + */ + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .attach_btf_id = btf_id_test2, + ); + fd2 = bpf_program__clone(skel->progs.fentry_handler, &opts); + if (!ASSERT_GE(fd2, 0, "clone_override")) + goto out; + ASSERT_EQ(get_prog_attach_btf_id(fd2), btf_id_test2, + "attach_btf_id_override"); + +out: + if (fd1 >= 0) + close(fd1); + if (fd2 >= 0) + close(fd2); + clone_attach_btf_id__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/connect_force_port.c b/tools/testing/selftests/bpf/prog_tests/connect_force_port.c index 24d553109f8d..dfb7f6cf3ee1 100644 --- a/tools/testing/selftests/bpf/prog_tests/connect_force_port.c +++ b/tools/testing/selftests/bpf/prog_tests/connect_force_port.c @@ -53,6 +53,9 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type) __u16 expected_peer_port = 60000; struct bpf_program *prog; struct bpf_object *obj; + struct bpf_map *map; + __u16 *port_ptr; + size_t port_size; const char *obj_file = v4 ? "connect_force_port4.bpf.o" : "connect_force_port6.bpf.o"; int fd, err; __u32 duration = 0; @@ -61,6 +64,21 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type) if (!ASSERT_OK_PTR(obj, "bpf_obj_open")) return -1; + map = bpf_object__find_map_by_name(obj, ".bss"); + if (!ASSERT_OK_PTR(map, "find bss map")) { + err = -EIO; + goto close_bpf_object; + } + + port_ptr = bpf_map__initial_value(map, &port_size); + if (!ASSERT_OK_PTR(port_ptr, "get bss initial value")) { + err = -EIO; + goto close_bpf_object; + } + + /* Auto assigns the port according to availability */ + *port_ptr = ntohs(get_socket_local_port(server_fd)); + err = bpf_object__load(obj); if (!ASSERT_OK(err, "bpf_obj_load")) { err = -EIO; @@ -138,25 +156,25 @@ void test_connect_force_port(void) if (CHECK_FAIL(cgroup_fd < 0)) return; - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 60123, 0); + server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); if (CHECK_FAIL(server_fd < 0)) goto close_cgroup_fd; CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET, SOCK_STREAM)); close(server_fd); - server_fd = start_server(AF_INET6, SOCK_STREAM, NULL, 60124, 0); + server_fd = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0); if (CHECK_FAIL(server_fd < 0)) goto close_cgroup_fd; CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET6, SOCK_STREAM)); close(server_fd); - server_fd = start_server(AF_INET, SOCK_DGRAM, NULL, 60123, 0); + server_fd = start_server(AF_INET, SOCK_DGRAM, NULL, 0, 0); if (CHECK_FAIL(server_fd < 0)) goto close_cgroup_fd; CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET, SOCK_DGRAM)); close(server_fd); - server_fd = start_server(AF_INET6, SOCK_DGRAM, NULL, 60124, 0); + server_fd = start_server(AF_INET6, SOCK_DGRAM, NULL, 0, 0); if (CHECK_FAIL(server_fd < 0)) goto close_cgroup_fd; CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET6, SOCK_DGRAM)); diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c b/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c index a18d3680fb16..51f42b02a267 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc_raw.c @@ -14,6 +14,197 @@ static char log[16 * 1024]; +static int load_core_relo_insns(int btf_fd, struct bpf_insn *insns, int insn_cnt, + struct bpf_func_info *funcs, int func_cnt, + int enum_id, int access_str_off, int insn_idx, + bool relocate) +{ + struct bpf_core_relo relo = { + .insn_off = insn_idx * sizeof(struct bpf_insn), + .type_id = enum_id, + .access_str_off = access_str_off, + .kind = BPF_CORE_ENUMVAL_VALUE, + }; + union bpf_attr attr = { + .prog_type = BPF_PROG_TYPE_SOCKET_FILTER, + .insn_cnt = insn_cnt, + .insns = (__u64)insns, + .license = (__u64)"GPL", + .log_buf = (__u64)log, + .log_size = sizeof(log), + .log_level = 2, + .prog_btf_fd = btf_fd, + .func_info_rec_size = sizeof(struct bpf_func_info), + .func_info = (__u64)funcs, + .func_info_cnt = func_cnt, + }; + + if (relocate) { + attr.core_relo_cnt = 1; + attr.core_relos = (__u64)&relo; + attr.core_relo_rec_size = sizeof(relo); + } + memset(log, 0, sizeof(log)); + return sys_bpf_prog_load(&attr, sizeof(attr), 1); +} + +static void test_early_core_relo(void) +{ + static const char unrecognized[] = "trying to relocate unrecognized insn #2"; + static const struct { + const char *name; + struct bpf_insn insns[2]; + const char *err_msg; + } tests[] = { + { "poison_exit", { BPF_EXIT_INSN() }, unrecognized }, + { "poison_ja", { BPF_JMP_A(1) }, unrecognized }, + { "poison_jmp", { BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1) }, unrecognized }, + { "poison_jmp32", { BPF_JMP32_IMM(BPF_JEQ, BPF_REG_0, 0, 1) }, unrecognized }, + { "poison_call", { BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32) }, unrecognized }, + { "poison_alu_reg", { BPF_MOV32_REG(BPF_REG_0, BPF_REG_1) }, unrecognized }, + { "poison_alu64_reg", { BPF_MOV64_REG(BPF_REG_0, BPF_REG_1) }, unrecognized }, + { "poison_ld_abs", { BPF_LD_ABS(BPF_W, 0) }, + "insn #2 (LDIMM64) has unexpected form" }, + { "poison_alu_imm", { BPF_MOV32_IMM(BPF_REG_0, 0) } }, + { "poison_alu64_imm", { BPF_MOV64_IMM(BPF_REG_0, 0) } }, + { "poison_ldx", { BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0) } }, + { "poison_st", { BPF_ST_MEM(BPF_W, BPF_REG_10, -4, 0) } }, + { "poison_stx", { BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -4) } }, + { "poison_ldimm64", { BPF_LD_IMM64(BPF_REG_0, 0) } }, + }; + struct test_btf { + struct btf_header hdr; + __u32 types[18]; + char strings[64]; + } raw_btf = { + .hdr = { + .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(struct btf_header), + .type_off = 0, + .type_len = sizeof(raw_btf.types), + .str_off = offsetof(struct test_btf, strings) - + offsetof(struct test_btf, types), + .str_len = sizeof(raw_btf.strings), + }, + .types = { + BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] int */ + BTF_FUNC_PROTO_ENC(1, 0), /* [2] int (*)(void) */ + BTF_FUNC_ENC(5, 2), /* [3] main_fn */ + BTF_FUNC_ENC(13, 2), /* [4] sub_fn */ + BTF_TYPE_ENC(20, BTF_INFO_ENC(BTF_KIND_ENUM, 0, 1), 4), /* [5] enum */ + BTF_ENUM_ENC(45, 0), /* value = 0 */ + }, + .strings = "\0int\0main_fn\0sub_fn\0core_relo_poison_missing\0value\0" "0", + }; + struct bpf_func_info funcs[] = { + { .insn_off = 0, .type_id = 3 }, + { .insn_off = 3, .type_id = 4 }, + }; + struct bpf_insn core_only[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_insn subprog[] = { + BPF_CALL_REL(2), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + struct bpf_insn truncated_ldimm64[] = { + BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, 0, 0, 0, 0), + }; + int access_str_off = 51; /* offset of "0" */ + int enum_id = 5; + int btf_fd, prog_fd = -1, i; + + btf_fd = bpf_btf_load(&raw_btf, sizeof(raw_btf), NULL); + if (!ASSERT_GE(btf_fd, 0, "btf_load")) + goto cleanup; + + if (test__start_subtest("without_func_info")) { + prog_fd = load_core_relo_insns(btf_fd, core_only, ARRAY_SIZE(core_only), NULL, 0, + enum_id, access_str_off, 2, false); + if (!ASSERT_GE(prog_fd, 0, "control_load")) + goto cleanup; + close(prog_fd); + prog_fd = load_core_relo_insns(btf_fd, core_only, ARRAY_SIZE(core_only), NULL, 0, + enum_id, access_str_off, 2, true); + if (!ASSERT_GE(prog_fd, 0, "poisoned_load")) + goto cleanup; + ASSERT_HAS_SUBSTR(log, "substituting insn #2", "poison_log"); + close(prog_fd); + prog_fd = -1; + } + + if (test__start_subtest("before_subprog_validation")) { + prog_fd = load_core_relo_insns(btf_fd, subprog, ARRAY_SIZE(subprog), funcs, 2, + enum_id, access_str_off, 2, true); + if (!ASSERT_LT(prog_fd, 0, "poisoned_load")) + goto cleanup; + ASSERT_HAS_SUBSTR(log, "substituting insn #2", "poison_log"); + ASSERT_HAS_SUBSTR(log, "last insn is not an exit or jmp", "poisoned_load_log"); + } + + if (test__start_subtest("truncated_ldimm64")) { + prog_fd = load_core_relo_insns(btf_fd, truncated_ldimm64, + ARRAY_SIZE(truncated_ldimm64), NULL, 0, + enum_id, access_str_off, 0, true); + if (!ASSERT_LT(prog_fd, 0, "truncated_load")) + goto cleanup; + ASSERT_HAS_SUBSTR(log, "invalid bpf_ld_imm64 insn", "truncated_load_log"); + } + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1), + tests[i].insns[0], + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + bool is_ldimm64 = insns[2].code == (BPF_LD | BPF_DW | BPF_IMM); + + if (!test__start_subtest(tests[i].name)) + continue; + if (is_ldimm64) { + insns[1].off = 2; + insns[3] = tests[i].insns[1]; + } + prog_fd = load_core_relo_insns(btf_fd, insns, ARRAY_SIZE(insns), funcs, 1, + enum_id, access_str_off, 2, false); + if (!ASSERT_GE(prog_fd, 0, "control_load")) + goto cleanup; + close(prog_fd); + prog_fd = load_core_relo_insns(btf_fd, insns, ARRAY_SIZE(insns), funcs, 1, + enum_id, access_str_off, 2, true); + if (!tests[i].err_msg) { + ASSERT_GE(prog_fd, 0, "dead_poison_load"); + ASSERT_HAS_SUBSTR(log, "substituting insn #2", "poison_log"); + if (is_ldimm64) + ASSERT_HAS_SUBSTR(log, "substituting insn #3", "poison_ldimm64_log"); + } else { + ASSERT_LT(prog_fd, 0, "invalid_poison_load"); + ASSERT_HAS_SUBSTR(log, tests[i].err_msg, "invalid_poison_log"); + ASSERT_NULL(strstr(log, "substituting insn"), "invalid_poison_substitution"); + } + close(prog_fd); + prog_fd = -1; + } + +cleanup: + if (env.verbosity > VERBOSE_NORMAL && log[0]) { + printf("-------- program load log start --------\n"); + printf("%s", log); + printf("-------- program load log end ----------\n"); + } + close(prog_fd); + close(btf_fd); +} + /* Check that verifier rejects BPF program containing relocation * pointing to non-existent BTF type. */ @@ -120,6 +311,7 @@ out: void test_core_reloc_raw(void) { + test_early_core_relo(); if (test__start_subtest("bad_local_id")) test_bad_local_id(); } diff --git a/tools/testing/selftests/bpf/prog_tests/ctx_rewrite.c b/tools/testing/selftests/bpf/prog_tests/ctx_rewrite.c index 469e92869523..2c3124092b73 100644 --- a/tools/testing/selftests/bpf/prog_tests/ctx_rewrite.c +++ b/tools/testing/selftests/bpf/prog_tests/ctx_rewrite.c @@ -69,19 +69,19 @@ static struct test_case test_cases[] = { #if defined(__x86_64__) || defined(__aarch64__) { N(SCHED_CLS, struct __sk_buff, tstamp), - .read = "r11 = *(u8 *)($ctx + sk_buff::__mono_tc_offset);" - "if w11 & 0x4 goto pc+1;" + .read = "r12 = *(u8 *)($ctx + sk_buff::__mono_tc_offset);" + "if w12 & 0x4 goto pc+1;" "goto pc+4;" - "if w11 & 0x3 goto pc+1;" + "if w12 & 0x3 goto pc+1;" "goto pc+2;" "$dst = 0;" "goto pc+1;" "$dst = *(u64 *)($ctx + sk_buff::tstamp);", - .write = "r11 = *(u8 *)($ctx + sk_buff::__mono_tc_offset);" - "if w11 & 0x4 goto pc+1;" + .write = "r12 = *(u8 *)($ctx + sk_buff::__mono_tc_offset);" + "if w12 & 0x4 goto pc+1;" "goto pc+2;" - "w11 &= -4;" - "*(u8 *)($ctx + sk_buff::__mono_tc_offset) = r11;" + "w12 &= -4;" + "*(u8 *)($ctx + sk_buff::__mono_tc_offset) = r12;" "*(u64 *)($ctx + sk_buff::tstamp) = $src;", }, #endif @@ -253,8 +253,7 @@ static int find_field_offset_aux(struct btf *btf, int btf_id, char *field_name, { const struct btf_type *type = btf__type_by_id(btf, btf_id); const struct btf_member *m; - __u16 mnum; - int i; + __u32 mnum, i; if (!type) { PRINT_FAIL("Can't find btf_type for id %d\n", btf_id); diff --git a/tools/testing/selftests/bpf/prog_tests/dynptr.c b/tools/testing/selftests/bpf/prog_tests/dynptr.c index 5fda11590708..4396560365e8 100644 --- a/tools/testing/selftests/bpf/prog_tests/dynptr.c +++ b/tools/testing/selftests/bpf/prog_tests/dynptr.c @@ -9,6 +9,7 @@ enum test_setup_type { SETUP_SYSCALL_SLEEP, SETUP_SKB_PROG, + SETUP_SKB_PROG_NONLINEAR, SETUP_SKB_PROG_TP, SETUP_XDP_PROG, }; @@ -32,6 +33,7 @@ static struct { {"test_ringbuf", SETUP_SYSCALL_SLEEP}, {"test_skb_readonly", SETUP_SKB_PROG}, {"test_dynptr_skb_data", SETUP_SKB_PROG}, + {"test_dynptr_skb_slice_non_linear", SETUP_SKB_PROG_NONLINEAR}, {"test_dynptr_skb_meta_data", SETUP_SKB_PROG}, {"test_dynptr_skb_meta_flags", SETUP_SKB_PROG}, {"test_adjust", SETUP_SYSCALL_SLEEP}, @@ -94,7 +96,9 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ bpf_link__destroy(link); break; case SETUP_SKB_PROG: + case SETUP_SKB_PROG_NONLINEAR: { + struct __sk_buff ctx = {}; int prog_fd; char buf[64]; @@ -106,6 +110,12 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ .repeat = 1, ); + if (setup_type == SETUP_SKB_PROG_NONLINEAR) { + ctx.data_end = ETH_HLEN + sizeof(struct iphdr); + topts.ctx_in = &ctx; + topts.ctx_size_in = sizeof(ctx); + } + prog_fd = bpf_program__fd(prog); if (!ASSERT_GE(prog_fd, 0, "prog_fd")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/empty_skb.c b/tools/testing/selftests/bpf/prog_tests/empty_skb.c index 438583e1f2d1..c9fcb70cbafb 100644 --- a/tools/testing/selftests/bpf/prog_tests/empty_skb.c +++ b/tools/testing/selftests/bpf/prog_tests/empty_skb.c @@ -10,8 +10,8 @@ void test_empty_skb(void) struct empty_skb *bpf_obj = NULL; struct nstoken *tok = NULL; struct bpf_program *prog; + struct ethhdr eth_hlen; char eth_hlen_pp[15]; - char eth_hlen[14]; int veth_ifindex; int ipip_ifindex; int err; @@ -25,7 +25,9 @@ void test_empty_skb(void) int err; int ret; int lwt_egress_ret; /* expected retval at lwt/egress */ + __be16 h_proto; bool success_on_tc; + bool adjust_room; } tests[] = { /* Empty packets are always rejected. */ @@ -46,6 +48,28 @@ void test_empty_skb(void) .err = -EINVAL, }, + /* ETH_HLEN-sized packets with IPv4/IPv6 EtherType but + * no L3 header are rejected. + */ + { + .msg = "veth short IPv4 ingress packet", + .data_in = ð_hlen, + .data_size_in = sizeof(eth_hlen), + .ifindex = &veth_ifindex, + .err = -EINVAL, + .h_proto = htons(ETH_P_IP), + .adjust_room = true, + }, + { + .msg = "veth short IPv6 ingress packet", + .data_in = ð_hlen, + .data_size_in = sizeof(eth_hlen), + .ifindex = &veth_ifindex, + .err = -EINVAL, + .h_proto = htons(ETH_P_IPV6), + .adjust_room = true, + }, + /* ETH_HLEN-sized packets: * - can not be redirected at LWT_XMIT * - can be redirected at TC to non-tunneling dest @@ -54,7 +78,7 @@ void test_empty_skb(void) { /* __bpf_redirect_common */ .msg = "veth ETH_HLEN packet ingress", - .data_in = eth_hlen, + .data_in = ð_hlen, .data_size_in = sizeof(eth_hlen), .ifindex = &veth_ifindex, .ret = -ERANGE, @@ -68,7 +92,7 @@ void test_empty_skb(void) * tc: skb->len=14 <= skb_network_offset=14 */ .msg = "ipip ETH_HLEN packet ingress", - .data_in = eth_hlen, + .data_in = ð_hlen, .data_size_in = sizeof(eth_hlen), .ifindex = &ipip_ifindex, .ret = -ERANGE, @@ -108,17 +132,27 @@ void test_empty_skb(void) SYS(out, "ip addr add 192.168.1.1/16 dev ipip0"); ipip_ifindex = if_nametoindex("ipip0"); + memset(eth_hlen_pp, 0, sizeof(eth_hlen_pp)); + memset(ð_hlen, 0, sizeof(eth_hlen)); + bpf_obj = empty_skb__open_and_load(); if (!ASSERT_OK_PTR(bpf_obj, "open skeleton")) goto out; for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (tests[i].data_in == ð_hlen) + eth_hlen.h_proto = tests[i].h_proto; + bpf_object__for_each_program(prog, bpf_obj->obj) { bool at_egress = strstr(bpf_program__name(prog), "egress") != NULL; bool at_tc = !strncmp(bpf_program__section_name(prog), "tc", 2); + bool is_adjust_room = !strcmp(bpf_program__name(prog), "tc_adjust_room"); int expected_ret; char buf[128]; + if (tests[i].adjust_room != is_adjust_room) + continue; + expected_ret = at_egress && !at_tc ? tests[i].lwt_egress_ret : tests[i].ret; tattr.data_in = tests[i].data_in; diff --git a/tools/testing/selftests/bpf/prog_tests/exceptions.c b/tools/testing/selftests/bpf/prog_tests/exceptions.c index 516f4a13013c..639866ce09a9 100644 --- a/tools/testing/selftests/bpf/prog_tests/exceptions.c +++ b/tools/testing/selftests/bpf/prog_tests/exceptions.c @@ -55,6 +55,7 @@ static void test_exceptions_success(void) RUN_SUCCESS(exception_ext, 0); RUN_SUCCESS(exception_ext_mod_cb_runtime, 35); RUN_SUCCESS(exception_throw_subprog, 1); + RUN_SUCCESS(exception_throw_subprog_stack_cb, 0x1234); RUN_SUCCESS(exception_assert_nz_gfunc, 1); RUN_SUCCESS(exception_assert_zero_gfunc, 1); RUN_SUCCESS(exception_assert_neg_gfunc, 1); @@ -83,6 +84,14 @@ static void test_exceptions_success(void) RUN_SUCCESS(exception_assert_range_with, 1); RUN_SUCCESS(exception_bad_assert_range, 0); RUN_SUCCESS(exception_bad_assert_range_with, 10); + RUN_SUCCESS(exception_throw_from_void_global, 11); + + if (skel->rodata->has_stack_arg) { + RUN_SUCCESS(exception_throw_stack_arg, 56); + RUN_SUCCESS(exception_throw_after_stack_arg, 56); + RUN_SUCCESS(exception_throw_subprog_stack_arg, 56); + RUN_SUCCESS(exception_throw_subprog_after_stack_arg, 56); + } #define RUN_EXT(load_ret, attach_err, expr, msg, after_link) \ { \ @@ -127,7 +136,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), "exception_cb_mod"), "set_attach_target")) goto done; - }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); + }), "Tracing programs cannot attach to exception callback", 0); if (test__start_subtest("throwing fentry -> exception_cb")) RUN_EXT(-EINVAL, true, ({ @@ -137,7 +146,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), "exception_cb_mod"), "set_attach_target")) goto done; - }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); + }), "Tracing programs cannot attach to exception callback", 0); if (test__start_subtest("non-throwing fexit -> exception_cb")) RUN_EXT(-EINVAL, true, ({ @@ -147,7 +156,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), "exception_cb_mod"), "set_attach_target")) goto done; - }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); + }), "Tracing programs cannot attach to exception callback", 0); if (test__start_subtest("throwing fexit -> exception_cb")) RUN_EXT(-EINVAL, true, ({ @@ -157,7 +166,7 @@ static void test_exceptions_success(void) bpf_program__fd(skel->progs.exception_ext_mod_cb_runtime), "exception_cb_mod"), "set_attach_target")) goto done; - }), "FENTRY/FEXIT programs cannot attach to exception callback", 0); + }), "Tracing programs cannot attach to exception callback", 0); if (test__start_subtest("throwing extension (with custom cb) -> exception_cb")) RUN_EXT(-EINVAL, true, ({ diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index f29fc789c14b..2523c07a16c6 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -5,6 +5,7 @@ #include <bpf/btf.h> #include "bind4_prog.skel.h" #include "freplace_progmap.skel.h" +#include "fentry_sleepable.skel.h" #include "xdp_dummy.skel.h" typedef int (*test_cb)(struct bpf_object *obj); @@ -111,7 +112,7 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, struct bpf_link_info link_info; struct bpf_program *pos; const char *pos_sec_name; - char *tgt_name; + const char *tgt_name; __s32 btf_id; tgt_name = strstr(prog_name[i], "/"); @@ -335,14 +336,13 @@ out: bpf_object__close(pkt_obj); } - -static void test_func_sockmap_update(void) +static void test_func_replace_void(void) { const char *prog_name[] = { - "freplace/cls_redirect", + "freplace/foo", }; - test_fexit_bpf2bpf_common("./freplace_cls_redirect.bpf.o", - "./test_cls_redirect.bpf.o", + test_fexit_bpf2bpf_common("./freplace_void.bpf.o", + "./test_global_func7.bpf.o", ARRAY_SIZE(prog_name), prog_name, false, NULL); } @@ -432,6 +432,15 @@ static void test_func_replace_global_func(void) prog_name, false, NULL); } +static void test_func_replace_int_with_void(void) +{ + /* Make sure we can't freplace with the wrong type */ + test_obj_load_failure_common("freplace_int_with_void.bpf.o", + "./test_global_func2.bpf.o", + "Return type UNKNOWN of test_freplace_int_with_void()" + " doesn't match type INT of global_func2()"); +} + static int find_prog_btf_id(const char *name, __u32 attach_prog_fd) { struct bpf_prog_info info = {}; @@ -568,6 +577,60 @@ out: freplace_progmap__destroy(skel); } +static void test_sleepable_fentry_to_xdp(void) +{ + struct fentry_sleepable *skel = NULL; + struct xdp_dummy *skel_xdp = NULL; + int ifindex, prog_fd, err; + char buff[64] = {}; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + ifindex = if_nametoindex("lo"); + if (!ASSERT_GT(ifindex, 0, "if_nametoindex")) + return; + + skel_xdp = xdp_dummy__open_and_load(); + if (!ASSERT_OK_PTR(skel_xdp, "xdp_dummy__open_and_load")) + return; + + skel = fentry_sleepable__open(); + if (!ASSERT_OK_PTR(skel, "fentry_sleepable__open")) + goto out; + + skel->bss->user_ptr = buff; + + prog_fd = bpf_program__fd(skel_xdp->progs.__x64_sys_nop); + err = bpf_program__set_attach_target(skel->progs.fentry_xdp, prog_fd, "__x64_sys_nop"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = fentry_sleepable__load(skel); + ASSERT_ERR(err, "fentry_sleepable__load"); + if (err) + goto out; + + skel->links.fentry_xdp = bpf_program__attach_trace(skel->progs.fentry_xdp); + if (!ASSERT_OK_PTR(skel->links.fentry_xdp, "bpf_program__attach_trace")) + goto out; + + skel_xdp->links.__x64_sys_nop = bpf_program__attach_xdp(skel_xdp->progs.__x64_sys_nop, + ifindex); + if (!ASSERT_OK_PTR(skel_xdp->links.__x64_sys_nop, "bpf_program__attach_xdp")) + goto out; + + err = system("ping -q -c 1 -W 1 127.0.0.1 > /dev/null"); + ASSERT_OK(err, "ping"); + ASSERT_ERR(skel->bss->retval, "retval"); + +out: + fentry_sleepable__destroy(skel); + xdp_dummy__destroy(skel_xdp); +} + /* NOTE: affect other tests, must run in serial mode */ void serial_test_fexit_bpf2bpf(void) { @@ -579,8 +642,6 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace(); if (test__start_subtest("func_replace_verify")) test_func_replace_verify(); - if (test__start_subtest("func_sockmap_update")) - test_func_sockmap_update(); if (test__start_subtest("func_replace_return_code")) test_func_replace_return_code(); if (test__start_subtest("func_map_prog_compatibility")) @@ -597,4 +658,10 @@ void serial_test_fexit_bpf2bpf(void) test_fentry_to_cgroup_bpf(); if (test__start_subtest("func_replace_progmap")) test_func_replace_progmap(); + if (test__start_subtest("freplace_int_with_void")) + test_func_replace_int_with_void(); + if (test__start_subtest("freplace_void")) + test_func_replace_void(); + if (test__start_subtest("sleepable_fentry_to_xdp")) + test_sleepable_fentry_to_xdp(); } diff --git a/tools/testing/selftests/bpf/prog_tests/fib_lookup.c b/tools/testing/selftests/bpf/prog_tests/fib_lookup.c index bd7658958004..8f4779dd802e 100644 --- a/tools/testing/selftests/bpf/prog_tests/fib_lookup.c +++ b/tools/testing/selftests/bpf/prog_tests/fib_lookup.c @@ -2,6 +2,7 @@ /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ #include <linux/rtnetlink.h> +#include <linux/if_ether.h> #include <sys/types.h> #include <net/if.h> @@ -23,6 +24,7 @@ #define IPV4_TBID_ADDR "172.0.0.254" #define IPV4_TBID_NET "172.0.0.0" #define IPV4_TBID_DST "172.0.0.2" +#define IPV4_TBID_NONEIGH_DST "172.0.0.5" #define IPV6_TBID_ADDR "fd00::FFFF" #define IPV6_TBID_NET "fd00::" #define IPV6_TBID_DST "fd00::2" @@ -37,6 +39,41 @@ #define IPV6_LOCAL "fd01::3" #define IPV6_GW1 "fd01::1" #define IPV6_GW2 "fd01::2" +#define VLAN_ID 100 +#define VLAN_IFACE "veth1.100" +#define VLAN_ID_DOWN 102 +#define VLAN_IFACE_DOWN "veth1.102" +#define QINQ_OUTER_IFACE "veth1.200" +#define QINQ_INNER_IFACE "veth1.200.300" +#define VLAN_TABLE "300" +#define IPV4_VLAN_IFACE_ADDR "10.5.0.254" +#define IPV4_VLAN_EGRESS_DST "10.5.0.2" +#define IPV4_QINQ_DST "10.7.0.2" +#define IPV4_VLAN_DST "10.6.0.2" +#define IPV4_VLAN_GW "10.5.0.1" +#define IPV6_VLAN_IFACE_ADDR "fd02::254" +#define IPV6_VLAN_EGRESS_DST "fd02::2" +#define IPV6_VLAN_DST "fd03::2" +#define IPV6_VLAN_GW "fd02::1" +#define VLAN_VID_UNUSED 999 +#define VRF_IFACE "vrf-blue" +#define VRF_TABLE "1000" +#define VRF_VLAN_ID 101 +#define VRF_VLAN_IFACE "veth1.101" +#define IPV4_VRF_IFACE_ADDR "10.8.0.254" +#define IPV4_VRF_GW "10.8.0.1" +#define IPV4_VRF_DST "10.9.0.2" +#define TBID_VLAN_ID 50 +#define TBID_VLAN_IFACE "veth2.50" +#define IPV4_TBID_VLAN_DST "172.2.0.2" +#define IPV4_BOND_VLAN_DST "10.11.0.2" +#define IPV4_VLAN_MTU_DST "10.5.9.2" +#define QINQ_AD_VLAN_ID 200 +#define QINQ_INNER_VLAN_ID 300 +#define BOND_IFACE "bond99" +#define BOND_PORT "veth3" +#define BOND_PORT_PEER "veth4" +#define BOND_VLAN_ID 500 #define DMAC "11:11:11:11:11:11" #define DMAC_INIT { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, } #define DMAC2 "01:01:01:01:01:01" @@ -52,6 +89,17 @@ struct fib_lookup_test { __u32 tbid; __u8 dmac[6]; __u32 mark; + /* + * input tag with BPF_FIB_LOOKUP_VLAN_INPUT; expected output tag + * with BPF_FIB_LOOKUP_VLAN (checked when check_vlan is set) + */ + __u16 vlan_proto; + __u16 vlan_id; + bool check_vlan; + const char *expected_dev; /* expected params->ifindex after lookup */ + const char *iif; /* override the default veth1 input device */ + __u16 tot_len; /* triggers the in-lookup mtu check when set */ + __u16 expected_mtu; /* expected mtu_result (union with tot_len) */ }; static const struct fib_lookup_test tests[] = { @@ -79,6 +127,17 @@ static const struct fib_lookup_test tests[] = { .daddr = IPV4_TBID_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, .tbid = 100, .dmac = DMAC_INIT2, }, + /* + * An error that returns after the egress device is resolved must + * report the egress ifindex, not the input. This routes from input + * veth1 via veth2 (table 100) to a dst with no neighbour, so + * input != egress, pinning NO_NEIGH to the egress device. + */ + { .desc = "IPv4 NO_NEIGH reports the egress ifindex, not the input", + .daddr = IPV4_TBID_NONEIGH_DST, + .expected_ret = BPF_FIB_LKUP_RET_NO_NEIGH, + .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, .tbid = 100, + .expected_dev = "veth2", }, { .desc = "IPv6 TBID lookup failure", .daddr = IPV6_TBID_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, @@ -142,12 +201,237 @@ static const struct fib_lookup_test tests[] = { .expected_dst = IPV6_GW1, .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, .mark = MARK, }, + /* vlan egress resolution */ + /* + * Invariant the VLAN-egress arms jointly enforce: a + * BPF_FIB_LOOKUP_VLAN SUCCESS always carries a physical, + * xmit-capable ifindex; no SUCCESS ever returns a VLAN-device + * ifindex. Reducible arms pin ifindex == the physical parent; the + * QinQ and foreign-netns arms pin VLAN_FAILURE with params->ifindex + * left at the input, so a regression to best-effort (SUCCESS + the + * VLAN ifindex) fails one. + */ + { .desc = "IPv4 VLAN egress, no flag", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = VLAN_IFACE, .check_vlan = true, }, + { .desc = "IPv4 VLAN egress, single VLAN", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + /* + * skb path without tot_len: mtu_result is the VLAN device's mtu + * (1400), not the parent's (1500) + */ + { .desc = "IPv4 VLAN egress, skb-path mtu is the VLAN device's without the flag", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = VLAN_IFACE, .check_vlan = true, .expected_mtu = 1400, }, + { .desc = "IPv4 VLAN egress, flag set but egress is not a VLAN", + .daddr = IPV4_NUD_FAILED_ADDR, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + { .desc = "IPv4 VLAN egress, QinQ not reducible (VLAN_FAILURE)", + .daddr = IPV4_QINQ_DST, + .expected_ret = BPF_FIB_LKUP_RET_VLAN_FAILURE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + { .desc = "IPv4 QinQ egress without the flag (escape hatch)", + .daddr = IPV4_QINQ_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = QINQ_INNER_IFACE, }, + { .desc = "IPv6 VLAN egress, single VLAN", + .daddr = IPV6_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress, neighbour on the VLAN device", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, .dmac = DMAC_INIT, }, + { .desc = "IPv4 VLAN egress in OUTPUT mode", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .iif = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_OUTPUT | BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress over a bond", + .daddr = IPV4_BOND_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = BOND_IFACE, .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv4 VLAN egress via TBID table", + .daddr = IPV4_TBID_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID | + BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .tbid = 100, + .expected_dev = "veth2", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = TBID_VLAN_ID, }, + { .desc = "IPv4 VLAN egress, success writes mtu_result with the swap", + .daddr = IPV4_VLAN_MTU_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .tot_len = 500, .expected_mtu = 1000, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress, FRAG_NEEDED reports mtu, swap unwritten", + .daddr = IPV4_VLAN_MTU_DST, .expected_ret = BPF_FIB_LKUP_RET_FRAG_NEEDED, + .tot_len = 1400, .expected_mtu = 1000, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + /* vlan tag as lookup input */ + { .desc = "IPv4 VLAN input, no flag", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_GW1, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input, tag selects subinterface route", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv6 VLAN input, tag selects subinterface route", + .daddr = IPV6_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV6_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input and egress combined", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = "veth1", + .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, neighbour resolved on the route", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, .dmac = DMAC_INIT2, }, + { .desc = "IPv4 VLAN input, source address from the subinterface", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_src = IPV4_VLAN_IFACE_ADDR, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SRC | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + /* + * VRF: the resolved subinterface is enslaved, so the l3mdev rule + * (full lookup) and l3mdev_fib_table_rcu() (DIRECT) must select + * the VRF table from the resolved ingress + */ + { .desc = "IPv4 VLAN input, VRF subinterface, no flag", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_GW1, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input, tag selects VRF table", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VRF_GW, .expected_dev = VRF_VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VRF_VLAN_ID, }, + { .desc = "IPv4 VLAN input, DIRECT uses VRF table from resolved ingress", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VRF_GW, .expected_dev = VRF_VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_DIRECT | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VRF_VLAN_ID, }, + /* + * failure arms also assert params is left untouched: ifindex still + * names the physical device and the input tag bytes survive + */ + { .desc = "IPv4 VLAN input, invalid proto", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = 0x1234, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, unmatched VID", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_VID_UNUSED, }, + { .desc = "IPv4 VLAN input, subinterface down", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID_DOWN, }, + /* + * the resolver runs before the forwarding check, so on devices + * with forwarding off FWD_DISABLED (not NOT_FWDED) proves the tag + * resolved to that device and the lookup used it as ingress + */ + { .desc = "IPv4 VLAN input, 802.1ad tag", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021AD, .vlan_id = QINQ_AD_VLAN_ID, }, + { .desc = "IPv4 VLAN input, PCP and DEI bits ignored in TCI", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = 0xe000 | VLAN_ID, }, + { .desc = "IPv4 VLAN input, inner QinQ device from VLAN ifindex", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .iif = QINQ_OUTER_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = QINQ_INNER_VLAN_ID, }, + /* + * bonding: the VLANs live on the master, as on receive, where the + * frame is steered to the master before VLAN processing; a port + * ifindex does not match (ports carry vid state but no VLAN devs) + */ + { .desc = "IPv4 VLAN input, tag on bond master resolves", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .iif = BOND_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv4 VLAN input, tag on bond port does not match", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .iif = BOND_PORT, .expected_dev = BOND_PORT, .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv6 VLAN input, invalid proto", + .daddr = IPV6_VLAN_DST, .expected_ret = -EINVAL, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = 0x1234, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, VID 0 priority tag fails closed", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = 0, }, + { .desc = "IPv6 VLAN input, unmatched VID", + .daddr = IPV6_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_VID_UNUSED, }, + { .desc = "unknown flag bit rejected", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = (1 << 14) | BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input rejected with TBID", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_TBID, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input rejected with OUTPUT", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_OUTPUT, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, }; static int setup_netns(void) { int err; + /* + * a new netns copies the IPv4 conf from init_net, so on a host with + * forwarding enabled the arms that expect FWD_DISABLED would see the + * lookup succeed instead; pin it off here and enable it per device + */ + err = write_sysctl("/proc/sys/net/ipv4/conf/all/forwarding", "0"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf.all.forwarding)")) + goto fail; + + err = write_sysctl("/proc/sys/net/ipv4/conf/default/forwarding", "0"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf.default.forwarding)")) + goto fail; + SYS(fail, "ip link add veth1 type veth peer name veth2"); SYS(fail, "ip link set dev veth1 up"); SYS(fail, "ip link set dev veth2 up"); @@ -204,6 +488,105 @@ static int setup_netns(void) SYS(fail, "ip rule add prio 2 fwmark %d lookup %s", MARK, MARK_TABLE); SYS(fail, "ip -6 rule add prio 2 fwmark %d lookup %s", MARK, MARK_TABLE); + /* + * Setup for vlan tests: a subinterface for egress resolution and + * tag-as-input, a QinQ stack, and an iif rule so the input tests + * observe which device the lookup used as ingress. + */ + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VLAN_IFACE, VLAN_ID); + SYS(fail, "ip link set dev %s up", VLAN_IFACE); + /* + * lower than the veth1 parent (1500): the skb-path mtu check uses the + * FIB result (VLAN) device, so mtu_result is this value, which the + * no-flag arm below pins + */ + SYS(fail, "ip link set dev %s mtu 1400", VLAN_IFACE); + SYS(fail, "ip addr add %s/24 dev %s", IPV4_VLAN_IFACE_ADDR, VLAN_IFACE); + SYS(fail, "ip addr add %s/64 dev %s nodad", IPV6_VLAN_IFACE_ADDR, VLAN_IFACE); + + /* + * stays down: the input flag must treat its tag the way real + * ingress treats a frame arriving on a down VLAN device (drop) + */ + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VLAN_IFACE_DOWN, VLAN_ID_DOWN); + + err = write_sysctl("/proc/sys/net/ipv4/conf/" VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf." VLAN_IFACE ".forwarding)")) + goto fail; + + err = write_sysctl("/proc/sys/net/ipv6/conf/" VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv6.conf." VLAN_IFACE ".forwarding)")) + goto fail; + + SYS(fail, "ip link add link veth1 name %s type vlan proto 802.1ad id 200", + QINQ_OUTER_IFACE); + SYS(fail, "ip link add link %s name %s type vlan id 300", + QINQ_OUTER_IFACE, QINQ_INNER_IFACE); + SYS(fail, "ip link set dev %s up", QINQ_OUTER_IFACE); + SYS(fail, "ip link set dev %s up", QINQ_INNER_IFACE); + SYS(fail, "ip route add %s/32 dev %s", IPV4_QINQ_DST, QINQ_INNER_IFACE); + + SYS(fail, "ip route add %s/32 via %s", IPV4_VLAN_DST, IPV4_GW1); + SYS(fail, "ip route add table %s %s/32 via %s", + VLAN_TABLE, IPV4_VLAN_DST, IPV4_VLAN_GW); + SYS(fail, "ip rule add prio 3 iif %s lookup %s", VLAN_IFACE, VLAN_TABLE); + SYS(fail, "ip -6 route add %s/128 via %s", IPV6_VLAN_DST, IPV6_GW1); + SYS(fail, "ip -6 route add table %s %s/128 via %s", + VLAN_TABLE, IPV6_VLAN_DST, IPV6_VLAN_GW); + SYS(fail, "ip -6 rule add prio 3 iif %s lookup %s", VLAN_IFACE, VLAN_TABLE); + + /* a bond with one port and a VLAN on the bond */ + SYS(fail, "ip link add %s type bond", BOND_IFACE); + SYS(fail, "ip link add %s type veth peer name %s", BOND_PORT, BOND_PORT_PEER); + SYS(fail, "ip link set %s master %s", BOND_PORT, BOND_IFACE); + SYS(fail, "ip link set dev %s up", BOND_IFACE); + SYS(fail, "ip link set dev %s up", BOND_PORT); + SYS(fail, "ip link add link %s name %s.%d type vlan id %d", + BOND_IFACE, BOND_IFACE, BOND_VLAN_ID, BOND_VLAN_ID); + SYS(fail, "ip link set dev %s.%d up", BOND_IFACE, BOND_VLAN_ID); + SYS(fail, "ip route add %s/32 dev %s.%d", + IPV4_BOND_VLAN_DST, BOND_IFACE, BOND_VLAN_ID); + + /* + * a VRF with its own dedicated subinterface (the iif rules above + * must not see it), for the table-selection-by-ingress cases + */ + SYS(fail, "ip link add %s type vrf table %s", VRF_IFACE, VRF_TABLE); + SYS(fail, "ip link set dev %s up", VRF_IFACE); + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VRF_VLAN_IFACE, VRF_VLAN_ID); + SYS(fail, "ip link set %s master %s", VRF_VLAN_IFACE, VRF_IFACE); + SYS(fail, "ip link set dev %s up", VRF_VLAN_IFACE); + SYS(fail, "ip addr add %s/24 dev %s", IPV4_VRF_IFACE_ADDR, VRF_VLAN_IFACE); + err = write_sysctl("/proc/sys/net/ipv4/conf/" VRF_VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf." VRF_VLAN_IFACE ".forwarding)")) + goto fail; + SYS(fail, "ip route add %s/32 via %s", IPV4_VRF_DST, IPV4_GW1); + SYS(fail, "ip route add table %s %s/32 via %s", + VRF_TABLE, IPV4_VRF_DST, IPV4_VRF_GW); + + /* neighbours on the VLAN subinterface for the non-SKIP_NEIGH cases */ + err = write_sysctl("/proc/sys/net/ipv4/neigh/" VLAN_IFACE "/gc_stale_time", "900"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.neigh." VLAN_IFACE ".gc_stale_time)")) + goto fail; + SYS(fail, "ip neigh add %s dev %s lladdr %s nud stale", + IPV4_VLAN_EGRESS_DST, VLAN_IFACE, DMAC); + SYS(fail, "ip neigh add %s dev %s lladdr %s nud stale", + IPV4_VLAN_GW, VLAN_IFACE, DMAC2); + + /* a VLAN on veth2 with a route in the tbid test table */ + SYS(fail, "ip link add link veth2 name %s type vlan id %d", + TBID_VLAN_IFACE, TBID_VLAN_ID); + SYS(fail, "ip link set dev %s up", TBID_VLAN_IFACE); + SYS(fail, "ip route add table 100 %s/32 dev %s", + IPV4_TBID_VLAN_DST, TBID_VLAN_IFACE); + + /* a locked-mtu route via the subinterface for the FRAG_NEEDED case */ + SYS(fail, "ip route add %s/32 dev %s mtu lock 1000", + IPV4_VLAN_MTU_DST, VLAN_IFACE); + return 0; fail: return -1; @@ -218,9 +601,16 @@ static int set_lookup_params(struct bpf_fib_lookup *params, memset(params, 0, sizeof(*params)); params->l4_protocol = IPPROTO_TCP; - params->ifindex = ifindex; + params->ifindex = test->iif ? if_nametoindex(test->iif) : ifindex; params->tbid = test->tbid; params->mark = test->mark; + params->tot_len = test->tot_len; + + /* h_vlan_proto/h_vlan_TCI union with tbid */ + if (test->lookup_flags & BPF_FIB_LOOKUP_VLAN_INPUT) { + params->h_vlan_proto = htons(test->vlan_proto); + params->h_vlan_TCI = htons(test->vlan_id); + } if (inet_pton(AF_INET6, test->daddr, params->ipv6_dst) == 1) { params->family = AF_INET6; @@ -298,7 +688,7 @@ void test_fib_lookup(void) struct nstoken *nstoken = NULL; struct __sk_buff skb = { }; struct fib_lookup *skel; - int prog_fd, err, ret, i; + int prog_fd, xdp_fd, err, ret, i; /* The test does not use the skb->data, so * use pkt_v6 for both v6 and v4 test. @@ -309,11 +699,16 @@ void test_fib_lookup(void) .ctx_in = &skb, .ctx_size_in = sizeof(skb), ); + LIBBPF_OPTS(bpf_test_run_opts, xdp_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + ); skel = fib_lookup__open_and_load(); if (!ASSERT_OK_PTR(skel, "skel open_and_load")) return; prog_fd = bpf_program__fd(skel->progs.fib_lookup); + xdp_fd = bpf_program__fd(skel->progs.fib_lookup_xdp); SYS(fail, "ip netns add %s", NS_TEST); @@ -343,6 +738,16 @@ void test_fib_lookup(void) if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) continue; + /* + * BPF_FIB_LOOKUP_VLAN is XDP-only; the tc helper rejects it. + * These cases are exercised on the XDP path below. + */ + if (tests[i].lookup_flags & BPF_FIB_LOOKUP_VLAN) { + ASSERT_EQ(skel->bss->fib_lookup_ret, -EINVAL, + "tc rejects BPF_FIB_LOOKUP_VLAN"); + continue; + } + ASSERT_EQ(skel->bss->fib_lookup_ret, tests[i].expected_ret, "fib_lookup_ret"); @@ -352,6 +757,21 @@ void test_fib_lookup(void) if (tests[i].expected_dst) assert_dst_ip(fib_params, tests[i].expected_dst); + if (tests[i].expected_dev) + ASSERT_EQ(fib_params->ifindex, + if_nametoindex(tests[i].expected_dev), "ifindex"); + + if (tests[i].expected_mtu) + ASSERT_EQ(fib_params->mtu_result, tests[i].expected_mtu, + "mtu_result"); + + if (tests[i].check_vlan) { + ASSERT_EQ(fib_params->h_vlan_proto, + htons(tests[i].vlan_proto), "h_vlan_proto"); + ASSERT_EQ(fib_params->h_vlan_TCI, + htons(tests[i].vlan_id), "h_vlan_TCI"); + } + ret = memcmp(tests[i].dmac, fib_params->dmac, sizeof(tests[i].dmac)); if (!ASSERT_EQ(ret, 0, "dmac not match")) { char expected[18], actual[18]; @@ -361,17 +781,330 @@ void test_fib_lookup(void) printf("dmac expected %s actual %s ", expected, actual); } - // ensure tbid is zero'd out after fib lookup. - if (tests[i].lookup_flags & BPF_FIB_LOOKUP_DIRECT) { + /* + * ensure tbid is zero'd out after fib lookup. With + * BPF_FIB_LOOKUP_VLAN the union holds the packed vlan + * fields instead, so skip the check for those. + */ + if ((tests[i].lookup_flags & BPF_FIB_LOOKUP_DIRECT) && + !(tests[i].lookup_flags & BPF_FIB_LOOKUP_VLAN)) { if (!ASSERT_EQ(skel->bss->fib_params.tbid, 0, "expected fib_params.tbid to be zero")) goto fail; } } + /* + * Re-run the cases through bpf_xdp_fib_lookup(). test_run uses the + * current netns' loopback for ctx->rxq->dev, so dev_net() is NS_TEST + * and the lookup runs against its FIB. The path-independent results + * (return code, swapped ifindex, vlan tag, gateway) must match the skb + * path; the no-tot_len mtu_result is skb-specific and not rechecked. + */ + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (set_lookup_params(fib_params, &tests[i], skb.ifindex)) + continue; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = tests[i].lookup_flags; + + err = bpf_prog_test_run_opts(xdp_fd, &xdp_opts); + if (!ASSERT_OK(err, "xdp test_run")) + continue; + + if (!ASSERT_EQ(skel->bss->fib_lookup_ret, tests[i].expected_ret, + "xdp fib_lookup_ret")) + printf("(xdp) %s\n", tests[i].desc); + + if (tests[i].expected_dev) + ASSERT_EQ(fib_params->ifindex, + if_nametoindex(tests[i].expected_dev), + "xdp ifindex"); + + if (tests[i].expected_dst) + assert_dst_ip(fib_params, tests[i].expected_dst); + + if (tests[i].check_vlan) { + ASSERT_EQ(fib_params->h_vlan_proto, + htons(tests[i].vlan_proto), "xdp h_vlan_proto"); + ASSERT_EQ(fib_params->h_vlan_TCI, + htons(tests[i].vlan_id), "xdp h_vlan_TCI"); + } + + ret = memcmp(tests[i].dmac, fib_params->dmac, sizeof(tests[i].dmac)); + ASSERT_EQ(ret, 0, "xdp dmac"); + + /* + * mtu_result from a tot_len lookup is the route mtu and is + * path-independent; the no-tot_len arm reads dev->mtu and is + * skb-only, so gate on tot_len + */ + if (tests[i].expected_mtu && tests[i].tot_len) + ASSERT_EQ(fib_params->mtu_result, tests[i].expected_mtu, + "xdp mtu_result"); + } + fail: if (nstoken) close_netns(nstoken); SYS_NOFAIL("ip netns del " NS_TEST); fib_lookup__destroy(skel); } + +#define NS_VLAN_A "fib_lookup_vlan_ns_a" +#define NS_VLAN_B "fib_lookup_vlan_ns_b" +#define IPV4_VLAN_NETNS_ADDR "10.66.0.1" +#define IPV4_VLAN_NETNS_DST "10.66.0.2" + +/* + * A VLAN device can be moved to another netns while staying registered + * on its parent. Neither direction may then cross the boundary: the + * egress flag must not publish the foreign parent's ifindex, and the + * input flag must fail closed rather than use a foreign ingress. + */ +void test_fib_lookup_vlan_netns(void) +{ + struct bpf_fib_lookup *fib_params; + struct nstoken *nstoken = NULL; + struct __sk_buff skb = { }; + struct fib_lookup *skel = NULL; + int prog_fd, xdp_fd, err, parent_idx, vlan_idx; + + LIBBPF_OPTS(bpf_test_run_opts, run_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + .ctx_in = &skb, + .ctx_size_in = sizeof(skb), + ); + LIBBPF_OPTS(bpf_test_run_opts, xdp_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + ); + + skel = fib_lookup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + prog_fd = bpf_program__fd(skel->progs.fib_lookup); + xdp_fd = bpf_program__fd(skel->progs.fib_lookup_xdp); + fib_params = &skel->bss->fib_params; + + SYS(fail, "ip netns add %s", NS_VLAN_A); + SYS(fail, "ip netns add %s", NS_VLAN_B); + + nstoken = open_netns(NS_VLAN_A); + if (!ASSERT_OK_PTR(nstoken, "open_netns(a)")) + goto fail; + + SYS(fail, "ip link add veth7 type veth peer name veth8"); + SYS(fail, "ip link set dev veth7 up"); + SYS(fail, "ip link add link veth7 name veth7.66 type vlan id 66"); + SYS(fail, "ip link set veth7.66 netns %s", NS_VLAN_B); + /* + * up it in B before the input lookup: the move closed it, and a + * down device fails the resolver on IFF_UP before reaching the + * netns check this subtest exists to pin + */ + SYS(fail, "ip -n %s link set dev veth7.66 up", NS_VLAN_B); + + parent_idx = if_nametoindex("veth7"); + if (!ASSERT_NEQ(parent_idx, 0, "if_nametoindex(veth7)")) + goto fail; + + /* + * give this netns a route to the destination: the lookup below runs + * against this FIB, so without the route a kernel that resolved the + * moved device anyway would still return NOT_FWDED and the arm would + * pass for the wrong reason + */ + SYS(fail, "ip route add %s/32 dev veth7", IPV4_VLAN_NETNS_DST); + + /* + * input: the moved device is still in veth7's VLAN group, but it + * lives in another netns, so the lookup must fail closed + */ + skb.ifindex = parent_idx; + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = parent_idx; + fib_params->h_vlan_proto = htons(ETH_P_8021Q); + fib_params->h_vlan_TCI = htons(66); + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(dst)")) + goto fail; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | + BPF_FIB_LOOKUP_SKIP_NEIGH; + err = bpf_prog_test_run_opts(prog_fd, &run_opts); + if (!ASSERT_OK(err, "test_run(input)")) + goto fail; + ASSERT_EQ(skel->bss->fib_lookup_ret, BPF_FIB_LKUP_RET_NOT_FWDED, + "input across netns fails closed"); + ASSERT_EQ(fib_params->ifindex, parent_idx, "ifindex untouched"); + ASSERT_EQ(fib_params->h_vlan_TCI, htons(66), "tag untouched"); + + close_netns(nstoken); + nstoken = open_netns(NS_VLAN_B); + if (!ASSERT_OK_PTR(nstoken, "open_netns(b)")) + goto fail; + + /* + * egress: the fib result is the VLAN device here, but its parent + * is in the other netns, so the swap must not happen + */ + SYS(fail, "ip addr add %s/24 dev veth7.66", IPV4_VLAN_NETNS_ADDR); + err = write_sysctl("/proc/sys/net/ipv4/conf/veth7.66/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(forwarding)")) + goto fail; + + vlan_idx = if_nametoindex("veth7.66"); + if (!ASSERT_NEQ(vlan_idx, 0, "if_nametoindex(veth7.66)")) + goto fail; + + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = vlan_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(dst)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)")) + goto fail; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH; + err = bpf_prog_test_run_opts(xdp_fd, &xdp_opts); + if (!ASSERT_OK(err, "test_run(egress)")) + goto fail; + ASSERT_EQ(skel->bss->fib_lookup_ret, BPF_FIB_LKUP_RET_VLAN_FAILURE, + "egress returns VLAN_FAILURE"); + ASSERT_EQ(fib_params->ifindex, vlan_idx, + "foreign parent not published"); + ASSERT_EQ(fib_params->h_vlan_TCI, 0, "vlan fields zero"); + +fail: + if (nstoken) + close_netns(nstoken); + SYS_NOFAIL("ip netns del " NS_VLAN_A); + SYS_NOFAIL("ip netns del " NS_VLAN_B); + fib_lookup__destroy(skel); +} + +#define REDIRECT_NPKTS 1000 +#define NS_REDIRECT "fib_lookup_redirect_ns" + +/* + * The egress flag exists so an XDP program can redirect to the physical + * parent. A redirect that lands on a VLAN device is dropped at + * xdp_do_flush(), because a VLAN device has no ndo_xdp_xmit. Drive real + * frames with BPF_F_TEST_XDP_LIVE_FRAMES, which runs the native + * xdp_do_redirect() + xdp_do_flush() path: a reducible VLAN egress + * resolves to veth1 and is delivered to its peer veth2, while a QinQ + * egress returns VLAN_FAILURE and is passed to the stack instead of + * redirected to a device that would silently drop it. + */ +void test_fib_lookup_vlan_redirect(void) +{ + int redirect_fd, err, veth1_idx, veth2_idx = -1; + struct bpf_fib_lookup *fib_params; + struct nstoken *nstoken = NULL; + struct fib_lookup *skel = NULL; + bool xdp_attached = false; + + LIBBPF_OPTS(bpf_test_run_opts, lf_opts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .flags = BPF_F_TEST_XDP_LIVE_FRAMES, + .repeat = REDIRECT_NPKTS, + ); + + skel = fib_lookup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + redirect_fd = bpf_program__fd(skel->progs.fib_lookup_redirect); + fib_params = &skel->bss->fib_params; + + SYS(fail, "ip netns add %s", NS_REDIRECT); + nstoken = open_netns(NS_REDIRECT); + if (!ASSERT_OK_PTR(nstoken, "open_netns")) + goto fail; + if (setup_netns()) + goto fail; + + veth1_idx = if_nametoindex("veth1"); + veth2_idx = if_nametoindex("veth2"); + if (!ASSERT_NEQ(veth1_idx, 0, "if_nametoindex(veth1)") || + !ASSERT_NEQ(veth2_idx, 0, "if_nametoindex(veth2)")) + goto fail; + + /* + * A redirect to veth1 is delivered to its peer veth2. veth_xdp_xmit() + * only accepts the frame if veth2's NAPI is up, which on veth means + * veth2 carries an XDP program; xdp_count tallies what arrives. + */ + err = bpf_xdp_attach(veth2_idx, bpf_program__fd(skel->progs.xdp_count), + XDP_FLAGS_DRV_MODE, NULL); + if (!ASSERT_OK(err, "attach xdp_count on veth2")) + goto fail; + xdp_attached = true; + + /* reducible VLAN egress: resolves to the physical parent veth1 */ + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = veth1_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_IFACE_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_EGRESS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(reducible dst)")) + goto fail; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH; + skel->bss->redirected = 0; + skel->bss->passed = 0; + skel->bss->delivered = 0; + + err = bpf_prog_test_run_opts(redirect_fd, &lf_opts); + if (!ASSERT_OK(err, "test_run(reducible egress)")) + goto fail; + ASSERT_EQ(skel->bss->redirected, REDIRECT_NPKTS, "reducible egress redirected"); + ASSERT_EQ(skel->bss->passed, 0, "reducible egress not passed"); + ASSERT_GT(skel->bss->delivered, 0, "reducible egress delivered to veth2"); + + /* + * QinQ egress: not reducible, so the lookup returns VLAN_FAILURE and + * the program passes the frame instead of redirecting to the inner + * VLAN device. redirected == 0 is the assertion that matters: the + * program did not redirect to a device that would drop the frame at + * xdp_do_flush(). veth2's delivered count is not checked here, since + * a passed frame can still reach veth2 through the stack's forwarding + * path, which is unrelated to the redirect under test. + */ + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = veth1_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_IFACE_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_QINQ_DST, &fib_params->ipv4_dst), + 1, "inet_pton(qinq dst)")) + goto fail; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH; + skel->bss->redirected = 0; + skel->bss->passed = 0; + + err = bpf_prog_test_run_opts(redirect_fd, &lf_opts); + if (!ASSERT_OK(err, "test_run(qinq egress)")) + goto fail; + ASSERT_EQ(skel->bss->passed, REDIRECT_NPKTS, "qinq egress passed"); + ASSERT_EQ(skel->bss->redirected, 0, "qinq egress not redirected"); + +fail: + if (xdp_attached) + bpf_xdp_detach(veth2_idx, XDP_FLAGS_DRV_MODE, NULL); + if (nstoken) + close_netns(nstoken); + SYS_NOFAIL("ip netns del " NS_REDIRECT); + fib_lookup__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/file_reader.c b/tools/testing/selftests/bpf/prog_tests/file_reader.c index 5cde32b35da4..48aae7ea0e4b 100644 --- a/tools/testing/selftests/bpf/prog_tests/file_reader.c +++ b/tools/testing/selftests/bpf/prog_tests/file_reader.c @@ -10,6 +10,7 @@ const char *user_ptr = "hello world"; char file_contents[256000]; +void *addr; void *get_executable_base_addr(void) { @@ -26,8 +27,7 @@ void *get_executable_base_addr(void) static int initialize_file_contents(void) { int fd, page_sz = sysconf(_SC_PAGESIZE); - ssize_t n = 0, cur, off; - void *addr; + ssize_t n = 0, cur; fd = open("/proc/self/exe", O_RDONLY); if (!ASSERT_OK_FD(fd, "Open /proc/self/exe\n")) @@ -52,16 +52,6 @@ static int initialize_file_contents(void) /* page-align base file address */ addr = (void *)((unsigned long)addr & ~(page_sz - 1)); - /* - * Page out range 0..512K, use 0..256K for positive tests and - * 256K..512K for negative tests expecting page faults - */ - for (off = 0; off < sizeof(file_contents) * 2; off += page_sz) { - if (!ASSERT_OK(madvise(addr + off, page_sz, MADV_PAGEOUT), - "madvise pageout")) - return errno; - } - return 0; } @@ -90,6 +80,14 @@ static void run_test(const char *prog_name) if (!ASSERT_OK(err, "file_reader__load")) goto cleanup; + /* + * Page out range 0..512K, use 0..256K for positive tests and + * 256K..512K for negative tests expecting page faults + */ + if (!ASSERT_OK(madvise(addr, sizeof(file_contents) * 2, MADV_PAGEOUT), + "madvise pageout")) + goto cleanup; + err = file_reader__attach(skel); if (!ASSERT_OK(err, "file_reader__attach")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c index e40114620751..0918321c8e63 100644 --- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c +++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c @@ -4,6 +4,7 @@ #include <string.h> #include <linux/bpf.h> #include <linux/limits.h> +#include <bpf/btf.h> #include <test_progs.h> #include "trace_helpers.h" #include "test_fill_link_info.skel.h" @@ -24,6 +25,22 @@ static __u64 kmulti_cookies[] = { 3, 1, 2 }; #define KPROBE_FUNC "bpf_fentry_test1" static __u64 kprobe_addr; +static const char * const tmulti_syms[] = { + "bpf_fentry_test2", + "bpf_fentry_test1", + "bpf_fentry_test3", +}; + +static __u64 tmulti_cookies[] = { 30, 10, 20 }; +#define TRACING_MULTI_CNT ARRAY_SIZE(tmulti_syms) + +struct tmulti_target { + const char *name; + __u64 addr; + __u64 cookie; + __u32 id; +}; + #define UPROBE_FILE "/proc/self/exe" static ssize_t uprobe_offset; /* uprobe attach point */ @@ -396,6 +413,224 @@ static void test_kprobe_multi_fill_link_info(struct test_fill_link_info *skel, bpf_link__destroy(link); } +static int tmulti_target_cmp(const void *a, const void *b) +{ + const struct tmulti_target *ta = a; + const struct tmulti_target *tb = b; + + return (ta->id > tb->id) - (ta->id < tb->id); +} + +static int setup_tmulti_targets(const struct bpf_program *prog, + struct tmulti_target *targets, + __u32 *btf_obj_id) +{ + struct bpf_prog_info prog_info; + __u32 len = sizeof(prog_info); + struct btf *btf; + int err, i; + __s32 id; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return -1; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + id = btf__find_by_name_kind(btf, tmulti_syms[i], BTF_KIND_FUNC); + if (!ASSERT_GT(id, 0, "btf__find_by_name_kind")) + goto error; + + targets[i].name = tmulti_syms[i]; + targets[i].addr = ksym_get_addr(tmulti_syms[i]); + targets[i].cookie = tmulti_cookies[i]; + targets[i].id = id; + } + + memset(&prog_info, 0, len); + err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len); + if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd")) + goto error; + if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id")) + goto error; + *btf_obj_id = prog_info.attach_btf_obj_id; + + /* + * The kernel tracing multi attach sorts ids. We sort as well, + * so we can easily compare ids and cookies later. + */ + qsort(targets, TRACING_MULTI_CNT, sizeof(targets[0]), tmulti_target_cmp); + btf__free(btf); + return 0; + +error: + btf__free(btf); + return -1; +} + +static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog, + const struct tmulti_target *targets, + __u32 btf_obj_id, bool has_cookies) +{ + enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog); + __u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT]; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + memset(&info, 0, sizeof(info)); + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + + memset(ids, 0, sizeof(ids)); + memset(cookies, 0, sizeof(cookies)); + memset(addrs, 0, sizeof(addrs)); + + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.addrs = ptr_to_u64(addrs); + info.tracing_multi.cookies = has_cookies ? ptr_to_u64(cookies) : 0; + info.tracing_multi.count = TRACING_MULTI_CNT; + + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + ASSERT_EQ(info.tracing_multi.btf_obj_id, btf_obj_id, "tracing_multi.btf_obj_id"); + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + ASSERT_EQ(cookies[i], has_cookies ? targets[i].cookie : 0, "tracing_multi.cookies"); + + if (targets[i].addr) { + struct ksym *ksym; + + if (!ASSERT_NEQ(addrs[i], 0, "tracing_multi.addrs")) + return -1; + ksym = ksym_search(addrs[i]); + if (!ASSERT_OK_PTR(ksym, "ksym_search")) + return -1; + ASSERT_STREQ(ksym->name, targets[i].name, "tracing_multi.addr_name"); + } else { + ASSERT_EQ(addrs[i], 0, "tracing_multi.addrs"); + } + } + + return 0; +} + +static void verify_tracing_multi_invalid_user_buffer(int fd, const struct tmulti_target *targets) +{ + __u32 ids[TRACING_MULTI_CNT] = {}; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + /* Wrong info setup (ids != NULL and cnt == 0) -> EINVAL */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EINVAL, "tracing_multi.invalid_count"); + + /* Smaller than actual count provided -> ENOSPC */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT - 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -ENOSPC, "tracing_multi.small_count"); + for (i = 0; i < TRACING_MULTI_CNT - 1; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.partial_ids"); + /* check that the last entry is not populated */ + ASSERT_EQ(ids[i], 0, "tracing_multi.partial_ids"); + + /* Bigger than actual count provided -> OK */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT + 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_OK(err, "tracing_multi.big_count"); + for (i = 0; i < TRACING_MULTI_CNT; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + + /* Invalid ids pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_btf_ids"); + + /* Invalid cookies pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.cookies = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_cookies"); + + /* Invalid addrs pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.addrs = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_addrs"); +} + +static void test_tracing_multi_fill_link_info(struct test_fill_link_info *skel, + bool has_cookies, bool invalid) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tmulti_target targets[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT], btf_obj_id; + __u64 cookies[TRACING_MULTI_CNT]; + struct bpf_link *link; + int link_fd, err, i; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + if (setup_tmulti_targets(skel->progs.tmulti_run, targets, &btf_obj_id)) + return; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ids[i] = targets[i].id; + cookies[i] = targets[i].cookie; + } + + opts.ids = ids; + opts.cnt = TRACING_MULTI_CNT; + if (has_cookies) + opts.cookies = cookies; + + link = bpf_program__attach_tracing_multi(skel->progs.tmulti_run, NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi")) + return; + + link_fd = bpf_link__fd(link); + if (invalid) { + verify_tracing_multi_invalid_user_buffer(link_fd, targets); + } else { + err = verify_tracing_multi_link_info(link_fd, skel->progs.tmulti_run, + targets, btf_obj_id, has_cookies); + ASSERT_OK(err, "verify_tracing_multi_link_info"); + } + + bpf_link__destroy(link); +} + #define SEC(name) __attribute__((section(name), used)) static short uprobe_link_info_sema_1 SEC(".probes"); @@ -469,7 +704,7 @@ verify_umulti_link_info(int fd, bool retprobe, __u64 *offsets, ASSERT_EQ(info.uprobe_multi.pid, getpid(), "info.uprobe_multi.pid"); ASSERT_EQ(info.uprobe_multi.count, 3, "info.uprobe_multi.count"); - ASSERT_EQ(info.uprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN, + ASSERT_EQ(info.uprobe_multi.flags & BPF_F_UPROBE_MULTI_RETURN, retprobe, "info.uprobe_multi.flags.retprobe"); ASSERT_EQ(info.uprobe_multi.path_size, strlen(path) + 1, "info.uprobe_multi.path_size"); ASSERT_STREQ(path_buf, path, "info.uprobe_multi.path"); @@ -640,6 +875,13 @@ void test_fill_link_info(void) if (test__start_subtest("kprobe_multi_invalid_ubuff")) test_kprobe_multi_fill_link_info(skel, true, true, true); + if (test__start_subtest("tracing_multi_link_info")) { + test_tracing_multi_fill_link_info(skel, false, false); + test_tracing_multi_fill_link_info(skel, true, false); + } + if (test__start_subtest("tracing_multi_invalid_ubuff")) + test_tracing_multi_fill_link_info(skel, true, true); + if (test__start_subtest("uprobe_multi_link_info")) test_uprobe_multi_fill_link_info(skel, false, false); if (test__start_subtest("uretprobe_multi_link_info")) diff --git a/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c b/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c index 96b27de05524..7bf8adc41e99 100644 --- a/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c +++ b/tools/testing/selftests/bpf/prog_tests/get_func_args_test.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> #include "get_func_args_test.skel.h" +#include "get_func_args_fsession_test.skel.h" void test_get_func_args_test(void) { @@ -41,8 +42,30 @@ void test_get_func_args_test(void) ASSERT_EQ(skel->bss->test4_result, 1, "test4_result"); ASSERT_EQ(skel->bss->test5_result, 1, "test5_result"); ASSERT_EQ(skel->bss->test6_result, 1, "test6_result"); - ASSERT_EQ(skel->bss->test7_result, 1, "test7_result"); cleanup: get_func_args_test__destroy(skel); } + +void test_get_func_args_fsession_test(void) +{ + struct get_func_args_fsession_test *skel = NULL; + int err; + LIBBPF_OPTS(bpf_test_run_opts, topts); + + skel = get_func_args_fsession_test__open_and_load(); + if (!ASSERT_OK_PTR(skel, "get_func_args_fsession_test__open_and_load")) + return; + + err = get_func_args_fsession_test__attach(skel); + if (!ASSERT_OK(err, "get_func_args_fsession_test__attach")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test1), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "test_run"); + + ASSERT_EQ(skel->bss->test1_result, 1, "test1_result"); +cleanup: + get_func_args_fsession_test__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c b/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c index 7772a0f288d3..357fdedfea93 100644 --- a/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c +++ b/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c @@ -2,6 +2,7 @@ #include <test_progs.h> #include "get_func_ip_test.skel.h" #include "get_func_ip_uprobe_test.skel.h" +#include "get_func_ip_fsession_test.skel.h" static noinline void uprobe_trigger(void) { @@ -46,8 +47,6 @@ static void test_function_entry(void) ASSERT_EQ(skel->bss->test5_result, 1, "test5_result"); ASSERT_EQ(skel->bss->test7_result, 1, "test7_result"); ASSERT_EQ(skel->bss->test8_result, 1, "test8_result"); - ASSERT_EQ(skel->bss->test9_entry_result, 1, "test9_entry_result"); - ASSERT_EQ(skel->bss->test9_exit_result, 1, "test9_exit_result"); cleanup: get_func_ip_test__destroy(skel); @@ -139,3 +138,28 @@ void test_get_func_ip_test(void) test_function_entry(); test_function_body(); } + +void test_get_func_ip_fsession_test(void) +{ + struct get_func_ip_fsession_test *skel = NULL; + int err; + LIBBPF_OPTS(bpf_test_run_opts, topts); + + skel = get_func_ip_fsession_test__open_and_load(); + if (!ASSERT_OK_PTR(skel, "get_func_ip_fsession_test__open_and_load")) + return; + + err = get_func_ip_fsession_test__attach(skel); + if (!ASSERT_OK(err, "get_func_ip_fsession_test__attach")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test1), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "test_run"); + + ASSERT_EQ(skel->bss->test1_entry_result, 1, "test1_entry_result"); + ASSERT_EQ(skel->bss->test1_exit_result, 1, "test1_exit_result"); + +cleanup: + get_func_ip_fsession_test__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c b/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c new file mode 100644 index 000000000000..1b5c738ab81f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "bpf/libbpf_internal.h" +#include "get_smp_processor_id.skel.h" + +void test_get_smp_processor_id(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts, + .flags = BPF_F_TEST_RUN_ON_CPU, + .cpu = 0, + ); + struct get_smp_processor_id *skel; + int prog_fd, err, online_cpu_nr, i; + bool *online = NULL; + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", + &online, &online_cpu_nr); + if (!ASSERT_OK(err, "parse_cpu_mask_file")) + return; + + skel = get_smp_processor_id__open_and_load(); + if (!ASSERT_OK_PTR(skel, "get_smp_processor_id__open_and_load")) + goto cleanup; + + prog_fd = bpf_program__fd(skel->progs.call_bpf_get_smp_processor_id); + + for (i = 0; i < online_cpu_nr; i++) { + if (!online[i]) + continue; + + opts.cpu = i; + skel->bss->cpu_nr_result = -1; + + err = bpf_prog_test_run_opts(prog_fd, &opts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto cleanup; + + ASSERT_EQ(skel->bss->cpu_nr_result, opts.cpu, "cpu_nr_result"); + } + +cleanup: + free(online); + get_smp_processor_id__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 8466332d7406..5671c31085cd 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -1,5 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> +#include "bpf/libbpf_internal.h" +#include "test_global_percpu_data.skel.h" +#include "test_global_percpu_data.lskel.h" void test_global_data_init(void) { @@ -60,3 +63,336 @@ out: free(newval); bpf_object__close(obj); } + +static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_fd, int *runp) +{ + struct test_global_percpu_data__percpu *data = NULL; + int i, err, key = 0, num_online, run = 0; + __u64 args[2] = {0x1234ULL, 0x5678ULL}; + size_t data_sz; + bool *online; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", &online, &num_online); + if (!ASSERT_OK(err, "parse_cpu_mask_file")) + return; + + data_sz = map ? bpf_map__value_size(map) : sizeof(*data); + data = calloc(1, data_sz); + if (!ASSERT_OK_PTR(data, "calloc percpu data")) + goto out; + + /* run on every online-CPU */ + for (i = 0; i < num_online; i++) { + __u64 flags; + + if (!online[i]) + continue; + + topts.cpu = i; + topts.retval = -1; + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + ASSERT_EQ(topts.retval, 0, "bpf_prog_test_run_opts retval"); + + memset(data, 0, data_sz); + flags = ((__u64) i << 32) | BPF_F_CPU; + if (map) + err = bpf_map__lookup_elem(map, &key, sizeof(key), data, data_sz, flags); + else + err = bpf_map_lookup_elem_flags(map_fd, &key, data, flags); + if (!ASSERT_OK(err, "lookup_elem on cpu")) + break; + + ASSERT_EQ(*runp, ++run, "run"); + ASSERT_EQ(data->cpu_id[0], i, "cpu_id"); + ASSERT_EQ(data->data, 1, "data"); + ASSERT_TRUE(data->set, "set"); + ASSERT_EQ(data->nums[6], 0xc0de, "nums[6]"); + ASSERT_EQ(data->struct_data.i, 1, "struct_data.i"); + ASSERT_TRUE(data->struct_data.set, "struct_data.set"); + ASSERT_EQ(data->struct_data.nums[6], 0xc0de, "struct_data.nums[6]"); + } + +out: + free(data); + free(online); +} + +static void test_global_percpu_data_init(void) +{ + struct test_global_percpu_data__percpu init_value = {}; + struct test_global_percpu_data__percpu *init_data; + const __u32 desired_sz = sysconf(_SC_PAGE_SIZE); + struct test_global_percpu_data *skel = NULL; + size_t init_data_sz; + struct bpf_map *map; + int prog_fd, err; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu, "skel->percpu")) + goto out; + if (!ASSERT_OK_PTR(skel->data_percpu, "skel->data_percpu")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_data, "skel->percpu_data")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_looooooooong, "skel->percpu_looooooooong")) + goto out; + + ASSERT_STREQ(bpf_map__name(skel->maps.percpu_data), ".percpu.data", + ".percpu.data map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.data_percpu), ".data.percpu", + ".data.percpu map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.percpu_looooooooong), ".percpu.looooooooong", + "long map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.percpu), ".percpu", "map name"); + ASSERT_EQ(skel->percpu->data, -1, "skel->percpu->data"); + ASSERT_FALSE(skel->percpu->set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->nums[6], 0, "skel->percpu->nums[6]"); + ASSERT_EQ(skel->percpu->struct_data.i, -1, "struct_data.i"); + ASSERT_FALSE(skel->percpu->struct_data.set, "struct_data.set"); + ASSERT_EQ(skel->percpu->struct_data.nums[6], 0, "struct_data.nums[6]"); + + map = skel->maps.percpu; + if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type")) + goto out; + + init_value.data = 2; + init_value.nums[6] = -1; + init_value.struct_data.i = 2; + init_value.struct_data.nums[6] = -1; + err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value)); + if (!ASSERT_OK(err, "bpf_map__set_initial_value")) + goto out; + + init_data = bpf_map__initial_value(map, &init_data_sz); + if (!ASSERT_OK_PTR(init_data, "bpf_map__initial_value")) + goto out; + + ASSERT_EQ(init_data->data, init_value.data, "init_value data"); + ASSERT_EQ(init_data->set, init_value.set, "init_value set"); + ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i"); + ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6], + "init_value struct_data.nums[6]"); + ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size"); + ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data"); + ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data"); + ASSERT_EQ(skel->percpu->set, init_value.set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i, + "skel->percpu->struct_data.i"); + ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6], + "skel->percpu->struct_data.nums[6]"); + + ASSERT_GT(desired_sz, sizeof(init_value), "desired_sz"); + err = bpf_map__set_value_size(map, desired_sz); + if (!ASSERT_OK(err, "bpf_map__set_value_size")) + goto out; + if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "percpu value size")) + goto out; + if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "percpu BTF value type")) + goto out; + + init_data = bpf_map__initial_value(map, &init_data_sz); + if (!ASSERT_OK_PTR(init_data, "resized bpf_map__initial_value")) + goto out; + if (!ASSERT_EQ(init_data_sz, desired_sz, "resized initial value size")) + goto out; + if (!ASSERT_EQ(init_data->data, init_value.data, "resized initial value data")) + goto out; + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + ASSERT_OK_PTR(skel->percpu, "skel->percpu"); + + prog_fd = bpf_program__fd(skel->progs.update_percpu_data); + test_percpu_data_on_cpus(map, bpf_map__fd(map), prog_fd, &skel->bss->run); + +out: + test_global_percpu_data__destroy(skel); +} + +static void test_global_percpu_data_lskel(void) +{ + struct test_global_percpu_data_lskel *lskel = NULL; + int prog_fd, map_fd; + + lskel = test_global_percpu_data_lskel__open_and_load(); + if (!ASSERT_OK_PTR(lskel, "test_global_percpu_data_lskel__open_and_load")) + goto out; + + map_fd = lskel->maps.percpu.map_fd; + prog_fd = lskel->progs.update_percpu_data.prog_fd; + test_percpu_data_on_cpus(NULL, map_fd, prog_fd, &lskel->bss->run); + +out: + test_global_percpu_data_lskel__destroy(lskel); +} + +static int create_rdonly_percpu_array(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .map_flags = BPF_F_RDONLY_PROG, + ); + int key = 0, map_fd, err; + __u64 value = 0; + + map_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "percpu_ro_map", sizeof(int), + sizeof(__u64), 1, &map_opts); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) + return -1; + + err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out; + + err = bpf_map_freeze(map_fd); + if (!ASSERT_OK(err, "bpf_map_freeze")) + goto out; + + return map_fd; + +out: + close(map_fd); + return -1; +} + +static void test_global_percpu_data_rdonly_direct_read(void) +{ + /* + * Raw instructions with manually prepared rdonly percpu_array map + * for testing direct-read global percpu data, because libbpf + * doesn't have rdonly internal percpu_array map support for + * global percpu data. + */ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_EXIT_INSN(), + }; + int map_fd, prog_fd; + + map_fd = create_rdonly_percpu_array(); + if (map_fd < 0) + return; + + insns[0].imm = map_fd; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns, + ARRAY_SIZE(insns), NULL); + if (ASSERT_GE(prog_fd, 0, "bpf_prog_load")) + close(prog_fd); + close(map_fd); +} + +static void test_global_percpu_data_rdonly_direct_write(void) +{ + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); + /* See the comment in test_global_percpu_data_rdonly_direct_read() */ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0), + BPF_EXIT_INSN(), + }; + char log_buf[256] = {}; + int map_fd, prog_fd; + + prog_opts.log_buf = log_buf; + prog_opts.log_size = sizeof(log_buf); + prog_opts.log_level = 1; + + map_fd = create_rdonly_percpu_array(); + if (map_fd < 0) + return; + + insns[0].imm = map_fd; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns, + ARRAY_SIZE(insns), &prog_opts); + if (!ASSERT_LT(prog_fd, 0, "bpf_prog_load")) + close(prog_fd); + else + ASSERT_HAS_SUBSTR(log_buf, "write into map forbidden", "verifier log"); + close(map_fd); +} + +static void test_global_percpu_data_verifier_log(void) +{ + RUN_TESTS(test_global_percpu_data); +} + +static void test_global_percpu_data_iter(void) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + struct test_global_percpu_data *skel; + union bpf_iter_link_info linfo = {}; + struct bpf_link *link = NULL; + int fd, num_cpus, len, err; + char buf[16]; + + num_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(num_cpus, 0, "libbpf_num_possible_cpus")) + return; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + return; + + skel->rodata->num_cpus = num_cpus; + skel->rodata->num_off = offsetof(struct test_global_percpu_data__percpu, + struct_data.nums[6]); + skel->rodata->elem_sz = roundup(sizeof(struct test_global_percpu_data__percpu), 8); + skel->percpu->struct_data.nums[6] = 0xc0de; + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + linfo.map.map_fd = bpf_map__fd(skel->maps.percpu); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + link = bpf_program__attach_iter(skel->progs.dump_percpu_data, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_iter")) + goto out; + + fd = bpf_iter_create(bpf_link__fd(link)); + if (!ASSERT_GE(fd, 0, "bpf_iter_create")) + goto out; + + while ((len = read(fd, buf, sizeof(buf))) > 0) + do { } while (0); + ASSERT_EQ(len, 0, "read iter"); + ASSERT_TRUE(skel->bss->run_iter, "run_iter"); + ASSERT_EQ(skel->bss->sum, 0xc0de * num_cpus, "sum"); + + close(fd); +out: + bpf_link__destroy(link); + test_global_percpu_data__destroy(skel); +} + +void test_global_percpu_data(void) +{ + if (!feat_supported(NULL, FEAT_PERCPU_DATA)) { + test__skip(); + return; + } + + if (test__start_subtest("init")) + test_global_percpu_data_init(); + if (test__start_subtest("lskel")) + test_global_percpu_data_lskel(); + if (test__start_subtest("rdonly_direct_read")) + test_global_percpu_data_rdonly_direct_read(); + if (test__start_subtest("rdonly_direct_write")) + test_global_percpu_data_rdonly_direct_write(); + test_global_percpu_data_verifier_log(); + if (test__start_subtest("iter")) + test_global_percpu_data_iter(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c index 56b5baef35c8..602ce30f1720 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c +++ b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c @@ -23,6 +23,7 @@ static void global_map_resize_bss_subtest(void) struct bpf_map *map; const __u32 desired_sz = sizeof(skel->bss->sum) + sysconf(_SC_PAGE_SIZE) * 2; size_t array_len, actual_sz, new_sz; + int *array; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -58,10 +59,13 @@ static void global_map_resize_bss_subtest(void) goto teardown; /* fill the newly resized array with ones, - * skipping the first element which was previously set + * skipping the first element which was previously set; + * access through a plain pointer to avoid -Warray-bounds + * since the array was resized beyond its declared length. */ + array = skel->bss->array; for (int i = 1; i < array_len; i++) - skel->bss->array[i] = 1; + array[i] = 1; /* set global const values before loading */ skel->rodata->pid = getpid(); diff --git a/tools/testing/selftests/bpf/prog_tests/htab_reuse.c b/tools/testing/selftests/bpf/prog_tests/htab_reuse.c index a742dd994d60..d7c3df165adc 100644 --- a/tools/testing/selftests/bpf/prog_tests/htab_reuse.c +++ b/tools/testing/selftests/bpf/prog_tests/htab_reuse.c @@ -59,7 +59,7 @@ static void *htab_update_fn(void *arg) return NULL; } -void test_htab_reuse(void) +static void test_htab_reuse_basic(void) { unsigned int i, wr_nr = 1, rd_nr = 4; pthread_t tids[wr_nr + rd_nr]; @@ -99,3 +99,170 @@ reap: } htab_reuse__destroy(skel); } + +/* + * Writes consistency test for BPF_F_LOCK update + * + * The race: + * 1. Thread A: BPF_F_LOCK|BPF_EXIST update + * 2. Thread B: delete element then update it with BPF_ANY + */ + +struct htab_val_large { + struct bpf_spin_lock lock; + __u32 seq; + __u64 data[256]; +}; + +struct consistency_ctx { + int fd; + int start_fd; + int loop; + volatile bool torn_write; +}; + +static void wait_for_start(int fd) +{ + char buf; + + read(fd, &buf, 1); +} + +static void *locked_update_fn(void *arg) +{ + struct consistency_ctx *ctx = arg; + struct htab_val_large value; + unsigned int key = 1; + int i; + + memset(&value, 0xAA, sizeof(value)); + wait_for_start(ctx->start_fd); + + for (i = 0; i < ctx->loop; i++) { + value.seq = i; + bpf_map_update_elem(ctx->fd, &key, &value, + BPF_F_LOCK | BPF_EXIST); + } + + return NULL; +} + +/* Delete + update: removes the element then re-creates it with BPF_ANY. */ +static void *delete_update_fn(void *arg) +{ + struct consistency_ctx *ctx = arg; + struct htab_val_large value; + unsigned int key = 1; + int i; + + memset(&value, 0xBB, sizeof(value)); + + wait_for_start(ctx->start_fd); + + for (i = 0; i < ctx->loop; i++) { + value.seq = i; + bpf_map_delete_elem(ctx->fd, &key); + bpf_map_update_elem(ctx->fd, &key, &value, BPF_ANY | BPF_F_LOCK); + } + + return NULL; +} + +static void *locked_lookup_fn(void *arg) +{ + struct consistency_ctx *ctx = arg; + struct htab_val_large value; + unsigned int key = 1; + int i, j; + + wait_for_start(ctx->start_fd); + + for (i = 0; i < ctx->loop && !ctx->torn_write; i++) { + if (bpf_map_lookup_elem_flags(ctx->fd, &key, &value, BPF_F_LOCK)) + continue; + + for (j = 0; j < 256; j++) { + if (value.data[j] != value.data[0]) { + ctx->torn_write = true; + return NULL; + } + } + } + + return NULL; +} + +static void test_htab_reuse_consistency(void) +{ + int threads_total = 6, threads = 2; + pthread_t tids[threads_total]; + struct consistency_ctx ctx; + struct htab_val_large seed; + struct htab_reuse *skel; + unsigned int key = 1, i; + int pipefd[2]; + int err; + + skel = htab_reuse__open_and_load(); + if (!ASSERT_OK_PTR(skel, "htab_reuse__open_and_load")) + return; + + if (!ASSERT_OK(pipe(pipefd), "pipe")) + goto out; + + ctx.fd = bpf_map__fd(skel->maps.htab_lock_consistency); + ctx.start_fd = pipefd[0]; + ctx.loop = 100000; + ctx.torn_write = false; + + /* Seed the element so locked updaters have something to find */ + memset(&seed, 0xBB, sizeof(seed)); + err = bpf_map_update_elem(ctx.fd, &key, &seed, BPF_ANY); + if (!ASSERT_OK(err, "seed_element")) + goto close_pipe; + + memset(tids, 0, sizeof(tids)); + for (i = 0; i < threads; i++) { + err = pthread_create(&tids[i], NULL, locked_update_fn, &ctx); + if (!ASSERT_OK(err, "pthread_create")) + goto stop; + } + for (i = 0; i < threads; i++) { + err = pthread_create(&tids[threads + i], NULL, delete_update_fn, &ctx); + if (!ASSERT_OK(err, "pthread_create")) + goto stop; + } + for (i = 0; i < threads; i++) { + err = pthread_create(&tids[threads * 2 + i], NULL, locked_lookup_fn, &ctx); + if (!ASSERT_OK(err, "pthread_create")) + goto stop; + } + + /* Release all threads simultaneously */ + close(pipefd[1]); + pipefd[1] = -1; + +stop: + for (i = 0; i < threads_total; i++) { + if (!tids[i]) + continue; + pthread_join(tids[i], NULL); + } + + ASSERT_FALSE(ctx.torn_write, "no torn writes detected"); + +close_pipe: + if (pipefd[1] >= 0) + close(pipefd[1]); + close(pipefd[0]); +out: + htab_reuse__destroy(skel); +} + +void test_htab_reuse(void) +{ + if (test__start_subtest("basic")) + test_htab_reuse_basic(); + if (test__start_subtest("consistency")) + test_htab_reuse_consistency(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/htab_update.c b/tools/testing/selftests/bpf/prog_tests/htab_update.c index ea1a6766fbe9..0a28d4346924 100644 --- a/tools/testing/selftests/bpf/prog_tests/htab_update.c +++ b/tools/testing/selftests/bpf/prog_tests/htab_update.c @@ -23,7 +23,7 @@ static void test_reenter_update(void) if (!ASSERT_OK_PTR(skel, "htab_update__open")) return; - bpf_program__set_autoload(skel->progs.bpf_obj_free_fields, true); + bpf_program__set_autoload(skel->progs.bpf_obj_cancel_fields, true); err = htab_update__load(skel); if (!ASSERT_TRUE(!err, "htab_update__load") || err) goto out; @@ -50,7 +50,7 @@ static void test_reenter_update(void) /* * Second update: replace existing element with same key and trigger * the reentrancy of bpf_map_update_elem(). - * check_and_free_fields() calls bpf_obj_free_fields() on the old + * check_and_cancel_fields() calls bpf_obj_cancel_fields() on the old * value, which is where fentry program runs and performs a nested * bpf_map_update_elem(), triggering -EDEADLK. */ diff --git a/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c b/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c new file mode 100644 index 000000000000..9318d4bc7ce8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <network_helpers.h> +#include <cgroup_helpers.h> +#include <linux/errqueue.h> +#include <poll.h> +#include <unistd.h> +#include "icmp_send.skel.h" + +#define TIMEOUT_MS 1000 + +#define ICMP_DEST_UNREACH 3 +#define ICMPV6_DEST_UNREACH 1 + +#define ICMP_HOST_UNREACH 1 +#define ICMP_FRAG_NEEDED 4 +#define NR_ICMP_UNREACH 15 +#define ICMPV6_REJECT_ROUTE 6 + +#define KFUNC_RET_UNSET -1 + +static int connect_to_fd_nonblock(int server_fd) +{ + struct sockaddr_storage addr; + socklen_t len = sizeof(addr); + int fd, err, on = 1; + + if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) + return -1; + + fd = socket(addr.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0); + if (fd < 0) + return -1; + + if (addr.ss_family == AF_INET6 && + setsockopt(fd, IPPROTO_IPV6, IPV6_RECVERR, &on, sizeof(on)) < 0) { + close(fd); + return -1; + } + + err = connect(fd, (struct sockaddr *)&addr, len); + if (err < 0 && errno != EINPROGRESS) { + close(fd); + return -1; + } + + return fd; +} + +static void read_icmp_errqueue(int sockfd, int expected_code, int af) +{ + int expected_ee_type = (af == AF_INET) ? ICMP_DEST_UNREACH : + ICMPV6_DEST_UNREACH; + int expected_origin = (af == AF_INET) ? SO_EE_ORIGIN_ICMP : + SO_EE_ORIGIN_ICMP6; + int expected_level = (af == AF_INET) ? IPPROTO_IP : IPPROTO_IPV6; + int expected_type = (af == AF_INET) ? IP_RECVERR : IPV6_RECVERR; + struct sock_extended_err *sock_err; + char ctrl_buf[512]; + struct msghdr msg = { + .msg_control = ctrl_buf, + .msg_controllen = sizeof(ctrl_buf), + }; + struct pollfd pfd = { + .fd = sockfd, + .events = POLLERR, + }; + struct cmsghdr *cm; + ssize_t n; + + if (!ASSERT_GE(poll(&pfd, 1, TIMEOUT_MS), 1, "poll_errqueue")) + return; + + n = recvmsg(sockfd, &msg, MSG_ERRQUEUE); + if (!ASSERT_GE(n, 0, "recvmsg_errqueue")) + return; + + cm = CMSG_FIRSTHDR(&msg); + if (!ASSERT_NEQ(cm, NULL, "cm_firsthdr_null")) + return; + + for (; cm; cm = CMSG_NXTHDR(&msg, cm)) { + if (cm->cmsg_level != expected_level || + cm->cmsg_type != expected_type) + continue; + + sock_err = (struct sock_extended_err *)CMSG_DATA(cm); + + if (!ASSERT_EQ(sock_err->ee_origin, expected_origin, + "sock_err_origin")) + return; + if (!ASSERT_EQ(sock_err->ee_type, expected_ee_type, + "sock_err_type_dest_unreach")) + return; + ASSERT_EQ(sock_err->ee_code, expected_code, "sock_err_code"); + return; + } + + ASSERT_FAIL("no IP_RECVERR/IPV6_RECVERR control message found"); +} + +static bool valid_unreach_code(int code, int af) +{ + if (code < 0) + return false; + + if (af == AF_INET) + return code <= NR_ICMP_UNREACH && code != ICMP_FRAG_NEEDED; + + return code <= ICMPV6_REJECT_ROUTE; +} + +static void trigger_prog_read_icmp_errqueue(struct icmp_send *skel, int code, + int af, const char *ip) +{ + int srv_fd = -1, client_fd = -1; + int port; + + srv_fd = start_server(af, SOCK_STREAM, ip, 0, TIMEOUT_MS); + if (!ASSERT_OK_FD(srv_fd, "start_server")) + return; + + port = get_socket_local_port(srv_fd); + if (!ASSERT_GE(port, 0, "get_socket_local_port")) { + close(srv_fd); + return; + } + + skel->bss->server_port = ntohs(port); + skel->bss->unreach_type = (af == AF_INET) ? ICMP_DEST_UNREACH : + ICMPV6_DEST_UNREACH; + skel->bss->unreach_code = code; + skel->data->kfunc_ret = KFUNC_RET_UNSET; + + client_fd = connect_to_fd_nonblock(srv_fd); + if (!ASSERT_OK_FD(client_fd, "client_connect_nonblock")) { + close(srv_fd); + return; + } + + if (valid_unreach_code(code, af)) + read_icmp_errqueue(client_fd, code, af); + + close(client_fd); + close(srv_fd); +} + +static void run_icmp_test(struct icmp_send *skel, int af, const char *ip, + int max_code) +{ + for (int code = 0; code <= max_code; code++) { + if (af == AF_INET && code == ICMP_FRAG_NEEDED) + continue; + + trigger_prog_read_icmp_errqueue(skel, code, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, 0, "kfunc_ret"); + } + + /* Test invalid codes */ + trigger_prog_read_icmp_errqueue(skel, -1, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + + trigger_prog_read_icmp_errqueue(skel, max_code + 1, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + + if (af == AF_INET) { + trigger_prog_read_icmp_errqueue(skel, ICMP_FRAG_NEEDED, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + } +} + +static void run_icmp_no_route_test(struct icmp_send *skel, int af) +{ + union { + struct ipv4_packet v4; + struct ipv6_packet v6; + } pkt; + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = &pkt, + ); + int err; + + switch (af) { + case AF_INET: + pkt.v4 = pkt_v4; + pkt.v4.iph.version = 4; + pkt.v4.iph.daddr = htonl(INADDR_LOOPBACK); + pkt.v4.tcp.dest = htons(80); + opts.data_size_in = sizeof(pkt.v4); + skel->bss->unreach_type = ICMP_DEST_UNREACH; + break; + case AF_INET6: + pkt.v6 = pkt_v6; + pkt.v6.iph.version = 6; + pkt.v6.iph.daddr = in6addr_loopback; + pkt.v6.tcp.dest = htons(80); + opts.data_size_in = sizeof(pkt.v6); + skel->bss->unreach_type = ICMPV6_DEST_UNREACH; + break; + default: + ASSERT_FAIL("af_not_supported"); + return; + } + + skel->bss->server_port = 80; + skel->data->kfunc_ret = KFUNC_RET_UNSET; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.egress), &opts); + if (!ASSERT_OK(err, "test_run")) + return; + + ASSERT_EQ(skel->data->kfunc_ret, -ENETUNREACH, "kfunc_ret_no_route"); +} + +void test_icmp_send_unreach_cgroup(void) +{ + struct icmp_send *skel; + int cgroup_fd = -1; + + skel = icmp_send__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + cgroup_fd = test__join_cgroup("/icmp_send_unreach_cgroup"); + if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup")) + goto cleanup; + + skel->links.egress = + bpf_program__attach_cgroup(skel->progs.egress, cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.egress, "prog_attach_cgroup")) + goto cleanup; + + if (test__start_subtest("ipv4")) + run_icmp_test(skel, AF_INET, "127.0.0.1", NR_ICMP_UNREACH); + + if (test__start_subtest("ipv6")) + run_icmp_test(skel, AF_INET6, "::1", ICMPV6_REJECT_ROUTE); + + if (test__start_subtest("no_route_ipv4")) + run_icmp_no_route_test(skel, AF_INET); + + if (test__start_subtest("no_route_ipv6")) + run_icmp_no_route_test(skel, AF_INET6); + +cleanup: + icmp_send__destroy(skel); + if (cgroup_fd >= 0) + close(cgroup_fd); +} + +void test_icmp_send_unreach_recursion(void) +{ + struct icmp_send *skel; + int cgroup_fd = -1; + int err; + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + return; + + skel = icmp_send__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + cgroup_fd = get_root_cgroup(); + if (!ASSERT_OK_FD(cgroup_fd, "get_root_cgroup")) + goto cleanup; + + skel->data->target_pid = getpid(); + skel->links.recursion = + bpf_program__attach_cgroup(skel->progs.recursion, cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.recursion, "prog_attach_cgroup")) + goto cleanup; + + trigger_prog_read_icmp_errqueue(skel, ICMP_HOST_UNREACH, AF_INET, + "127.0.0.1"); + + /* + * Because there's recursion involved, the first call will return at + * index 1 since it will return the second, and the second call will + * return at index 0 since it will return the first. + */ + ASSERT_EQ(skel->bss->rec_count, 2, "rec_count"); + ASSERT_EQ(skel->data->rec_kfunc_rets[0], -EBUSY, "kfunc_rets[0]"); + ASSERT_EQ(skel->data->rec_kfunc_rets[1], 0, "kfunc_rets[1]"); + +cleanup: + icmp_send__destroy(skel); + if (cgroup_fd >= 0) + close(cgroup_fd); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/iter_buf_null_fail.c b/tools/testing/selftests/bpf/prog_tests/iter_buf_null_fail.c new file mode 100644 index 000000000000..ea97787b870d --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/iter_buf_null_fail.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "iter_buf_null_fail.skel.h" + +void test_iter_buf_null_fail(void) +{ + RUN_TESTS(iter_buf_null_fail); +} diff --git a/tools/testing/selftests/bpf/prog_tests/iters.c b/tools/testing/selftests/bpf/prog_tests/iters.c index a539980a2fbe..c0b6082f345a 100644 --- a/tools/testing/selftests/bpf/prog_tests/iters.c +++ b/tools/testing/selftests/bpf/prog_tests/iters.c @@ -202,8 +202,6 @@ cleanup: iters_task__destroy(skel); } -extern int stack_mprotect(void); - static void subtest_css_task_iters(void) { struct iters_css_task *skel = NULL; diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c index f79c8e53cb3e..2b39cc1b09f9 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c @@ -66,14 +66,20 @@ static struct kfunc_test_params kfunc_tests[] = { TC_FAIL(kfunc_call_test_get_mem_fail_rdonly, 0, "R0 cannot write into rdonly_mem"), TC_FAIL(kfunc_call_test_get_mem_fail_use_after_free, 0, "invalid mem access 'scalar'"), TC_FAIL(kfunc_call_test_get_mem_fail_oob, 0, "min value is outside of the allowed memory range"), + TC_FAIL(kfunc_call_test_get_mem_fail_zero_size, 0, "min value is outside of the allowed memory range"), + TC_FAIL(kfunc_call_test_get_mem_fail_oversized, 0, "allocation size exceeds u32 max"), TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"), TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"), - TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "arg#0 expected pointer to ctx, but got scalar"), + TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 expected pointer to ctx, but got scalar"), + TC_FAIL(kfunc_call_test_spin_lock_unsafe, 0, "function calls are not allowed while holding a lock"), /* success cases */ + TC_TEST(kfunc_call_test_spin_lock_safe, 0), TC_TEST(kfunc_call_test1, 12), TC_TEST(kfunc_call_test2, 3), TC_TEST(kfunc_call_test4, -1234), + TC_TEST(kfunc_call_test5, 0), + TC_TEST(kfunc_call_test5_asm, 0), TC_TEST(kfunc_call_test_ref_btf_id, 0), TC_TEST(kfunc_call_test_get_mem, 42), SYSCALL_TEST(kfunc_syscall_test, 0), diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c b/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c index 8cd298b78e44..04aaf4c9cf5e 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c @@ -14,7 +14,7 @@ static struct { const char *prog_name; int expected_runtime_err; } kfunc_dynptr_tests[] = { - {"dynptr_data_null", -EBADMSG}, + {"dynptr_data_null", -EINVAL}, }; static bool kfunc_not_supported; diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_implicit_args_tracing.c b/tools/testing/selftests/bpf/prog_tests/kfunc_implicit_args_tracing.c new file mode 100644 index 000000000000..61cc5aaba025 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_implicit_args_tracing.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include "kfunc_implicit_args_tracing.skel.h" + +void test_kfunc_implicit_args_tracing(void) +{ + struct kfunc_implicit_args_tracing *skel; + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, fd; + + skel = kfunc_implicit_args_tracing__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + err = kfunc_implicit_args_tracing__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto cleanup; + + fd = bpf_program__fd(skel->progs.trigger_implicit_arg); + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run")) + goto cleanup; + + ASSERT_EQ(topts.retval, 5, "kfunc_retval"); + ASSERT_EQ(skel->bss->fentry_arg_cnt, 2, "fentry_arg_cnt"); + ASSERT_NEQ(skel->bss->fentry_aux_arg, 0, "fentry_aux_arg"); + ASSERT_EQ(skel->bss->fentry_result, 1, "fentry_result"); + ASSERT_EQ(skel->bss->fexit_arg_cnt, 2, "fexit_arg_cnt"); + ASSERT_NEQ(skel->bss->fexit_aux_arg, 0, "fexit_aux_arg"); + ASSERT_EQ(skel->bss->fexit_result, 1, "fexit_result"); + +cleanup: + kfunc_implicit_args_tracing__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c index f81dcd609ee9..2e0ddef77ba5 100644 --- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c @@ -10,6 +10,7 @@ #include "kprobe_multi_session_cookie.skel.h" #include "kprobe_multi_verifier.skel.h" #include "kprobe_write_ctx.skel.h" +#include "kprobe_multi_sleepable.skel.h" #include "bpf/libbpf_internal.h" #include "bpf/hashmap.h" @@ -220,7 +221,9 @@ static void test_attach_api_syms(void) static void test_attach_api_fails(void) { LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); struct kprobe_multi *skel = NULL; + struct kprobe_multi_sleepable *sl_skel = NULL; struct bpf_link *link = NULL; unsigned long long addrs[2]; const char *syms[2] = { @@ -228,7 +231,7 @@ static void test_attach_api_fails(void) "bpf_fentry_test2", }; __u64 cookies[2]; - int saved_error; + int saved_error, err; addrs[0] = ksym_get_addr("bpf_fentry_test1"); addrs[1] = ksym_get_addr("bpf_fentry_test2"); @@ -327,9 +330,63 @@ static void test_attach_api_fails(void) if (!ASSERT_EQ(saved_error, -E2BIG, "fail_6_error")) goto cleanup; + /* fail_7 - non-existent wildcard pattern (slow path) */ + LIBBPF_OPTS_RESET(opts); + + link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, + "__nonexistent_func_xyz_*", + &opts); + saved_error = -errno; + if (!ASSERT_ERR_PTR(link, "fail_7")) + goto cleanup; + + if (!ASSERT_EQ(saved_error, -ENOENT, "fail_7_error")) + goto cleanup; + + /* fail_8 - non-existent exact name (fast path), same error as wildcard */ + link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, + "__nonexistent_func_xyz_123", + &opts); + saved_error = -errno; + if (!ASSERT_ERR_PTR(link, "fail_8")) + goto cleanup; + + if (!ASSERT_EQ(saved_error, -ENOENT, "fail_8_error")) + goto cleanup; + + /* fail_9 - sleepable kprobe multi should not attach */ + sl_skel = kprobe_multi_sleepable__open(); + if (!ASSERT_OK_PTR(sl_skel, "sleep_skel_open")) + goto cleanup; + + sl_skel->bss->user_ptr = sl_skel; + + err = bpf_program__set_flags(sl_skel->progs.handle_kprobe_multi_sleepable, + BPF_F_SLEEPABLE); + if (!ASSERT_OK(err, "sleep_skel_set_flags")) + goto cleanup; + + err = kprobe_multi_sleepable__load(sl_skel); + if (!ASSERT_OK(err, "sleep_skel_load")) + goto cleanup; + + link = bpf_program__attach_kprobe_multi_opts(sl_skel->progs.handle_kprobe_multi_sleepable, + "bpf_fentry_test1", NULL); + saved_error = -errno; + + if (!ASSERT_ERR_PTR(link, "fail_9")) + goto cleanup; + + if (!ASSERT_EQ(saved_error, -EINVAL, "fail_9_error")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(sl_skel->progs.fentry), &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + cleanup: bpf_link__destroy(link); kprobe_multi__destroy(skel); + kprobe_multi_sleepable__destroy(sl_skel); } static void test_session_skel_api(void) @@ -355,8 +412,13 @@ static void test_session_skel_api(void) ASSERT_OK(err, "test_run"); ASSERT_EQ(topts.retval, 0, "test_run"); - /* bpf_fentry_test1-4 trigger return probe, result is 2 */ - for (i = 0; i < 4; i++) + /* + * bpf_fentry_test1 is hit by both the wildcard probe and the exact + * name probe (test_kprobe_syms), so entry + return fires twice: 4. + * bpf_fentry_test2-4 are hit only by the wildcard probe: 2. + */ + ASSERT_EQ(skel->bss->kprobe_session_result[0], 4, "kprobe_session_result"); + for (i = 1; i < 4; i++) ASSERT_EQ(skel->bss->kprobe_session_result[i], 2, "kprobe_session_result"); /* bpf_fentry_test5-8 trigger only entry probe, result is 1 */ @@ -604,6 +666,44 @@ static void test_attach_write_ctx(void) } #endif +/* + * Test kprobe_multi handles shadow symbols (vmlinux + module duplicate). + * bpf_fentry_shadow_test exists in both vmlinux and bpf_testmod. + * kprobe_multi resolves via ftrace_lookup_symbols() which finds the + * vmlinux symbol first and stops, so this should always succeed. + */ +static void test_attach_probe_dup_sym(void) +{ + LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); + const char *syms[1] = { "bpf_fentry_shadow_test" }; + struct kprobe_multi *skel = NULL; + struct bpf_link *link1 = NULL, *link2 = NULL; + + skel = kprobe_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load")) + goto cleanup; + + skel->bss->pid = getpid(); + opts.syms = syms; + opts.cnt = ARRAY_SIZE(syms); + + link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual, + NULL, &opts); + if (!ASSERT_OK_PTR(link1, "attach_kprobe_multi_dup_sym")) + goto cleanup; + + opts.retprobe = true; + link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual, + NULL, &opts); + if (!ASSERT_OK_PTR(link2, "attach_kretprobe_multi_dup_sym")) + goto cleanup; + +cleanup: + bpf_link__destroy(link2); + bpf_link__destroy(link1); + kprobe_multi__destroy(skel); +} + void serial_test_kprobe_multi_bench_attach(void) { if (test__start_subtest("kernel")) @@ -647,5 +747,7 @@ void test_kprobe_multi_test(void) test_unique_match(); if (test__start_subtest("attach_write_ctx")) test_attach_write_ctx(); + if (test__start_subtest("dup_sym")) + test_attach_probe_dup_sym(); RUN_TESTS(kprobe_multi_verifier); } diff --git a/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c b/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c index 7def158da9eb..1215d6edd590 100644 --- a/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c +++ b/tools/testing/selftests/bpf/prog_tests/kptr_xchg_inline.c @@ -14,7 +14,8 @@ void test_kptr_xchg_inline(void) int err; #if !(defined(__x86_64__) || defined(__aarch64__) || \ - (defined(__riscv) && __riscv_xlen == 64)) + (defined(__riscv) && __riscv_xlen == 64) || \ + (defined(__loongarch__) && __loongarch_grlen == 64)) test__skip(); return; #endif diff --git a/tools/testing/selftests/bpf/prog_tests/ksock.c b/tools/testing/selftests/bpf/prog_tests/ksock.c new file mode 100644 index 000000000000..dd6b167623d9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ksock.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <arpa/inet.h> + +#include "test_progs.h" +#include "network_helpers.h" +#include "ksock_lsm.skel.h" +#include "ksock_lsm_verifier.skel.h" + +#define NS_TEST "ksock_lsm_ns" +#define RECV_PORT 7777 +#define RECV_TIMEOUT_SEC 5 + +struct ksock_test_env { + struct nstoken *nstoken; + int rfd; +}; + +static bool ksock_test_env_setup(struct ksock_test_env *env) +{ + struct sockaddr_in addr = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + .sin_port = htons(RECV_PORT), + }; + struct timeval tv = { .tv_sec = RECV_TIMEOUT_SEC }; + int err; + + memset(env, 0, sizeof(*env)); + env->rfd = -1; + + if (!ASSERT_OK(make_netns(NS_TEST), "make_netns")) + goto fail; + + env->nstoken = open_netns(NS_TEST); + if (!ASSERT_OK_PTR(env->nstoken, "open_netns")) + goto fail; + + env->rfd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (!ASSERT_OK_FD(env->rfd, "receiver socket")) + goto fail; + + err = bind(env->rfd, (struct sockaddr *)&addr, sizeof(addr)); + if (!ASSERT_OK(err, "bind receiver")) + goto fail; + + err = setsockopt(env->rfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + if (!ASSERT_OK(err, "set rcvtimeo")) + goto fail; + + return true; + +fail: + return false; +} + +void test_ksock_lsm(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct ksock_test_env env; + struct sockaddr_in trigger_addr = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + }; + struct ksock_lsm *skel; + char recv_data[sizeof(skel->data->send_data)] = {}; + ssize_t n; + int tfd = -1; + int err; + + skel = ksock_lsm__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + + if (!ksock_test_env_setup(&env)) + goto fail; + + /* Step 1: Run the setup SYSCALL prog to create the ksock */ + skel->bss->ipv4_remote = htonl(INADDR_LOOPBACK); + skel->bss->remote_port = RECV_PORT; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_setup), + &opts); + if (!ASSERT_OK(err, "ksock_setup run")) + goto fail; + if (!ASSERT_OK(opts.retval, "ksock_setup retval")) + goto fail; + + /* Step 2: Attach LSM prog and trigger socket_bind from userspace */ + skel->links.ksock_socket_bind = + bpf_program__attach_lsm(skel->progs.ksock_socket_bind); + if (!ASSERT_OK_PTR(skel->links.ksock_socket_bind, + "attach socket_bind lsm")) + goto fail; + + tfd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (!ASSERT_OK_FD(tfd, "trigger socket")) + goto fail; + + skel->bss->target_pid = getpid(); + err = bind(tfd, (struct sockaddr *)&trigger_addr, sizeof(trigger_addr)); + skel->bss->target_pid = 0; + if (!ASSERT_OK(err, "trigger bind")) + goto fail; + + /* Step 3: Verify the LSM hook sent the notification */ + if (!ASSERT_EQ(skel->data->send_ret, sizeof(skel->data->send_data), + "LSM send bytes")) + goto fail; + + n = recvfrom(env.rfd, recv_data, sizeof(recv_data), 0, NULL, NULL); + if (ASSERT_EQ(n, sizeof(recv_data), "recvfrom len")) + ASSERT_MEMEQ(recv_data, skel->data->send_data, sizeof(recv_data), + "payload match"); + +fail: + if (tfd >= 0) + close(tfd); + if (env.rfd >= 0) + close(env.rfd); + if (env.nstoken) + close_netns(env.nstoken); + remove_netns(NS_TEST); + ksock_lsm__destroy(skel); +} + +void test_ksock_lsm_verifier(void) +{ + RUN_TESTS(ksock_lsm_verifier); +} diff --git a/tools/testing/selftests/bpf/prog_tests/ksock_wq.c b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c new file mode 100644 index 000000000000..d6dc20b8f95b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <unistd.h> + +#include "test_progs.h" +#include "ksock_wq.skel.h" + +#define CALLBACK_WAIT_RETRIES 1000 +#define CALLBACK_WAIT_US 1000 + +void test_ksock_wq(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct ksock_wq *skel; + u32 callback_done; + int err, i; + + skel = ksock_wq__open_and_load(); + if (!ASSERT_OK_PTR(skel, "ksock_wq open and load")) + return; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_wq_start), + &opts); + if (!ASSERT_OK(err, "run ksock_wq_start")) + goto out; + if (!ASSERT_OK(opts.retval, "ksock_wq_start retval")) + goto out; + + for (i = 0; i < CALLBACK_WAIT_RETRIES; i++) { + if (__atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE)) + break; + usleep(CALLBACK_WAIT_US); + } + callback_done = __atomic_load_n(&skel->bss->callback_done, + __ATOMIC_ACQUIRE); + if (!ASSERT_EQ(callback_done, 1, "workqueue callback completed")) + goto out; + + ASSERT_EQ(skel->bss->create_err, -EOPNOTSUPP, + "workqueue create rejected"); + +out: + ksock_wq__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/libarena.c b/tools/testing/selftests/bpf/prog_tests/libarena.c new file mode 100644 index 000000000000..daade4150af6 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/libarena.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include <unistd.h> + +#include <libarena/common.h> +#include <libarena/asan.h> +#include <libarena/buddy.h> +#include <libarena/userspace.h> + +#include "libarena/libarena.skel.h" + +static void run_libarena_test(struct libarena *skel, struct bpf_program *prog, + const char *name) +{ + int ret; + + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); + if (!ASSERT_OK(ret, "arena_buddy_reset")) + return; + } + + ret = libarena_run_prog(bpf_program__fd(prog)); + + ASSERT_OK(ret, name); +} + +static void *run_libarena_parallel_prog(void *arg) +{ + struct bpf_program *prog = arg; + + return (void *)(long)libarena_run_prog(bpf_program__fd(prog)); +} + +/* Max suffix is ceil((lg 2^32) / (lg 10)) + sizeof("__") = 10 + 2 = 12. */ +#define MAX_PARTEST_SUFFIX (12) +#define MAX_PARTEST_NAME (1024) +#define MAX_PARTEST_PREFIX (MAX_PARTEST_NAME - MAX_PARTEST_SUFFIX) + +static int run_libarena_parallel_fini(struct libarena *skel, const char *name, + size_t prefixlen) +{ + char tdname[MAX_PARTEST_NAME]; + struct bpf_program *fini_prog; + int ret; + + ret = snprintf(tdname, sizeof(tdname), "%.*s__fini", (int)prefixlen, name); + if (!ASSERT_LT(ret, sizeof(tdname), "partest fini name")) + return -ENAMETOOLONG; + + fini_prog = bpf_object__find_program_by_name(skel->obj, tdname); + if (!ASSERT_TRUE(fini_prog, "partest fini prog")) + return -ENOENT; + + ret = libarena_run_prog(bpf_program__fd(fini_prog)); + ASSERT_OK(ret, tdname); + + return ret; +} + +static int run_libarena_parallel_test_workers(struct libarena *skel, + const char *name, size_t prefixlen) +{ + pthread_t *threads = NULL, *tmp_threads; + char tdname[MAX_PARTEST_NAME]; + struct bpf_program *tdprog; + uint32_t nthreads; + void *thread_ret; + int ret, err = 0; + int worker_err; + int i; + + for (nthreads = 0; nthreads < UINT_MAX; nthreads++) { + ret = snprintf(tdname, sizeof(tdname), "%.*s__%u", (int)prefixlen, + name, nthreads); + if (!ASSERT_LT(ret, sizeof(tdname), "test worker name")) { + err = -ENAMETOOLONG; + break; + } + + /* + * We enumerate the worker threads for a given test with __0, __1, + * and so on. The suffixes always start from 0 and are contiguous, + * so if we don't find a program with the requested name we have + * discovered all available worker programs. + */ + tdprog = bpf_object__find_program_by_name(skel->obj, tdname); + if (!tdprog) + break; + + /* Bump the alloc array to accommodate the new thread. */ + tmp_threads = realloc(threads, (nthreads + 1) * sizeof(*threads)); + if (!ASSERT_TRUE(tmp_threads, "realloc")) { + err = -ENOMEM; + break; + } + threads = tmp_threads; + + ret = pthread_create(&threads[nthreads], NULL, + run_libarena_parallel_prog, + tdprog); + if (!ASSERT_OK(ret, "pthread_create")) { + err = ret; + break; + } + } + + + for (i = 0; i < nthreads; i++) { + ret = pthread_join(threads[i], &thread_ret); + if (!ASSERT_OK(ret, "pthread_join")) { + err = err ?: ret; + continue; + } + + worker_err = (long)thread_ret; + + /* + * A worker that bails out because another one already gave up + * reports -EINTR. It is collateral damage that carries no + * information, so skip it entirely: never let it become the + * reported error, and don't log it either. + */ + if (!worker_err || worker_err == -EINTR) + continue; + + if (!err) + err = worker_err; + + fprintf(stdout, "%.*s__%d returned %d\n", (int)prefixlen, name, + i, worker_err); + } + + free(threads); + + return err; +} + +static bool libarena_parallel_test_enabled(struct libarena *skel, + const char *prefix, + size_t prefixlen) +{ + struct bpf_program *prog; + char progname[MAX_PARTEST_NAME]; + int ret; + + ret = snprintf(progname, sizeof(progname), "%.*s__enabled", (int)prefixlen, + prefix); + if (!ASSERT_LT(ret, sizeof(progname), "partest enabled name")) + return false; + + prog = bpf_object__find_program_by_name(skel->obj, progname); + if (!prog) + return true; + + ret = libarena_run_prog(bpf_program__fd(prog)); + if (ret == -EOPNOTSUPP) + return false; + if (!ASSERT_OK(ret, progname)) + return false; + return true; +} + +static void run_libarena_parallel_test(struct libarena *skel, struct bpf_program *prog, + const char *name) +{ + char testname[MAX_PARTEST_NAME]; + size_t prefixlen; + const char *pos; + int ret; + + /* + * We annotate the initialization prog with __init. If the current prog does + * not match, it is one of the parallel threads instead and is ignored. + * + * We assume the test writer knows what they are doing and do not add __init + * randomly in the middle of a test name. + */ + pos = strstr(name, "__init"); + if (!pos) + return; + + prefixlen = pos - name; + if (!ASSERT_LT(prefixlen, MAX_PARTEST_PREFIX, "partest prefix too long")) + return; + + /* The name of the test without the __init suffix. Looks nicer in the test log. */ + ret = snprintf(testname, sizeof(testname), "%.*s", (int)prefixlen, name); + if (!ASSERT_LT(ret, sizeof(testname), "partest test name")) + return; + + if (!test__start_subtest(testname)) + return; + + if (!libarena_parallel_test_enabled(skel, testname, prefixlen)) { + test__skip(); + return; + } + + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); + if (!ASSERT_OK(ret, "arena_buddy_reset")) + return; + + ret = libarena_run_prog(bpf_program__fd(prog)); + if (!ASSERT_OK(ret, testname)) + return; + + ret = run_libarena_parallel_test_workers(skel, name, prefixlen); + + ASSERT_OK(ret, testname); + + run_libarena_parallel_fini(skel, name, prefixlen); +} + +void serial_test_libarena(void) +{ + struct arena_alloc_reserve_args args; + struct libarena *skel; + struct bpf_program *prog; + int ret; + + skel = libarena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + ret = libarena__attach(skel); + if (!ASSERT_OK(ret, "attach")) + goto out; + + args.nr_pages = ARENA_RESERVE_PAGES_DFL; + + ret = libarena_run_prog_args(bpf_program__fd(skel->progs.arena_alloc_reserve), + &args, sizeof(args)); + if (!ASSERT_OK(ret, "arena_alloc_reserve")) + goto out; + + bpf_object__for_each_program(prog, skel->obj) { + const char *name = bpf_program__name(prog); + + /* + * Handle parallel test progs separately. For those + * progs it's not a matter of test/skip, because each + * parallel test prog includes an initialization prog + * and a set of progs to be run in parallel. For the + * latter we do not record them as skipped or run, + * because we run them all at once when we come across + * the initialization prog. For more details on how we + * discover the progs see the comment on + * run_libarena_parallel_test. + */ + if (libarena_is_parallel_test_prog(name)) { + run_libarena_parallel_test(skel, prog, name); + continue; + } + + if (!libarena_is_test_prog(name)) + continue; + + if (!test__start_subtest(name)) + continue; + + run_libarena_test(skel, prog, name); + } + +out: + libarena__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c new file mode 100644 index 000000000000..9c31b17dbf39 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> + +#ifdef HAS_BPF_ARENA_ASAN +#include <unistd.h> + +#include <libarena/common.h> +#include <libarena/asan.h> +#include <libarena/buddy.h> +#include <libarena/userspace.h> + +#include "libarena/libarena_asan.skel.h" + +static void run_libarena_asan_test(struct libarena_asan *skel, + struct bpf_program *prog, const char *name) +{ + int ret; + + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); + if (!ASSERT_OK(ret, "arena_buddy_reset")) + return; + } + + ret = libarena_run_prog(bpf_program__fd(prog)); + ASSERT_OK(ret, name); + + verify_test_stderr(skel->obj, prog); +} + +static void run_test(void) +{ + struct arena_alloc_reserve_args args; + struct libarena_asan *skel; + struct bpf_program *prog; + int ret; + + skel = libarena_asan__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + ret = libarena_asan__attach(skel); + if (!ASSERT_OK(ret, "attach")) + goto out; + + args.nr_pages = ARENA_RESERVE_PAGES_DFL; + + ret = libarena_run_prog_args(bpf_program__fd(skel->progs.arena_alloc_reserve), + &args, sizeof(args)); + if (!ASSERT_OK(ret, "arena_alloc_reserve")) + goto out; + + ret = libarena_asan_init( + bpf_program__fd(skel->progs.arena_get_info), + bpf_program__fd(skel->progs.asan_init), + (1ULL << 32) / sysconf(_SC_PAGESIZE)); + if (!ASSERT_OK(ret, "libarena_asan_init")) + goto out; + + bpf_object__for_each_program(prog, skel->obj) { + const char *name = bpf_program__name(prog); + + if (!libarena_is_asan_test_prog(name)) + continue; + + if (!test__start_subtest(name)) + continue; + + run_libarena_asan_test(skel, prog, name); + } + +out: + libarena_asan__destroy(skel); +} + +#endif /* HAS_BPF_ARENA_ASAN */ + +/* + * Run the test depending on whether LLVM can compile arena ASAN + * programs. + */ +void serial_test_libarena_asan(void) +{ +#ifdef HAS_BPF_ARENA_ASAN + run_test(); +#else + test__skip(); +#endif + + return; +} diff --git a/tools/testing/selftests/bpf/prog_tests/linked_list.c b/tools/testing/selftests/bpf/prog_tests/linked_list.c index 14c5a7ef0e87..52fabbee3dd5 100644 --- a/tools/testing/selftests/bpf/prog_tests/linked_list.c +++ b/tools/testing/selftests/bpf/prog_tests/linked_list.c @@ -68,6 +68,7 @@ static struct { { "obj_type_id_oor", "local type ID argument must be in range [0, U32_MAX]" }, { "obj_new_no_composite", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" }, { "obj_new_no_struct", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" }, + { "obj_new_flex_array", "access beyond struct obj_new_flex" }, { "obj_drop_non_zero_off", "R1 must have zero offset when passed to release func" }, { "new_null_ret", "R0 invalid mem access 'ptr_or_null_'" }, { "obj_new_acq", "Unreleased reference id=" }, @@ -81,18 +82,18 @@ static struct { { "direct_write_node", "direct access to bpf_list_node is disallowed" }, { "use_after_unlock_push_front", "invalid mem access 'scalar'" }, { "use_after_unlock_push_back", "invalid mem access 'scalar'" }, - { "double_push_front", "arg#1 expected pointer to allocated object" }, - { "double_push_back", "arg#1 expected pointer to allocated object" }, + { "double_push_front", "R2 expected pointer to allocated object" }, + { "double_push_back", "R2 expected pointer to allocated object" }, { "no_node_value_type", "bpf_list_node not found at offset=0" }, { "incorrect_value_type", "operation on bpf_list_head expects arg#1 bpf_list_node at offset=48 in struct foo, " "but arg is at offset=0 in struct bar" }, - { "incorrect_node_var_off", "variable ptr_ access var_off=(0x0; 0xffffffff) disallowed" }, + { "incorrect_node_var_off", "variable ptr_ access var_off=(0x0; 0x1ffffffff) disallowed" }, { "incorrect_node_off1", "bpf_list_node not found at offset=49" }, { "incorrect_node_off2", "arg#1 offset=0, but expected bpf_list_node at offset=48 in struct foo" }, { "no_head_type", "bpf_list_head not found at offset=0" }, { "incorrect_head_var_off1", "R1 doesn't have constant offset" }, - { "incorrect_head_var_off2", "variable ptr_ access var_off=(0x0; 0xffffffff) disallowed" }, + { "incorrect_head_var_off2", "variable ptr_ access var_off=(0x0; 0x1ffffffff) disallowed" }, { "incorrect_head_off1", "bpf_list_head not found at offset=25" }, { "incorrect_head_off2", "bpf_list_head not found at offset=1" }, { "pop_front_off", "off 48 doesn't point to 'struct bpf_spin_lock' that is at 40" }, @@ -131,13 +132,14 @@ end: linked_list_fail__destroy(skel); } -static void clear_fields(struct bpf_map *map) +static void clear_fields(struct bpf_program *prog) { - char buf[24]; - int key = 0; + LIBBPF_OPTS(bpf_test_run_opts, opts); + int ret; - memset(buf, 0xff, sizeof(buf)); - ASSERT_OK(bpf_map__update_elem(map, &key, sizeof(key), buf, sizeof(buf), 0), "check_and_free_fields"); + ret = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + ASSERT_OK(ret, "clear_fields"); + ASSERT_OK(opts.retval, "clear_fields retval"); } enum { @@ -170,31 +172,31 @@ static void test_linked_list_success(int mode, bool leave_in_map) ASSERT_OK(ret, "map_list_push_pop"); ASSERT_OK(opts.retval, "map_list_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.array_map); + clear_fields(skel->progs.clear_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.inner_map_list_push_pop), &opts); ASSERT_OK(ret, "inner_map_list_push_pop"); ASSERT_OK(opts.retval, "inner_map_list_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.inner_map); + clear_fields(skel->progs.clear_inner_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_push_pop), &opts); ASSERT_OK(ret, "global_list_push_pop"); ASSERT_OK(opts.retval, "global_list_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_push_pop_nested), &opts); ASSERT_OK(ret, "global_list_push_pop_nested"); ASSERT_OK(opts.retval, "global_list_push_pop_nested retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_nested_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_array_push_pop), &opts); ASSERT_OK(ret, "global_list_array_push_pop"); ASSERT_OK(opts.retval, "global_list_array_push_pop retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_array_list); if (mode == PUSH_POP) goto end; @@ -204,19 +206,19 @@ ppm: ASSERT_OK(ret, "map_list_push_pop_multiple"); ASSERT_OK(opts.retval, "map_list_push_pop_multiple retval"); if (!leave_in_map) - clear_fields(skel->maps.array_map); + clear_fields(skel->progs.clear_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.inner_map_list_push_pop_multiple), &opts); ASSERT_OK(ret, "inner_map_list_push_pop_multiple"); ASSERT_OK(opts.retval, "inner_map_list_push_pop_multiple retval"); if (!leave_in_map) - clear_fields(skel->maps.inner_map); + clear_fields(skel->progs.clear_inner_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_push_pop_multiple), &opts); ASSERT_OK(ret, "global_list_push_pop_multiple"); ASSERT_OK(opts.retval, "global_list_push_pop_multiple retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_list); if (mode == PUSH_POP_MULT) goto end; @@ -226,19 +228,19 @@ lil: ASSERT_OK(ret, "map_list_in_list"); ASSERT_OK(opts.retval, "map_list_in_list retval"); if (!leave_in_map) - clear_fields(skel->maps.array_map); + clear_fields(skel->progs.clear_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.inner_map_list_in_list), &opts); ASSERT_OK(ret, "inner_map_list_in_list"); ASSERT_OK(opts.retval, "inner_map_list_in_list retval"); if (!leave_in_map) - clear_fields(skel->maps.inner_map); + clear_fields(skel->progs.clear_inner_map_list); ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.global_list_in_list), &opts); ASSERT_OK(ret, "global_list_in_list"); ASSERT_OK(opts.retval, "global_list_in_list retval"); if (!leave_in_map) - clear_fields(skel->maps.bss_A); + clear_fields(skel->progs.clear_global_list); end: linked_list__destroy(skel); } @@ -712,7 +714,7 @@ static void test_btf(void) break; err = btf__load_into_kernel(btf); - ASSERT_EQ(err, -ELOOP, "check btf"); + ASSERT_EQ(err, 0, "check btf"); btf__free(btf); break; } @@ -771,7 +773,7 @@ static void test_btf(void) break; err = btf__load_into_kernel(btf); - ASSERT_EQ(err, -ELOOP, "check btf"); + ASSERT_EQ(err, 0, "check btf"); btf__free(btf); break; } diff --git a/tools/testing/selftests/bpf/prog_tests/livepatch_trampoline.c b/tools/testing/selftests/bpf/prog_tests/livepatch_trampoline.c index 72aa5376c30e..0a12af924a99 100644 --- a/tools/testing/selftests/bpf/prog_tests/livepatch_trampoline.c +++ b/tools/testing/selftests/bpf/prog_tests/livepatch_trampoline.c @@ -5,6 +5,8 @@ #include "testing_helpers.h" #include "livepatch_trampoline.skel.h" +#define LIVEPATCH_ENABLED_PATH "/sys/kernel/livepatch/livepatch_sample/enabled" + static int load_livepatch(void) { char path[4096]; @@ -19,7 +21,8 @@ static int load_livepatch(void) static void unload_livepatch(void) { /* Disable the livepatch before unloading the module */ - system("echo 0 > /sys/kernel/livepatch/livepatch_sample/enabled"); + if (!access(LIVEPATCH_ENABLED_PATH, F_OK)) + system("echo 0 > " LIVEPATCH_ENABLED_PATH); unload_module("livepatch_sample", env_verbosity > VERBOSE_NONE); } @@ -81,9 +84,22 @@ out: void test_livepatch_trampoline(void) { int retry_cnt = 0; + int err; + + /* Skip if kernel was built without CONFIG_LIVEPATCH */ + if (access("/sys/kernel/livepatch", F_OK)) { + test__skip(); + return; + } retry: - if (load_livepatch()) { + err = load_livepatch(); + if (err) { + if (err == -ENOENT) { + test__skip(); + return; + } + if (retry_cnt) { ASSERT_OK(1, "load_livepatch"); goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/local_kptr_ownership.c b/tools/testing/selftests/bpf/prog_tests/local_kptr_ownership.c new file mode 100644 index 000000000000..a487aa68f2ee --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/local_kptr_ownership.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <bpf/btf.h> +#include <linux/btf.h> +#include <test_progs.h> + +#define SPIN_LOCK 2 +#define LIST_HEAD 3 +#define LIST_NODE 4 +/* Keep in sync with BTF_MAX_OWNERSHIP_DEPTH. */ +#define MAX_OWNERSHIP_DEPTH 8 + +static struct btf *init_btf(void) +{ + struct btf *btf; + int id; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return NULL; + id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_EQ(id, 1, "btf__add_int")) + goto err_out; + id = btf__add_struct(btf, "bpf_spin_lock", 4); + if (!ASSERT_EQ(id, SPIN_LOCK, "btf__add_struct bpf_spin_lock")) + goto err_out; + id = btf__add_struct(btf, "bpf_list_head", 16); + if (!ASSERT_EQ(id, LIST_HEAD, "btf__add_struct bpf_list_head")) + goto err_out; + id = btf__add_struct(btf, "bpf_list_node", 24); + if (!ASSERT_EQ(id, LIST_NODE, "btf__add_struct bpf_list_node")) + goto err_out; + return btf; + +err_out: + btf__free(btf); + return NULL; +} + +static int add_local_kptr(struct btf *btf, int pointee_id, const char *tag) +{ + int id; + + id = btf__add_type_tag(btf, tag, pointee_id); + if (!ASSERT_GT(id, 0, "btf__add_type_tag")) + return id; + id = btf__add_ptr(btf, id); + ASSERT_GT(id, 0, "btf__add_ptr"); + return id; +} + +static void test_self_cycle(const char *tag, int expected_err) +{ + struct btf *btf; + int id, err; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + id = add_local_kptr(btf, 7, tag); + if (id <= 0) + goto out; + id = btf__add_struct(btf, "self_cycle", 8); + if (!ASSERT_EQ(id, 7, "btf__add_struct self_cycle")) + goto out; + err = btf__add_field(btf, "next", 6, 0, 0); + if (!ASSERT_OK(err, "btf__add_field self_cycle::next")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, expected_err, "check btf"); +out: + btf__free(btf); +} + +static void test_aba_cycle(void) +{ + struct btf *btf; + int id, err; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + id = add_local_kptr(btf, 10, "kptr"); + if (id <= 0) + goto out; + id = add_local_kptr(btf, 9, "kptr"); + if (id <= 0) + goto out; + id = btf__add_struct(btf, "cycle_a", 8); + if (!ASSERT_EQ(id, 9, "btf__add_struct cycle_a")) + goto out; + err = btf__add_field(btf, "b", 6, 0, 0); + if (!ASSERT_OK(err, "btf__add_field cycle_a::b")) + goto out; + id = btf__add_struct(btf, "cycle_b", 8); + if (!ASSERT_EQ(id, 10, "btf__add_struct cycle_b")) + goto out; + err = btf__add_field(btf, "a", 8, 0, 0); + if (!ASSERT_OK(err, "btf__add_field cycle_b::a")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, -ELOOP, "check btf"); +out: + btf__free(btf); +} + +static void test_mixed_cycle(void) +{ + struct btf *btf; + int id, err; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + id = add_local_kptr(btf, 7, "kptr"); + if (id <= 0) + goto out; + id = btf__add_struct(btf, "mixed_owner", 20); + if (!ASSERT_EQ(id, 7, "btf__add_struct mixed_owner")) + goto out; + err = btf__add_field(btf, "root", LIST_HEAD, 0, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_owner::root")) + goto out; + err = btf__add_field(btf, "lock", SPIN_LOCK, 128, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_owner::lock")) + goto out; + id = btf__add_decl_tag(btf, "contains:mixed_node:node", 7, 0); + if (!ASSERT_EQ(id, 8, "btf__add_decl_tag mixed_owner")) + goto out; + id = btf__add_struct(btf, "mixed_node", 32); + if (!ASSERT_EQ(id, 9, "btf__add_struct mixed_node")) + goto out; + err = btf__add_field(btf, "node", LIST_NODE, 0, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_node::node")) + goto out; + err = btf__add_field(btf, "owner", 6, 192, 0); + if (!ASSERT_OK(err, "btf__add_field mixed_node::owner")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, -ELOOP, "check btf"); +out: + btf__free(btf); +} + +static void test_acyclic_depth(int depth, bool child_first, bool shared_suffix, int expected_err) +{ + int ptr_id[MAX_OWNERSHIP_DEPTH + 1]; + int first_struct_id; + struct btf *btf; + int id, err, i, n, pointee_id; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + first_struct_id = 5 + 2 * depth; + for (i = 0; i < depth; i++) { + if (i == depth - 1) + pointee_id = first_struct_id + depth; + else + pointee_id = first_struct_id + (child_first ? depth - 2 - i : i + 1); + ptr_id[i] = add_local_kptr(btf, pointee_id, "kptr"); + if (ptr_id[i] <= 0) + goto out; + } + for (n = 0; n < depth; n++) { + char name[32]; + int offset = 0; + + i = child_first ? depth - 1 - n : n; + snprintf(name, sizeof(name), "owner_%d", i); + id = btf__add_struct(btf, name, shared_suffix && !i ? 16 : 8); + if (!ASSERT_EQ(id, first_struct_id + n, "btf__add_struct owner")) + goto out; + if (shared_suffix && !i) { + /* + * Visit the shared suffix through the shorter path before + * reaching it again with less remaining depth. + */ + err = btf__add_field(btf, "suffix", ptr_id[1], 0, 0); + if (!ASSERT_OK(err, "btf__add_field owner::suffix")) + goto out; + offset = 64; + } + err = btf__add_field(btf, "next", ptr_id[i], offset, 0); + if (!ASSERT_OK(err, "btf__add_field owner::next")) + goto out; + } + id = btf__add_struct(btf, "plain_leaf", 4); + if (!ASSERT_EQ(id, first_struct_id + depth, "btf__add_struct plain_leaf")) + goto out; + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, expected_err, "check btf"); +out: + btf__free(btf); +} + +static void test_graph_depth(bool rbtree, int depth, int expected_err) +{ + int root_type = LIST_HEAD, node_type = LIST_NODE, node_size = 24; + int id, err, i, lock_off, root_off, size; + struct btf *btf; + + btf = init_btf(); + if (!ASSERT_OK_PTR(btf, "init_btf")) + return; + if (rbtree) { + root_type = btf__add_struct(btf, "bpf_rb_root", 16); + if (!ASSERT_GT(root_type, 0, "btf__add_struct bpf_rb_root")) + goto out; + node_type = btf__add_struct(btf, "bpf_rb_node", 32); + if (!ASSERT_GT(node_type, 0, "btf__add_struct bpf_rb_node")) + goto out; + node_size = 32; + } + + for (i = 0; i < depth; i++) { + char name[32], tag[64]; + + lock_off = i ? node_size : 0; + root_off = lock_off + 8; + size = i == depth - 1 ? node_size : root_off + 16; + snprintf(name, sizeof(name), "graph_owner_%d", i); + id = btf__add_struct(btf, name, size); + if (!ASSERT_GT(id, 0, "btf__add_struct graph_owner")) + goto out; + if (i) { + err = btf__add_field(btf, "node", node_type, 0, 0); + if (!ASSERT_OK(err, "btf__add_field graph_owner::node")) + goto out; + } + if (i == depth - 1) + continue; + err = btf__add_field(btf, "lock", SPIN_LOCK, lock_off * 8, 0); + if (!ASSERT_OK(err, "btf__add_field graph_owner::lock")) + goto out; + err = btf__add_field(btf, "root", root_type, root_off * 8, 0); + if (!ASSERT_OK(err, "btf__add_field graph_owner::root")) + goto out; + snprintf(tag, sizeof(tag), "contains:graph_owner_%d:node", i + 1); + err = btf__add_decl_tag(btf, tag, id, i ? 2 : 1); + if (!ASSERT_GT(err, 0, "btf__add_decl_tag graph_owner")) + goto out; + } + + err = btf__load_into_kernel(btf); + ASSERT_EQ(err, expected_err, "check btf"); +out: + btf__free(btf); +} + +void test_local_kptr_ownership(void) +{ + if (test__start_subtest("self_cycle")) + test_self_cycle("kptr", -ELOOP); + if (test__start_subtest("untrusted_self_cycle")) + test_self_cycle("kptr_untrusted", 0); + if (test__start_subtest("percpu_self_cycle")) + test_self_cycle("percpu_kptr", -ELOOP); + if (test__start_subtest("ABA_cycle")) + test_aba_cycle(); + if (test__start_subtest("mixed_graph_root_cycle")) + test_mixed_cycle(); + if (test__start_subtest("max_acyclic")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH, false, false, 0); + if (test__start_subtest("too_deep_acyclic")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH + 1, false, false, -ELOOP); + if (test__start_subtest("max_acyclic_child_first")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH, true, false, 0); + if (test__start_subtest("too_deep_acyclic_child_first")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH + 1, true, false, -ELOOP); + if (test__start_subtest("max_acyclic_shared_suffix")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH, false, true, 0); + if (test__start_subtest("too_deep_acyclic_shared_suffix")) + test_acyclic_depth(MAX_OWNERSHIP_DEPTH + 1, false, true, -ELOOP); + if (test__start_subtest("list_three_types")) + test_graph_depth(false, 3, 0); + if (test__start_subtest("list_four_types")) + test_graph_depth(false, 4, 0); + if (test__start_subtest("list_max_depth")) + test_graph_depth(false, MAX_OWNERSHIP_DEPTH, 0); + if (test__start_subtest("list_too_deep")) + test_graph_depth(false, MAX_OWNERSHIP_DEPTH + 1, -ELOOP); + if (test__start_subtest("rbtree_three_types")) + test_graph_depth(true, 3, 0); + if (test__start_subtest("rbtree_four_types")) + test_graph_depth(true, 4, 0); + if (test__start_subtest("rbtree_max_depth")) + test_graph_depth(true, MAX_OWNERSHIP_DEPTH, 0); + if (test__start_subtest("rbtree_too_deep")) + test_graph_depth(true, MAX_OWNERSHIP_DEPTH + 1, -ELOOP); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lru_lock_nmi.c b/tools/testing/selftests/bpf/prog_tests/lru_lock_nmi.c new file mode 100644 index 000000000000..60666a9ba41f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lru_lock_nmi.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Stress every LRU lock-failure and orphan-recovery. + * perf_event NMI BPF on every online CPU does + * update+delete on a small LRU map; userspace threads on every CPU do + * the same from syscall context. + */ +#define _GNU_SOURCE +#include <pthread.h> +#include <sched.h> +#include <sys/syscall.h> +#include <linux/perf_event.h> +#include <test_progs.h> +#include "testing_helpers.h" +#include "lru_lock_nmi.skel.h" + +#define MAP_ENTRIES 64 +#define KEY_RANGE (MAP_ENTRIES * 2) +#define STRESS_NS (500 * 1000 * 1000ULL) + +struct hammer_arg { + int map_fd; + int cpu; + __u64 deadline_ns; +}; + +struct refill_arg { + int map_fd; + int cpu; + int per_cpu_quota; + int update_errors; +}; + +/* + * Pin the calling thread to @cpu. Uses dynamically-allocated CPU sets so + * we stay correct on hosts with @cpu >= CPU_SETSIZE (default 1024). + */ +static int pin_to_cpu(int cpu) +{ + cpu_set_t *cs; + size_t cs_size; + int err; + + cs = CPU_ALLOC(cpu + 1); + if (!cs) + return -ENOMEM; + cs_size = CPU_ALLOC_SIZE(cpu + 1); + + CPU_ZERO_S(cs_size, cs); + CPU_SET_S(cpu, cs_size, cs); + err = pthread_setaffinity_np(pthread_self(), cs_size, cs); + CPU_FREE(cs); + return err; +} + +static void *hammer_thread(void *p) +{ + struct hammer_arg *a = p; + int nr_possible_cpus = libbpf_num_possible_cpus(); + __u64 val[nr_possible_cpus]; + unsigned int seed; + __u32 key; + + memset(val, 0, sizeof(val)); + pin_to_cpu(a->cpu); + + seed = (unsigned int)a->cpu ^ (unsigned int)(uintptr_t)pthread_self(); + + while (get_time_ns() < a->deadline_ns) { + bool do_update = rand_r(&seed) & 1; + + key = rand_r(&seed) % KEY_RANGE; + if (do_update) + bpf_map_update_elem(a->map_fd, &key, val, BPF_ANY); + else + bpf_map_delete_elem(a->map_fd, &key); + } + return NULL; +} + +static void *refill_thread(void *p) +{ + struct refill_arg *a = p; + int nr_possible_cpus = libbpf_num_possible_cpus(); + __u64 val[nr_possible_cpus]; + __u32 start, end, key; + + memset(val, 0, sizeof(val)); + pin_to_cpu(a->cpu); + + start = (__u32)a->cpu * (__u32)a->per_cpu_quota; + end = start + (__u32)a->per_cpu_quota; + for (key = start; key < end; key++) + if (bpf_map_update_elem(a->map_fd, &key, val, BPF_ANY)) + a->update_errors++; + return NULL; +} + +/* + * Drain the map, then refill it with each CPU inserting only its own + * quota of keys. + * After refill, lookup every key we inserted - a stranded node on any + * CPU's pool would have forced eviction. + */ +static int drain_then_verify_capacity(int map_fd, int nr_cpus) +{ + int per_cpu_quota = MAP_ENTRIES / nr_cpus; + int total = per_cpu_quota * nr_cpus; + int nr_possible_cpus = libbpf_num_possible_cpus(); + pthread_t threads[nr_cpus]; + struct refill_arg args[nr_cpus]; + __u64 val[nr_possible_cpus]; + int i, hits = 0, nthreads = 0; + __u32 key; + + memset(val, 0, sizeof(val)); + + for (key = 0; key < KEY_RANGE; key++) + bpf_map_delete_elem(map_fd, &key); + + for (i = 0; i < nr_cpus; i++) { + args[i] = (struct refill_arg){ + .map_fd = map_fd, + .cpu = i, + .per_cpu_quota = per_cpu_quota, + }; + if (pthread_create(&threads[nthreads], NULL, refill_thread, &args[i]) == 0) + nthreads++; + } + for (i = 0; i < nthreads; i++) + pthread_join(threads[i], NULL); + + for (i = 0; i < nr_cpus; i++) + if (args[i].update_errors) + return -ENOMEM; + + for (key = 0; key < (__u32)total; key++) + if (bpf_map_lookup_elem(map_fd, &key, val) == 0) + hits++; + + return hits == total ? 0 : -EIO; +} + +static void run_variant(enum bpf_map_type type, __u32 map_flags, const char *name) +{ + struct perf_event_attr attr = { + .size = sizeof(attr), + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + .freq = 1, + }; + int nr_cpus, max_cpus = 64; + struct bpf_link *links[max_cpus]; + pthread_t threads[max_cpus]; + struct hammer_arg args[max_cpus]; + struct lru_lock_nmi *skel = NULL; + int map_fd, i, err, nr_threads = 0, pmu_fd = -1; + __u64 deadline; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "num_cpus")) + return; + + if (nr_cpus > max_cpus) + nr_cpus = max_cpus; + + if (!test__start_subtest(name)) + return; + + memset(links, 0, sizeof(links)); + skel = lru_lock_nmi__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + err = bpf_map__set_type(skel->maps.lru_map, type); + if (!ASSERT_OK(err, "set_type")) + goto cleanup; + err = bpf_map__set_map_flags(skel->maps.lru_map, map_flags); + if (!ASSERT_OK(err, "set_flags")) + goto cleanup; + err = bpf_map__set_max_entries(skel->maps.lru_map, MAP_ENTRIES); + if (!ASSERT_OK(err, "set_max_entries")) + goto cleanup; + + err = lru_lock_nmi__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + skel->bss->hits = 0; + map_fd = bpf_map__fd(skel->maps.lru_map); + attr.sample_freq = read_perf_max_sample_freq(); + + for (i = 0; i < nr_cpus; i++) { + pmu_fd = syscall(__NR_perf_event_open, &attr, -1, i, -1, 0); + if (pmu_fd < 0) { + if (i == 0 && + (errno == ENOENT || errno == EOPNOTSUPP)) { + test__skip(); + goto cleanup; + } + continue; + } + /* libbpf takes ownership of pfd on success */ + links[i] = bpf_program__attach_perf_event(skel->progs.oncpu, pmu_fd); + if (!links[i]) + close(pmu_fd); + } + + deadline = get_time_ns() + STRESS_NS; + for (i = 0; i < nr_cpus; i++) { + args[i].map_fd = map_fd; + args[i].cpu = i; + args[i].deadline_ns = deadline; + if (pthread_create(&threads[nr_threads], NULL, hammer_thread, &args[i]) == 0) + nr_threads++; + } + for (i = 0; i < nr_threads; i++) + pthread_join(threads[i], NULL); + + for (i = 0; i < nr_cpus; i++) { + if (links[i]) { + bpf_link__destroy(links[i]); + links[i] = NULL; + } + } + + ASSERT_GT(skel->bss->hits, 0, "nmi_bpf_ran"); + ASSERT_OK(drain_then_verify_capacity(map_fd, nr_cpus), "drain_then_verify_capacity"); + +cleanup: + for (i = 0; i < nr_cpus; i++) { + if (links[i]) + bpf_link__destroy(links[i]); + } + lru_lock_nmi__destroy(skel); +} + +void serial_test_lru_lock_nmi(void) +{ + run_variant(BPF_MAP_TYPE_LRU_HASH, 0, "common_lru"); + run_variant(BPF_MAP_TYPE_LRU_HASH, BPF_F_NO_COMMON_LRU, "no_common_lru"); + run_variant(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0, "percpu_lru"); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c new file mode 100644 index 000000000000..28bc4b117f41 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Christian Brauner <brauner@kernel.org> */ + +/* + * Test BPF LSM block device integrity hooks with dm-verity. + * + * Creates a dm-verity device over loopback, which triggers + * security_bdev_setintegrity() during verity_preresume(). + * Verifies that the BPF program correctly tracks the integrity + * metadata in its hashmap. + */ + +#define _GNU_SOURCE +#include <test_progs.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/sysmacros.h> +#include <sys/types.h> +#include <unistd.h> +#include "lsm_bdev.skel.h" + +/* Must match the definition in progs/lsm_bdev.c. */ +struct verity_info { + __u8 has_roothash; + __u8 sig_valid; + __u32 setintegrity_cnt; +}; + +#define DATA_SIZE_MB 8 +#define HASH_SIZE_MB 1 +#define DM_NAME "bpf_test_verity" +#define DM_DEV_PATH "/dev/mapper/" DM_NAME + +/* Run a command and optionally capture the first line of stdout. */ +static int run_cmd(const char *cmd, char *out, size_t out_sz) +{ + FILE *fp; + int ret; + + fp = popen(cmd, "r"); + if (!fp) + return -1; + + if (out && out_sz > 0) { + if (!fgets(out, out_sz, fp)) + out[0] = '\0'; + /* strip trailing newline */ + out[strcspn(out, "\n")] = '\0'; + } + + ret = pclose(fp); + return WIFEXITED(ret) ? WEXITSTATUS(ret) : -1; +} + +static bool has_prerequisites(void) +{ + if (getuid() != 0) { + printf("SKIP: must be root\n"); + return false; + } + + if (run_cmd("modprobe loop 2>/dev/null", NULL, 0) && + run_cmd("ls /dev/loop-control 2>/dev/null", NULL, 0)) { + printf("SKIP: no loop device support\n"); + return false; + } + + if (run_cmd("modprobe dm-verity 2>/dev/null", NULL, 0) && + run_cmd("dmsetup targets 2>/dev/null | grep -q verity", NULL, 0)) { + printf("SKIP: dm-verity module not available\n"); + return false; + } + + if (run_cmd("which veritysetup >/dev/null 2>&1", NULL, 0)) { + printf("SKIP: veritysetup not found\n"); + return false; + } + + return true; +} + +void test_lsm_bdev(void) +{ + char data_img[] = "/tmp/bpf_verity_data_XXXXXX"; + char hash_img[] = "/tmp/bpf_verity_hash_XXXXXX"; + char data_loop[64] = {}; + char hash_loop[64] = {}; + char roothash[256] = {}; + char cmd[512]; + int data_fd = -1, hash_fd = -1; + struct lsm_bdev *skel = NULL; + struct verity_info val; + struct stat st; + __u32 dev_key; + int err; + + if (!has_prerequisites()) { + test__skip(); + return; + } + + /* Clean up any stale device from a previous crashed run. */ + snprintf(cmd, sizeof(cmd), "dmsetup remove %s 2>/dev/null", DM_NAME); + run_cmd(cmd, NULL, 0); + + /* Create temporary image files. */ + data_fd = mkstemp(data_img); + if (!ASSERT_OK_FD(data_fd, "mkstemp data")) + return; + + hash_fd = mkstemp(hash_img); + if (!ASSERT_OK_FD(hash_fd, "mkstemp hash")) + goto cleanup; + + if (!ASSERT_OK(ftruncate(data_fd, DATA_SIZE_MB * 1024 * 1024), + "truncate data")) + goto cleanup; + + if (!ASSERT_OK(ftruncate(hash_fd, HASH_SIZE_MB * 1024 * 1024), + "truncate hash")) + goto cleanup; + + close(data_fd); + data_fd = -1; + close(hash_fd); + hash_fd = -1; + + /* Set up loop devices. */ + snprintf(cmd, sizeof(cmd), + "losetup --find --show %s 2>/dev/null", data_img); + if (!ASSERT_OK(run_cmd(cmd, data_loop, sizeof(data_loop)), + "losetup data")) + goto teardown; + + snprintf(cmd, sizeof(cmd), + "losetup --find --show %s 2>/dev/null", hash_img); + if (!ASSERT_OK(run_cmd(cmd, hash_loop, sizeof(hash_loop)), + "losetup hash")) + goto teardown; + + /* Format the dm-verity device and capture the root hash. */ + snprintf(cmd, sizeof(cmd), + "veritysetup format %s %s 2>/dev/null | " + "grep -i 'root hash' | awk '{print $NF}'", + data_loop, hash_loop); + if (!ASSERT_OK(run_cmd(cmd, roothash, sizeof(roothash)), + "veritysetup format")) + goto teardown; + + if (!ASSERT_GT((int)strlen(roothash), 0, "roothash not empty")) + goto teardown; + + /* Load and attach BPF program before activating dm-verity. */ + skel = lsm_bdev__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + goto teardown; + + err = lsm_bdev__attach(skel); + if (!ASSERT_OK(err, "skel attach")) + goto teardown; + + /* Activate dm-verity — triggers verity_preresume() hooks. */ + snprintf(cmd, sizeof(cmd), + "veritysetup open %s %s %s %s 2>/dev/null", + data_loop, DM_NAME, hash_loop, roothash); + if (!ASSERT_OK(run_cmd(cmd, NULL, 0), "veritysetup open")) + goto teardown; + + /* Get the dm device's dev_t. */ + if (!ASSERT_OK(stat(DM_DEV_PATH, &st), "stat dm dev")) + goto remove_dm; + + dev_key = (major(st.st_rdev) << 20) | minor(st.st_rdev); + + /* Look up the device in the BPF map and verify. */ + err = bpf_map__lookup_elem(skel->maps.verity_devices, + &dev_key, sizeof(dev_key), + &val, sizeof(val), 0); + if (!ASSERT_OK(err, "map lookup")) + goto remove_dm; + + ASSERT_EQ(val.has_roothash, 1, "has_roothash"); + ASSERT_EQ(val.sig_valid, 0, "sig_valid (unsigned)"); + /* + * verity_preresume() always calls security_bdev_setintegrity() + * for the roothash. The signature-validity call only happens + * when CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG is enabled. + */ + ASSERT_GE(val.setintegrity_cnt, 1, "setintegrity_cnt min"); + ASSERT_LE(val.setintegrity_cnt, 2, "setintegrity_cnt max"); + + /* Verify that the alloc hook fired at least once. */ + ASSERT_GT(skel->bss->alloc_count, 0, "alloc_count"); + +remove_dm: + snprintf(cmd, sizeof(cmd), "dmsetup remove %s 2>/dev/null", DM_NAME); + run_cmd(cmd, NULL, 0); + +teardown: + if (data_loop[0]) { + snprintf(cmd, sizeof(cmd), "losetup -d %s 2>/dev/null", + data_loop); + run_cmd(cmd, NULL, 0); + } + if (hash_loop[0]) { + snprintf(cmd, sizeof(cmd), "losetup -d %s 2>/dev/null", + hash_loop); + run_cmd(cmd, NULL, 0); + } + +cleanup: + lsm_bdev__destroy(skel); + if (data_fd >= 0) + close(data_fd); + if (hash_fd >= 0) + close(hash_fd); + unlink(data_img); + unlink(hash_img); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c b/tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c index 6df25de8f080..41e867467f6c 100644 --- a/tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c +++ b/tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c @@ -2,6 +2,7 @@ #include <sys/types.h> #include <sys/socket.h> +#include <sys/xattr.h> #include <test_progs.h> #include <bpf/btf.h> @@ -309,11 +310,89 @@ static void test_lsm_cgroup_nonvoid(void) lsm_cgroup_nonvoid__destroy(skel); } +static void test_lsm_cgroup_retval(void) +{ + struct lsm_cgroup *skel = NULL; + int skipcap_prog_fd1, skipcap_prog_fd2, socket_prog_fd1, socket_prog_fd2; + int cgroup_fd = -1; + int err, fd; + char tmpfile[] = "/tmp/test_lsm_cgroup_retval.XXXXXX"; + + fd = mkstemp(tmpfile); + if (!ASSERT_OK_FD(fd, "mkstemp")) + return; + close(fd); + + cgroup_fd = test__join_cgroup("/default_retval"); + if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup")) + goto cleanup_tmpfile; + + skel = lsm_cgroup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto cleanup_cgroup; + + skipcap_prog_fd1 = bpf_program__fd(skel->progs.skipcap_first); + skipcap_prog_fd2 = bpf_program__fd(skel->progs.skipcap_second); + socket_prog_fd1 = bpf_program__fd(skel->progs.socket_first); + socket_prog_fd2 = bpf_program__fd(skel->progs.socket_second); + + err = bpf_prog_attach(skipcap_prog_fd1, cgroup_fd, BPF_LSM_CGROUP, BPF_F_ALLOW_MULTI); + if (err == -ENOTSUPP) { + test__skip(); + goto cleanup_skeleton; + } + if (!ASSERT_OK(err, "attach first skipcap prog")) + goto cleanup_skeleton; + + err = bpf_prog_attach(skipcap_prog_fd2, cgroup_fd, BPF_LSM_CGROUP, BPF_F_ALLOW_MULTI); + if (!ASSERT_OK(err, "attach second skipcap prog")) + goto cleanup_skipcap1; + + err = bpf_prog_attach(socket_prog_fd1, cgroup_fd, BPF_LSM_CGROUP, BPF_F_ALLOW_MULTI); + if (!ASSERT_OK(err, "attach first sock_create prog")) + goto cleanup_skipcap2; + + err = bpf_prog_attach(socket_prog_fd2, cgroup_fd, BPF_LSM_CGROUP, BPF_F_ALLOW_MULTI); + if (!ASSERT_OK(err, "attach second sock_create prog")) + goto cleanup_sock_create1; + + /* trigger the bool hook by setxattr */ + err = setxattr(tmpfile, "user.test", "value", 5, 0); + if (!ASSERT_OK(err, "setxattr")) + goto cleanup_sock_create2; + + /* trigger the errno hook by creating a socket */ + fd = socket(AF_INET, SOCK_STREAM, 0); + if (!ASSERT_OK_FD(fd, "socket")) + goto cleanup_sock_create2; + close(fd); + + ASSERT_EQ(skel->data->skipcap_retval, 0, "bool_hook_retval_should_be_0"); + ASSERT_EQ(skel->data->socket_retval, -EPERM, "errno_hook_retval_should_be_EPERM"); + +cleanup_sock_create2: + bpf_prog_detach2(socket_prog_fd2, cgroup_fd, BPF_LSM_CGROUP); +cleanup_sock_create1: + bpf_prog_detach2(socket_prog_fd1, cgroup_fd, BPF_LSM_CGROUP); +cleanup_skipcap2: + bpf_prog_detach2(skipcap_prog_fd2, cgroup_fd, BPF_LSM_CGROUP); +cleanup_skipcap1: + bpf_prog_detach2(skipcap_prog_fd1, cgroup_fd, BPF_LSM_CGROUP); +cleanup_skeleton: + lsm_cgroup__destroy(skel); +cleanup_cgroup: + close(cgroup_fd); +cleanup_tmpfile: + unlink(tmpfile); +} + void test_lsm_cgroup(void) { if (test__start_subtest("functional")) test_lsm_cgroup_functional(); if (test__start_subtest("nonvoid")) test_lsm_cgroup_nonvoid(); + if (test__start_subtest("retval")) + test_lsm_cgroup_retval(); btf__free(btf); } diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c index b6391af5f6f9..39e8a3b8b6af 100644 --- a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c @@ -3,6 +3,7 @@ #include "network_helpers.h" #include "test_progs.h" +#include "test_lwt_ip_encap.skel.h" #define BPF_FILE "test_lwt_ip_encap.bpf.o" @@ -32,6 +33,9 @@ #define IP6_ADDR_8 "fb08::1" #define IP6_ADDR_GRE "fb10::1" +#define IP4_ADDR_VXLAN "172.16.17.100" +#define IP6_ADDR_VXLAN "fb11::1" + #define IP6_ADDR_SRC IP6_ADDR_1 #define IP6_ADDR_DST IP6_ADDR_4 @@ -406,7 +410,8 @@ close_netns: static int check_ping_ok(const char *ns1) { SYS(fail, "ip netns exec %s ping -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP4_ADDR_DST); - SYS(fail, "ip netns exec %s ping6 -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP6_ADDR_DST); + SYS(fail, "ip netns exec %s %s -c 1 -W1 -I veth1 %s > /dev/null", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); return 0; fail: return -1; @@ -420,7 +425,8 @@ static int check_ping_fails(const char *ns1) if (!ret) return -1; - ret = SYS_NOFAIL("ip netns exec %s ping6 -c 1 -W1 -I veth1 %s", ns1, IP6_ADDR_DST); + ret = SYS_NOFAIL("ip netns exec %s %s -c 1 -W1 -I veth1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ret) return -1; @@ -538,3 +544,145 @@ void test_lwt_ip_encap_ipv4(void) if (test__start_subtest("ingress")) lwt_ip_encap(IPV4_ENCAP, INGRESS, ""); } + +/* + * VxLAN Setup/topology: + * + * NS1 (IP*_ADDR_1) NS2 NS3 (IP*_ADDR_4) + * [ping src] + * | top route + * veth1 (LWT encap) <<-- veth2 veth3 <<-- veth4 (ping dst) + * | ^ + * (bottom route) | (inner pkt) + * v bottom route | + * veth5 -->> veth6 veth7 -->> veth8 (vxlan decap) + * (IP*_ADDR_VXLAN) + * + * Add the VxLAN endpoint addresses to NS3's veth8, create standard + * VxLAN decap devices bound to those addresses, and install routes so + * NS1/NS2 can reach the endpoints via the bottom route. NS2 here is to + * make sure the LWT-encap VxLAN packets are routed to NS3 correctly. + */ +static int setup_vxlan_routes(const char *ns3, const char *ns1, const char *ns2) +{ + struct nstoken *nstoken; + + nstoken = open_netns(ns3); + if (!ASSERT_OK_PTR(nstoken, "open ns3 for vxlan")) + return -1; + + SYS(fail_close, "ip a add %s/32 dev veth8", IP4_ADDR_VXLAN); + SYS(fail_close, "ip -6 a add %s/128 dev veth8", IP6_ADDR_VXLAN); + /* + * Standard VxLAN devices to decap the encapsulated packets. The inner + * Ethernet frame uses a broadcast dst MAC so the IP stack accepts it + * without ARP or FDB configuration. + */ + SYS(fail_close, "ip link add vxlan4 type vxlan id 1 dstport 4789 local %s dev veth8 nolearning noudpcsum", + IP4_ADDR_VXLAN); + SYS(fail_close, "ip link set vxlan4 up"); + SYS(fail_close, "ip link add vxlan6 type vxlan id 1 dstport 4789 local %s dev veth8 nolearning udp6zerocsumrx", + IP6_ADDR_VXLAN); + SYS(fail_close, "ip link set vxlan6 up"); + close_netns(nstoken); + + SYS(fail, "ip -n %s route add %s/32 dev veth5 via %s", + ns1, IP4_ADDR_VXLAN, IP4_ADDR_6); + SYS(fail, "ip -n %s route add %s/32 dev veth7 via %s", + ns2, IP4_ADDR_VXLAN, IP4_ADDR_8); + SYS(fail, "ip -n %s -6 route add %s/128 dev veth5 via %s", + ns1, IP6_ADDR_VXLAN, IP6_ADDR_6); + SYS(fail, "ip -n %s -6 route add %s/128 dev veth7 via %s", + ns2, IP6_ADDR_VXLAN, IP6_ADDR_8); + return 0; + +fail_close: + close_netns(nstoken); +fail: + return -1; +} + +static void lwt_ip_encap_vxlan(bool ipv4_encap) +{ + char ns1[NETNS_NAME_SIZE] = NETNS_BASE "-1-"; + char ns2[NETNS_NAME_SIZE] = NETNS_BASE "-2-"; + char ns3[NETNS_NAME_SIZE] = NETNS_BASE "-3-"; + const char *sec = ipv4_encap ? "encap_vxlan" : "encap_vxlan6"; + int expected_offset = ipv4_encap ? (int)sizeof(struct iphdr) + : (int)sizeof(struct ipv6hdr); + struct test_lwt_ip_encap *skel = NULL; + int thdr_offset, err; + + if (!ASSERT_OK(create_ns(ns1, NETNS_NAME_SIZE), "create ns1")) + goto out; + if (!ASSERT_OK(create_ns(ns2, NETNS_NAME_SIZE), "create ns2")) + goto out; + if (!ASSERT_OK(create_ns(ns3, NETNS_NAME_SIZE), "create ns3")) + goto out; + + if (!ASSERT_OK(setup_network(ns1, ns2, ns3, ""), "setup network")) + goto out; + + if (!ASSERT_OK(setup_vxlan_routes(ns3, ns1, ns2), "setup vxlan routes")) + goto out; + + skel = test_lwt_ip_encap__open(); + if (!ASSERT_OK_PTR(skel, "test_lwt_ip_encap__open")) + goto out; + + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_gre, false); + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_gre6, false); + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_vxlan, false); + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_vxlan6, false); + bpf_program__set_autoload(skel->progs.fexit_lwt_push_ip_encap, true); + skel->rodata->tgt_ip_version = ipv4_encap ? 4 : 6; + + err = test_lwt_ip_encap__load(skel); + if (!ASSERT_OK(err, "test_lwt_ip_encap__load")) + goto out; + + err = test_lwt_ip_encap__attach(skel); + if (!ASSERT_OK(err, "test_lwt_ip_encap__attach")) + goto out; + + /* Remove the direct NS2->DST route so packets must go via LWT encap. */ + SYS(out, "ip -n %s route del %s/32 dev veth3", ns2, IP4_ADDR_DST); + SYS(out, "ip -n %s -6 route del %s/128 dev veth3", ns2, IP6_ADDR_DST); + + if (ipv4_encap) + SYS(out, "ip -n %s route add %s encap bpf xmit obj %s sec %s dev veth1", + ns1, IP4_ADDR_DST, BPF_FILE, sec); + else + SYS(out, "ip -n %s -6 route add %s encap bpf xmit obj %s sec %s dev veth1", + ns1, IP6_ADDR_DST, BPF_FILE, sec); + + skel->bss->fexit_triggered = false; + + if (ipv4_encap) + SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); + else + SYS(out, "ip netns exec %s %s -c 1 -W1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); + + if (!ASSERT_TRUE(skel->bss->fexit_triggered, "fexit_triggered")) + goto out; + + thdr_offset = (int)skel->bss->transport_hdr - (int)skel->bss->network_hdr; + ASSERT_EQ(thdr_offset, expected_offset, "transport_hdr offset"); + +out: + test_lwt_ip_encap__destroy(skel); + SYS_NOFAIL("ip netns del %s", ns1); + SYS_NOFAIL("ip netns del %s", ns2); + SYS_NOFAIL("ip netns del %s", ns3); +} + +void test_lwt_ip_encap_vxlan_ipv4(void) +{ + lwt_ip_encap_vxlan(IPV4_ENCAP); +} + +void test_lwt_ip_encap_vxlan_ipv6(void) +{ + lwt_ip_encap_vxlan(IPV6_ENCAP); +} diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_misc.c b/tools/testing/selftests/bpf/prog_tests/lwt_misc.c new file mode 100644 index 000000000000..6940fca38512 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lwt_misc.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "lwt_misc.skel.h" + +void test_lwt_misc(void) +{ + RUN_TESTS(lwt_misc); +} diff --git a/tools/testing/selftests/bpf/prog_tests/map_excl.c b/tools/testing/selftests/bpf/prog_tests/map_excl.c index 6bdc6d6de0da..3f4422b9ffa6 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_excl.c +++ b/tools/testing/selftests/bpf/prog_tests/map_excl.c @@ -7,6 +7,11 @@ #include <bpf/btf.h> #include "map_excl.skel.h" +#include "bpf_iter_bpf_array_map.skel.h" + +#ifndef SHA256_DIGEST_SIZE +#define SHA256_DIGEST_SIZE 32 +#endif static void test_map_excl_allowed(void) { @@ -45,10 +50,127 @@ out: } +static void test_map_excl_no_map_in_map(void) +{ + __u8 hash[SHA256_DIGEST_SIZE] = {}; + LIBBPF_OPTS(bpf_map_create_opts, excl_opts, + .excl_prog_hash = hash, + .excl_prog_hash_size = sizeof(hash)); + LIBBPF_OPTS(bpf_map_create_opts, outer_opts); + int excl_fd, tmpl_fd = -1, outer_fd = -1, err; + __u32 key = 0; + + excl_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl_inner", 4, 4, 1, &excl_opts); + if (!ASSERT_OK_FD(excl_fd, "create exclusive map")) + return; + + outer_opts.inner_map_fd = excl_fd; + err = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer_from_excl", + 4, 4, 1, &outer_opts); + if (err >= 0) + close(err); + ASSERT_EQ(err, -ENOTSUPP, "reject exclusive map as map-in-map template"); + + tmpl_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "tmpl", 4, 4, 1, NULL); + if (!ASSERT_OK_FD(tmpl_fd, "create inner template")) + goto out; + + outer_opts.inner_map_fd = tmpl_fd; + outer_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer", 4, 4, 1, &outer_opts); + if (!ASSERT_OK_FD(outer_fd, "create map-of-maps")) + goto out; + + err = bpf_map_update_elem(outer_fd, &key, &excl_fd, 0); + ASSERT_EQ(err, -ENOTSUPP, "reject exclusive map as map-in-map element"); +out: + if (outer_fd >= 0) + close(outer_fd); + if (tmpl_fd >= 0) + close(tmpl_fd); + close(excl_fd); +} + +static void test_map_excl_no_map_iter(void) +{ + __u8 hash[SHA256_DIGEST_SIZE] = {}; + LIBBPF_OPTS(bpf_map_create_opts, excl_opts, + .excl_prog_hash = hash, + .excl_prog_hash_size = sizeof(hash)); + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + struct bpf_iter_bpf_array_map *skel = NULL; + union bpf_iter_link_info linfo; + struct bpf_link *link; + int excl_fd; + + excl_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl_iter", 4, 8, 3, &excl_opts); + if (!ASSERT_OK_FD(excl_fd, "create exclusive map")) + return; + + skel = bpf_iter_bpf_array_map__open_and_load(); + if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load")) + goto out; + + memset(&linfo, 0, sizeof(linfo)); + linfo.map.map_fd = excl_fd; + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts); + if (!ASSERT_ERR_PTR(link, "reject exclusive map as iter target")) { + bpf_link__destroy(link); + goto out; + } + ASSERT_EQ(libbpf_get_error(link), -EPERM, "iter attach errno"); +out: + bpf_iter_bpf_array_map__destroy(skel); + close(excl_fd); +} + +static void test_map_excl_create_validation(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, o); + __u8 hash[SHA256_DIGEST_SIZE] = {}; + int fd; + + o.excl_prog_hash = hash; + o.excl_prog_hash_size = SHA256_DIGEST_SIZE / 2; + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl", 4, 4, 1, &o); + if (fd >= 0) + close(fd); + ASSERT_EQ(fd, -EINVAL, "reject short excl_prog_hash_size"); + + o.excl_prog_hash = hash; + o.excl_prog_hash_size = SHA256_DIGEST_SIZE * 2; + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl", 4, 4, 1, &o); + if (fd >= 0) + close(fd); + ASSERT_EQ(fd, -EINVAL, "reject long excl_prog_hash_size"); + + o.excl_prog_hash = hash; + o.excl_prog_hash_size = 0; + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl", 4, 4, 1, &o); + if (fd >= 0) + close(fd); + ASSERT_EQ(fd, -EINVAL, "reject hash pointer with zero size"); + + o.excl_prog_hash = NULL; + o.excl_prog_hash_size = SHA256_DIGEST_SIZE; + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "excl", 4, 4, 1, &o); + if (fd >= 0) + close(fd); + ASSERT_EQ(fd, -EINVAL, "reject size with NULL hash pointer"); +} + void test_map_excl(void) { if (test__start_subtest("map_excl_allowed")) test_map_excl_allowed(); if (test__start_subtest("map_excl_denied")) test_map_excl_denied(); + if (test__start_subtest("map_excl_no_map_in_map")) + test_map_excl_no_map_in_map(); + if (test__start_subtest("map_excl_no_map_iter")) + test_map_excl_no_map_iter(); + if (test__start_subtest("map_excl_create_validation")) + test_map_excl_create_validation(); } diff --git a/tools/testing/selftests/bpf/prog_tests/map_init.c b/tools/testing/selftests/bpf/prog_tests/map_init.c index 14a31109dd0e..c804c3ce9be9 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_init.c +++ b/tools/testing/selftests/bpf/prog_tests/map_init.c @@ -212,3 +212,195 @@ void test_map_init(void) if (test__start_subtest("pcpu_lru_map_init")) test_pcpu_lru_map_init(); } + +static void test_map_create(enum bpf_map_type map_type, const char *map_name, + struct bpf_map_create_opts *opts, const char *exp_msg) +{ + const int key_size = 4, value_size = 4, max_entries = 1; + char log_buf[128]; + int fd; + LIBBPF_OPTS(bpf_log_opts, log_opts); + + log_buf[0] = '\0'; + log_opts.buf = log_buf; + log_opts.size = sizeof(log_buf); + log_opts.level = 1; + opts->log_opts = &log_opts; + fd = bpf_map_create(map_type, map_name, key_size, value_size, max_entries, opts); + if (!ASSERT_LT(fd, 0, "bpf_map_create")) { + close(fd); + return; + } + + ASSERT_STREQ(log_buf, exp_msg, "log_buf"); + ASSERT_EQ(log_opts.true_size, strlen(exp_msg) + 1, "true_size"); +} + +static void test_map_create_array(struct bpf_map_create_opts *opts, const char *exp_msg) +{ + test_map_create(BPF_MAP_TYPE_ARRAY, "test_map_create", opts, exp_msg); +} + +static void test_invalid_vmlinux_value_type_id_struct_ops(void) +{ + const char *msg = "btf_vmlinux_value_type_id can only be used with struct_ops maps.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .btf_vmlinux_value_type_id = 1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_invalid_vmlinux_value_type_id_kv_type_id(void) +{ + const char *msg = "btf_vmlinux_value_type_id is mutually exclusive with btf_key_type_id and btf_value_type_id.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .btf_vmlinux_value_type_id = 1, + .btf_key_type_id = 1, + ); + + test_map_create(BPF_MAP_TYPE_STRUCT_OPS, "test_map_create", &opts, msg); +} + +static void test_invalid_value_type_id(void) +{ + const char *msg = "Invalid btf_value_type_id.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .btf_key_type_id = 1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_invalid_map_extra(void) +{ + const char *msg = "Invalid map_extra.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_extra = 1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_invalid_numa_node(void) +{ + const char *msg = "Invalid numa_node.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_NUMA_NODE, + .numa_node = 0xFF, + ); + + test_map_create_array(&opts, msg); +} + +static void test_invalid_map_type(void) +{ + const char *msg = "Invalid map_type.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts); + + test_map_create(__MAX_BPF_MAP_TYPE, "test_map_create", &opts, msg); +} + +static void test_invalid_token_fd(void) +{ + const char *msg = "Invalid map_token_fd.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_TOKEN_FD, + .token_fd = -1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_invalid_map_name(void) +{ + const char *msg = "Invalid map_name.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts); + + test_map_create(BPF_MAP_TYPE_ARRAY, "test-!@#", &opts, msg); +} + +static void test_invalid_btf_fd(void) +{ + const char *msg = "Invalid btf_fd.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .btf_fd = -1, + .btf_key_type_id = 1, + .btf_value_type_id = 1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_excl_prog_hash_size_1(void) +{ + const char *msg = "Invalid excl_prog_hash_size.\n"; + const char *hash = "DEADCODE"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .excl_prog_hash = hash, + ); + + test_map_create_array(&opts, msg); +} + +static void test_excl_prog_hash_size_2(void) +{ + const char *msg = "Invalid excl_prog_hash_size.\n"; + LIBBPF_OPTS(bpf_map_create_opts, opts, + .excl_prog_hash_size = 1, + ); + + test_map_create_array(&opts, msg); +} + +static void test_common_attr_padding(void) +{ + struct bpf_common_attr_fake { + __u8 attrs[offsetofend(struct bpf_common_attr, log_true_size)]; + __u32 pad; + } attr_common = { + .pad = 1, + }; + union bpf_attr attr = { + .map_type = BPF_MAP_TYPE_ARRAY, + .key_size = 4, + .value_size = 4, + .max_entries = 1, + }; + int fd; + + fd = syscall(__NR_bpf, BPF_MAP_CREATE | BPF_COMMON_ATTRS, &attr, sizeof(attr), &attr_common, + sizeof(attr_common)); + if (!ASSERT_LT(fd, 0, "syscall")) + close(fd); + else + ASSERT_EQ(errno, E2BIG, "errno"); +} + +void test_map_create_failure(void) +{ + if (test__start_subtest("invalid_vmlinux_value_type_id_struct_ops")) + test_invalid_vmlinux_value_type_id_struct_ops(); + if (test__start_subtest("invalid_vmlinux_value_type_id_kv_type_id")) + test_invalid_vmlinux_value_type_id_kv_type_id(); + if (test__start_subtest("invalid_value_type_id")) + test_invalid_value_type_id(); + if (test__start_subtest("invalid_map_extra")) + test_invalid_map_extra(); + if (test__start_subtest("invalid_numa_node")) + test_invalid_numa_node(); + if (test__start_subtest("invalid_map_type")) + test_invalid_map_type(); + if (test__start_subtest("invalid_token_fd")) + test_invalid_token_fd(); + if (test__start_subtest("invalid_map_name")) + test_invalid_map_name(); + if (test__start_subtest("invalid_btf_fd")) + test_invalid_btf_fd(); + if (test__start_subtest("invalid_excl_prog_hash_size_1")) + test_excl_prog_hash_size_1(); + if (test__start_subtest("invalid_excl_prog_hash_size_2")) + test_excl_prog_hash_size_2(); + if (test__start_subtest("common_attr_padding")) + test_common_attr_padding(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/map_kptr.c b/tools/testing/selftests/bpf/prog_tests/map_kptr.c index 03b46f17cf53..17e707dddda8 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_kptr.c +++ b/tools/testing/selftests/bpf/prog_tests/map_kptr.c @@ -51,7 +51,6 @@ static void test_map_kptr_success(bool test_run) ret = bpf_map__update_elem(skel->maps.array_map, &key, sizeof(key), buf, sizeof(buf), 0); ASSERT_OK(ret, "array_map update"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); @@ -59,49 +58,42 @@ static void test_map_kptr_success(bool test_run) ret = bpf_map__update_elem(skel->maps.pcpu_array_map, &key, sizeof(key), pbuf, cpu * sizeof(buf), 0); ASSERT_OK(ret, "pcpu_array_map update"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.pcpu_hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "pcpu_hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.hash_malloc_map, &key, sizeof(key), 0); ASSERT_OK(ret, "hash_malloc_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.pcpu_hash_malloc_map, &key, sizeof(key), 0); ASSERT_OK(ret, "pcpu_hash_malloc_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.lru_hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "lru_hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); ret = bpf_map__delete_elem(skel->maps.lru_pcpu_hash_map, &key, sizeof(key), 0); ASSERT_OK(ret, "lru_pcpu_hash_map delete"); - skel->data->ref--; ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.test_map_kptr_ref3), &opts); ASSERT_OK(ret, "test_map_kptr_ref3 refcount"); ASSERT_OK(opts.retval, "test_map_kptr_ref3 retval"); @@ -151,12 +143,68 @@ static void wait_for_map_release(void) map_kptr__destroy(skel); } +enum map_update_kptr_case { + MAP_UPDATE_KPTR_ARRAY, + MAP_UPDATE_KPTR_HASH, + MAP_UPDATE_KPTR_HASH_MALLOC, +}; + +static struct bpf_program *map_update_kptr_prog(struct map_kptr *skel, + enum map_update_kptr_case test) +{ + switch (test) { + case MAP_UPDATE_KPTR_ARRAY: + return skel->progs.test_array_map_update_kptr; + case MAP_UPDATE_KPTR_HASH: + return skel->progs.test_hash_map_update_kptr; + case MAP_UPDATE_KPTR_HASH_MALLOC: + return skel->progs.test_hash_malloc_map_update_kptr; + } + + return NULL; +} + +static void test_map_update_kptr(enum map_update_kptr_case test) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct map_kptr *skel; + struct bpf_program *prog; + int ret; + + skel = map_kptr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "map_kptr__open_and_load")) + return; + + prog = map_update_kptr_prog(skel, test); + if (!ASSERT_OK_PTR(prog, "map_update_kptr_prog")) + goto out; + + ret = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(ret, "map_update_kptr")) + goto out; + if (!ASSERT_OK(opts.retval, "map_update_kptr retval")) + goto out; + + ASSERT_EQ(skel->bss->num_of_refs, 3, "refs_after_update"); + +out: + map_kptr__destroy(skel); + wait_for_map_release(); +} + void serial_test_map_kptr(void) { struct rcu_tasks_trace_gp *skel; RUN_TESTS(map_kptr_fail); + if (test__start_subtest("update_array_map_kptr")) + test_map_update_kptr(MAP_UPDATE_KPTR_ARRAY); + if (test__start_subtest("update_hash_map_kptr")) + test_map_update_kptr(MAP_UPDATE_KPTR_HASH); + if (test__start_subtest("update_hash_malloc_map_kptr")) + test_map_update_kptr(MAP_UPDATE_KPTR_HASH_MALLOC); + skel = rcu_tasks_trace_gp__open_and_load(); if (!ASSERT_OK_PTR(skel, "rcu_tasks_trace_gp__open_and_load")) return; @@ -175,7 +223,7 @@ void serial_test_map_kptr(void) ASSERT_OK(kern_sync_rcu(), "sync rcu"); wait_for_map_release(); - /* Observe refcount dropping to 1 on synchronous delete elem */ + /* Observe refcount dropping to 1 on map release. */ test_map_kptr_success(true); } diff --git a/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c b/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c index 653b0a20fab9..c62907732c19 100644 --- a/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c +++ b/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c @@ -7,24 +7,29 @@ * 3. call listen() for 1 server socket. (migration target) * 4. update a map to migrate all child sockets * to the last server socket (migrate_map[cookie] = 4) - * 5. call shutdown() for first 4 server sockets + * 5. for TCP_ESTABLISHED and TCP_SYN_RECV cases, verify via epoll + * that the last server socket is not ready before migration. + * 6. call shutdown() for first 4 server sockets * and migrate the requests in the accept queue * to the last server socket. - * 6. call listen() for the second server socket. - * 7. call shutdown() for the last server + * 7. for TCP_ESTABLISHED and TCP_SYN_RECV cases, verify via epoll + * that the last server socket is ready after migration. + * 8. call listen() for the second server socket. + * 9. call shutdown() for the last server * and migrate the requests in the accept queue * to the second server socket. - * 8. call listen() for the last server. - * 9. call shutdown() for the second server + * 10. call listen() for the last server. + * 11. call shutdown() for the second server * and migrate the requests in the accept queue * to the last server socket. - * 10. call accept() for the last server socket. + * 12. call accept() for the last server socket. * * Author: Kuniyuki Iwashima <kuniyu@amazon.co.jp> */ #include <bpf/bpf.h> #include <bpf/libbpf.h> +#include <sys/epoll.h> #include "test_progs.h" #include "test_migrate_reuseport.skel.h" @@ -350,21 +355,51 @@ static int update_maps(struct migrate_reuseport_test_case *test_case, static int migrate_dance(struct migrate_reuseport_test_case *test_case) { + struct epoll_event ev = { + .events = EPOLLIN, + }; + int epoll = -1, nfds; int i, err; + if (test_case->state != BPF_TCP_NEW_SYN_RECV) { + epoll = epoll_create1(0); + if (!ASSERT_NEQ(epoll, -1, "epoll_create1")) + return -1; + + ev.data.fd = test_case->servers[MIGRATED_TO]; + if (!ASSERT_OK(epoll_ctl(epoll, EPOLL_CTL_ADD, + test_case->servers[MIGRATED_TO], &ev), + "epoll_ctl")) + goto close_epoll; + + nfds = epoll_wait(epoll, &ev, 1, 0); + if (!ASSERT_EQ(nfds, 0, "epoll_wait 1")) + goto close_epoll; + } + /* Migrate TCP_ESTABLISHED and TCP_SYN_RECV requests * to the last listener based on eBPF. */ for (i = 0; i < MIGRATED_TO; i++) { err = shutdown(test_case->servers[i], SHUT_RDWR); if (!ASSERT_OK(err, "shutdown")) - return -1; + goto close_epoll; } /* No dance for TCP_NEW_SYN_RECV to migrate based on eBPF */ if (test_case->state == BPF_TCP_NEW_SYN_RECV) return 0; + nfds = epoll_wait(epoll, &ev, 1, 0); + if (!ASSERT_EQ(nfds, 1, "epoll_wait 2")) { +close_epoll: + if (epoll >= 0) + close(epoll); + return -1; + } + + close(epoll); + /* Note that we use the second listener instead of the * first one here. * diff --git a/tools/testing/selftests/bpf/prog_tests/modify_return.c b/tools/testing/selftests/bpf/prog_tests/modify_return.c index a70c99c2f8c8..4661d77ebdfc 100644 --- a/tools/testing/selftests/bpf/prog_tests/modify_return.c +++ b/tools/testing/selftests/bpf/prog_tests/modify_return.c @@ -5,6 +5,7 @@ */ #include <test_progs.h> +#include <unistd.h> #include "modify_return.skel.h" #define LOWER(x) ((x) & 0xffff) @@ -23,11 +24,13 @@ static void run_test(__u32 input_retval, __u16 want_side_effect, __s16 want_ret) if (!ASSERT_OK_PTR(skel, "skel_load")) goto cleanup; + skel->bss->input_retval = input_retval; + skel->bss->test_pid = getpid(); + err = modify_return__attach(skel); if (!ASSERT_OK(err, "modify_return__attach failed")) goto cleanup; - skel->bss->input_retval = input_retval; prog_fd = bpf_program__fd(skel->progs.fmod_ret_test); err = bpf_prog_test_run_opts(prog_fd, &topts); ASSERT_OK(err, "test_run"); @@ -49,8 +52,7 @@ cleanup: modify_return__destroy(skel); } -/* TODO: conflict with get_func_ip_test */ -void serial_test_modify_return(void) +void test_modify_return(void) { run_test(0 /* input_retval */, 2 /* want_side_effect */, diff --git a/tools/testing/selftests/bpf/prog_tests/module_attach.c b/tools/testing/selftests/bpf/prog_tests/module_attach.c index 70fa7ae93173..92c336333fcb 100644 --- a/tools/testing/selftests/bpf/prog_tests/module_attach.c +++ b/tools/testing/selftests/bpf/prog_tests/module_attach.c @@ -6,7 +6,22 @@ #include "test_module_attach.skel.h" #include "testing_helpers.h" -static int duration; +static const char * const read_tests[] = { + "handle_raw_tp", + "handle_tp_btf", + "handle_fentry", + "handle_fentry_explicit", + "handle_fmod_ret", +}; + +static const char * const detach_tests[] = { + "handle_fentry", + "handle_fexit", + "kprobe_multi", +}; + +static const int READ_SZ = 456; +static const int WRITE_SZ = 457; static int trigger_module_test_writable(int *val) { @@ -33,53 +48,73 @@ static int trigger_module_test_writable(int *val) return 0; } -void test_module_attach(void) +static void test_module_attach_prog(const char *prog_name, int sz, + const char *attach_target, int ret) { - const int READ_SZ = 456; - const int WRITE_SZ = 457; - struct test_module_attach* skel; - struct test_module_attach__bss *bss; - struct bpf_link *link; + struct test_module_attach *skel; + struct bpf_program *prog; int err; - int writable_val = 0; skel = test_module_attach__open(); - if (CHECK(!skel, "skel_open", "failed to open skeleton\n")) + if (!ASSERT_OK_PTR(skel, "module_attach open")) return; - err = bpf_program__set_attach_target(skel->progs.handle_fentry_manual, - 0, "bpf_testmod_test_read"); - ASSERT_OK(err, "set_attach_target"); + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "module_attach find_program")) + goto cleanup; + bpf_program__set_autoload(prog, true); - err = bpf_program__set_attach_target(skel->progs.handle_fentry_explicit_manual, - 0, "bpf_testmod:bpf_testmod_test_read"); - ASSERT_OK(err, "set_attach_target_explicit"); + if (attach_target) { + err = bpf_program__set_attach_target(prog, 0, attach_target); + if (!ASSERT_OK(err, attach_target)) + goto cleanup; + } err = test_module_attach__load(skel); - if (CHECK(err, "skel_load", "failed to load skeleton\n")) + if (!ASSERT_OK(err, "module_attach load")) + goto cleanup; + + err = test_module_attach__attach(skel); + if (!ASSERT_OK(err, "module_attach attach")) + goto cleanup; + + if (sz) { + /* trigger both read and write though each test uses only one */ + ASSERT_OK(trigger_module_test_read(sz), "trigger_read"); + ASSERT_OK(trigger_module_test_write(sz), "trigger_write"); + + ASSERT_EQ(skel->bss->sz, sz, prog_name); + } + + if (ret) + ASSERT_EQ(skel->bss->retval, ret, "ret"); +cleanup: + test_module_attach__destroy(skel); +} + +static void test_module_attach_writable(void) +{ + struct test_module_attach__bss *bss; + struct test_module_attach *skel; + int writable_val = 0; + int err; + + skel = test_module_attach__open(); + if (!ASSERT_OK_PTR(skel, "module_attach open")) return; + bpf_program__set_autoload(skel->progs.handle_raw_tp_writable_bare, true); + + err = test_module_attach__load(skel); + if (!ASSERT_OK(err, "module_attach load")) + goto cleanup; + bss = skel->bss; err = test_module_attach__attach(skel); - if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err)) + if (!ASSERT_OK(err, "module_attach attach")) goto cleanup; - /* trigger tracepoint */ - ASSERT_OK(trigger_module_test_read(READ_SZ), "trigger_read"); - ASSERT_OK(trigger_module_test_write(WRITE_SZ), "trigger_write"); - - ASSERT_EQ(bss->raw_tp_read_sz, READ_SZ, "raw_tp"); - ASSERT_EQ(bss->raw_tp_bare_write_sz, WRITE_SZ, "raw_tp_bare"); - ASSERT_EQ(bss->tp_btf_read_sz, READ_SZ, "tp_btf"); - ASSERT_EQ(bss->fentry_read_sz, READ_SZ, "fentry"); - ASSERT_EQ(bss->fentry_manual_read_sz, READ_SZ, "fentry_manual"); - ASSERT_EQ(bss->fentry_explicit_read_sz, READ_SZ, "fentry_explicit"); - ASSERT_EQ(bss->fentry_explicit_manual_read_sz, READ_SZ, "fentry_explicit_manual"); - ASSERT_EQ(bss->fexit_read_sz, READ_SZ, "fexit"); - ASSERT_EQ(bss->fexit_ret, -EIO, "fexit_tet"); - ASSERT_EQ(bss->fmod_ret_read_sz, READ_SZ, "fmod_ret"); - bss->raw_tp_writable_bare_early_ret = true; bss->raw_tp_writable_bare_out_val = 0xf1f2f3f4; ASSERT_OK(trigger_module_test_writable(&writable_val), @@ -87,31 +122,73 @@ void test_module_attach(void) ASSERT_EQ(bss->raw_tp_writable_bare_in_val, 1024, "writable_test_in"); ASSERT_EQ(bss->raw_tp_writable_bare_out_val, writable_val, "writable_test_out"); +cleanup: + test_module_attach__destroy(skel); +} - test_module_attach__detach(skel); - - /* attach fentry/fexit and make sure it gets module reference */ - link = bpf_program__attach(skel->progs.handle_fentry); - if (!ASSERT_OK_PTR(link, "attach_fentry")) - goto cleanup; +static void test_module_attach_detach(const char *prog_name) +{ + struct test_module_attach *skel; + struct bpf_program *prog; + struct bpf_link *link; + int err; - ASSERT_ERR(unload_bpf_testmod(false), "unload_bpf_testmod"); - bpf_link__destroy(link); + skel = test_module_attach__open(); + if (!ASSERT_OK_PTR(skel, "module_attach open")) + return; - link = bpf_program__attach(skel->progs.handle_fexit); - if (!ASSERT_OK_PTR(link, "attach_fexit")) + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "module_attach find_program")) goto cleanup; + bpf_program__set_autoload(prog, true); - ASSERT_ERR(unload_bpf_testmod(false), "unload_bpf_testmod"); - bpf_link__destroy(link); + err = test_module_attach__load(skel); + if (!ASSERT_OK(err, "module_attach load")) + goto cleanup; - link = bpf_program__attach(skel->progs.kprobe_multi); - if (!ASSERT_OK_PTR(link, "attach_kprobe_multi")) + /* attach and make sure it gets module reference */ + link = bpf_program__attach(prog); + if (!ASSERT_OK_PTR(link, "module_attach attach")) goto cleanup; - ASSERT_ERR(unload_bpf_testmod(false), "unload_bpf_testmod"); + ASSERT_ERR(try_unload_module("bpf_testmod", 1, false), "try_unload_module"); bpf_link__destroy(link); - cleanup: test_module_attach__destroy(skel); } + +void test_module_attach(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(read_tests); i++) { + if (!test__start_subtest(read_tests[i])) + continue; + test_module_attach_prog(read_tests[i], READ_SZ, NULL, 0); + } + if (test__start_subtest("handle_raw_tp_bare")) + test_module_attach_prog("handle_raw_tp_bare", WRITE_SZ, NULL, 0); + if (test__start_subtest("handle_raw_tp_writable_bare")) + test_module_attach_writable(); + if (test__start_subtest("handle_fentry_manual")) { + test_module_attach_prog("handle_fentry_manual", READ_SZ, + "bpf_testmod_test_read", 0); + } + if (test__start_subtest("handle_fentry_explicit_manual")) { + test_module_attach_prog("handle_fentry_explicit_manual", + READ_SZ, + "bpf_testmod:bpf_testmod_test_read", 0); + } + if (test__start_subtest("handle_fexit")) + test_module_attach_prog("handle_fexit", READ_SZ, NULL, -EIO); + if (test__start_subtest("handle_fexit_ret")) + test_module_attach_prog("handle_fexit_ret", 0, NULL, 0); + for (i = 0; i < ARRAY_SIZE(detach_tests); i++) { + char test_name[50]; + + snprintf(test_name, sizeof(test_name), "%s_detach", detach_tests[i]); + if (!test__start_subtest(test_name)) + continue; + test_module_attach_detach(detach_tests[i]); + } +} diff --git a/tools/testing/selftests/bpf/prog_tests/mptcp.c b/tools/testing/selftests/bpf/prog_tests/mptcp.c index 8fade8bdc451..32dfc1c511af 100644 --- a/tools/testing/selftests/bpf/prog_tests/mptcp.c +++ b/tools/testing/selftests/bpf/prog_tests/mptcp.c @@ -264,7 +264,7 @@ static int verify_mptcpify(int server_fd, int client_fd) return err; } -static int run_mptcpify(int cgroup_fd) +static int run_mptcpify(int cgroup_fd, int type) { int server_fd, client_fd, err = 0; struct mptcpify *mptcpify_skel; @@ -280,7 +280,7 @@ static int run_mptcpify(int cgroup_fd) goto out; /* without MPTCP */ - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); + server_fd = start_server(AF_INET, type, NULL, 0, 0); if (!ASSERT_GE(server_fd, 0, "start_server")) { err = -EIO; goto out; @@ -317,7 +317,14 @@ static void test_mptcpify(void) if (!ASSERT_OK_PTR(netns, "netns_new")) goto fail; - ASSERT_OK(run_mptcpify(cgroup_fd), "run_mptcpify"); + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM), "run_mptcpify"); + /* userspace sets flags such as SOCK_CLOEXEC together with the type; + * the BPF prog must still upgrade the socket to MPTCP. See + * update_socket_protocol() in net/socket.c, which runs before the + * type is masked with SOCK_TYPE_MASK. + */ + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM | SOCK_CLOEXEC), + "run_mptcpify_cloexec"); fail: netns_free(netns); diff --git a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c index a72ae0b29f6e..7b4a1e24363b 100644 --- a/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c +++ b/tools/testing/selftests/bpf/prog_tests/percpu_alloc.c @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include <sched.h> #include <test_progs.h> #include "cgroup_helpers.h" #include "percpu_alloc_array.skel.h" @@ -350,6 +352,103 @@ static void test_lru_percpu_hash_cpu_flag(void) test_percpu_map_cpu_flag(BPF_MAP_TYPE_LRU_PERCPU_HASH); } +/* + * A BPF_F_CPU update that creates an element must zero the value on the other + * cpus, rather than leave them holding whatever the recycled element last + * contained. max_entries is 1 so the second key can only reuse the element + * the first one released. + */ +static void test_percpu_map_cpu_flag_create(enum bpf_map_type map_type, __u32 map_flags) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = map_flags); + const u32 stale = 0xDEADC0DE, fresh = 0xC0FFEE; + int nr_cpus, cpu, map_fd, err, key; + int pinned_cpu, value_cpu; + cpu_set_t old_mask, new_mask; + bool restore_mask = false; + u32 value; + u64 flags; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + if (nr_cpus < 2) { + test__skip(); + return; + } + + map_fd = bpf_map_create(map_type, "cpu_flag_create", sizeof(key), sizeof(value), 1, &opts); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) + return; + + /* NO_PREALLOC recycles per cpu, so keep the delete and the create on one cpu. */ + err = sched_getaffinity(0, sizeof(old_mask), &old_mask); + if (!ASSERT_OK(err, "sched_getaffinity")) + goto out; + + pinned_cpu = sched_getcpu(); + if (!ASSERT_GE(pinned_cpu, 0, "sched_getcpu")) + goto out; + + CPU_ZERO(&new_mask); + CPU_SET(pinned_cpu, &new_mask); + err = sched_setaffinity(0, sizeof(new_mask), &new_mask); + if (!ASSERT_OK(err, "sched_setaffinity")) + goto out; + restore_mask = true; + + value_cpu = pinned_cpu ? 0 : 1; + + key = 1; + value = stale; + err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS); + if (!ASSERT_OK(err, "bpf_map_update_elem all_cpus")) + goto out; + + err = bpf_map_delete_elem(map_fd, &key); + if (!ASSERT_OK(err, "bpf_map_delete_elem")) + goto out; + + key = 2; + value = fresh; + flags = (u64)value_cpu << 32 | BPF_F_CPU; + err = bpf_map_update_elem(map_fd, &key, &value, flags); + if (!ASSERT_OK(err, "bpf_map_update_elem specified cpu")) + goto out; + + for (cpu = 0; cpu < nr_cpus; cpu++) { + value = 0; + flags = (u64)cpu << 32 | BPF_F_CPU; + err = bpf_map_lookup_elem_flags(map_fd, &key, &value, flags); + if (!ASSERT_OK(err, "bpf_map_lookup_elem_flags specified cpu")) + goto out; + if (!ASSERT_EQ(value, cpu == value_cpu ? fresh : 0, "value on specified cpu")) + goto out; + } + +out: + if (restore_mask) + sched_setaffinity(0, sizeof(old_mask), &old_mask); + close(map_fd); +} + +static void test_percpu_hash_cpu_flag_create(void) +{ + test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, 0); +} + +static void test_percpu_hash_cpu_flag_create_malloc(void) +{ + test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_PERCPU_HASH, BPF_F_NO_PREALLOC); +} + +static void test_lru_percpu_hash_cpu_flag_create(void) +{ + /* lru without prealloc is -ENOTSUPP, so there is no malloc variant */ + test_percpu_map_cpu_flag_create(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0); +} + static void test_percpu_cgroup_storage_cpu_flag(void) { struct percpu_alloc_array *skel = NULL; @@ -454,6 +553,12 @@ void test_percpu_alloc(void) test_percpu_hash_cpu_flag(); if (test__start_subtest("cpu_flag_lru_percpu_hash")) test_lru_percpu_hash_cpu_flag(); + if (test__start_subtest("cpu_flag_create_percpu_hash")) + test_percpu_hash_cpu_flag_create(); + if (test__start_subtest("cpu_flag_create_percpu_hash_malloc")) + test_percpu_hash_cpu_flag_create_malloc(); + if (test__start_subtest("cpu_flag_create_lru_percpu_hash")) + test_lru_percpu_hash_cpu_flag_create(); if (test__start_subtest("cpu_flag_percpu_cgroup_storage")) test_percpu_cgroup_storage_cpu_flag(); if (test__start_subtest("cpu_flag_array")) diff --git a/tools/testing/selftests/bpf/prog_tests/percpu_array_inner_map.c b/tools/testing/selftests/bpf/prog_tests/percpu_array_inner_map.c new file mode 100644 index 000000000000..2a8b2381306b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/percpu_array_inner_map.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> + +/* + * Test that replacing an inner percpu array map with one that has different + * max_entries is rejected. percpu_array_map_gen_lookup() inlines the + * template's index_mask, so allowing a smaller replacement would cause OOB. + */ +void test_percpu_array_inner_map(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + int outer_fd, tmpl_fd, good_fd, bad_fd, err; + int zero = 0; + + /* Create template: percpu array with 8 entries */ + tmpl_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "tmpl", + sizeof(int), sizeof(long), 8, NULL); + if (!ASSERT_OK_FD(tmpl_fd, "create_tmpl")) + return; + + /* Create outer array-of-maps using template */ + opts.inner_map_fd = tmpl_fd; + outer_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer", + sizeof(int), sizeof(int), 1, &opts); + if (!ASSERT_OK_FD(outer_fd, "create_outer")) + goto close_tmpl; + + /* Insert template as initial inner map */ + err = bpf_map_update_elem(outer_fd, &zero, &tmpl_fd, 0); + if (!ASSERT_OK(err, "insert_tmpl")) + goto close_outer; + + /* Replacement with same max_entries should succeed */ + good_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "good", + sizeof(int), sizeof(long), 8, NULL); + if (!ASSERT_OK_FD(good_fd, "create_good")) + goto close_outer; + + err = bpf_map_update_elem(outer_fd, &zero, &good_fd, 0); + ASSERT_OK(err, "replace_same_max_entries"); + close(good_fd); + + /* Replacement with fewer max_entries must fail */ + bad_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "bad", + sizeof(int), sizeof(long), 2, NULL); + if (!ASSERT_OK_FD(bad_fd, "create_bad")) + goto close_outer; + + err = bpf_map_update_elem(outer_fd, &zero, &bad_fd, 0); + ASSERT_ERR(err, "replace_smaller_max_entries"); + close(bad_fd); + +close_outer: + close(outer_fd); +close_tmpl: + close(tmpl_fd); +} diff --git a/tools/testing/selftests/bpf/prog_tests/perf_link.c b/tools/testing/selftests/bpf/prog_tests/perf_link.c index d940ff87fa08..9e3a0d217af8 100644 --- a/tools/testing/selftests/bpf/prog_tests/perf_link.c +++ b/tools/testing/selftests/bpf/prog_tests/perf_link.c @@ -1,8 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2021 Facebook */ #define _GNU_SOURCE -#include <pthread.h> -#include <sched.h> +#include <linux/compiler.h> #include <test_progs.h> #include "testing_helpers.h" #include "test_perf_link.skel.h" @@ -12,23 +11,14 @@ static void burn_cpu(void) { - volatile int j = 0; - cpu_set_t cpu_set; - int i, err; - - /* generate some branches on cpu 0 */ - CPU_ZERO(&cpu_set); - CPU_SET(0, &cpu_set); - err = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set), &cpu_set); - ASSERT_OK(err, "set_thread_affinity"); + int i; /* spin the loop for a while (random high number) */ for (i = 0; i < 1000000; ++i) - ++j; + barrier(); } -/* TODO: often fails in concurrent mode */ -void serial_test_perf_link(void) +void test_perf_link(void) { struct test_perf_link *skel = NULL; struct perf_event_attr attr; @@ -45,7 +35,7 @@ void serial_test_perf_link(void) attr.config = PERF_COUNT_SW_CPU_CLOCK; attr.freq = 1; attr.sample_freq = 1000; - pfd = syscall(__NR_perf_event_open, &attr, -1, 0, -1, PERF_FLAG_FD_CLOEXEC); + pfd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, PERF_FLAG_FD_CLOEXEC); if (!ASSERT_GE(pfd, 0, "perf_fd")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/probe_user.c b/tools/testing/selftests/bpf/prog_tests/probe_user.c index 8721671321de..7a9d13aa2c87 100644 --- a/tools/testing/selftests/bpf/prog_tests/probe_user.c +++ b/tools/testing/selftests/bpf/prog_tests/probe_user.c @@ -1,8 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> -/* TODO: corrupts other tests uses connect() */ -void serial_test_probe_user(void) +void test_probe_user(void) { static const char *const prog_names[] = { "handle_sys_connect", @@ -20,6 +19,11 @@ void serial_test_probe_user(void) struct bpf_program *kprobe_progs[prog_count]; struct bpf_object *obj; static const int zero = 0; + struct test_pro_bss { + struct sockaddr_in old; + __u32 test_pid; + }; + struct test_pro_bss results = {}; size_t i; obj = bpf_object__open_file(obj_file, &opts); @@ -34,6 +38,23 @@ void serial_test_probe_user(void) goto cleanup; } + { + struct bpf_map *bss_map; + struct test_pro_bss bss_init = {}; + + bss_init.test_pid = getpid(); + bss_map = bpf_object__find_map_by_name(obj, "test_pro.bss"); + if (!ASSERT_OK_PTR(bss_map, "find_bss_map")) + goto cleanup; + if (!ASSERT_EQ(bpf_map__value_size(bss_map), sizeof(bss_init), + "bss_size")) + goto cleanup; + err = bpf_map__set_initial_value(bss_map, &bss_init, + sizeof(bss_init)); + if (!ASSERT_OK(err, "set_bss_init")) + goto cleanup; + } + err = bpf_object__load(obj); if (CHECK(err, "obj_load", "err %d\n", err)) goto cleanup; @@ -62,11 +83,13 @@ void serial_test_probe_user(void) connect(sock_fd, &curr, sizeof(curr)); close(sock_fd); - err = bpf_map_lookup_elem(results_map_fd, &zero, &tmp); + err = bpf_map_lookup_elem(results_map_fd, &zero, &results); if (CHECK(err, "get_kprobe_res", "failed to get kprobe res: %d\n", err)) goto cleanup; + memcpy(&tmp, &results.old, sizeof(tmp)); + in = (struct sockaddr_in *)&tmp; if (CHECK(memcmp(&tmp, &orig, sizeof(orig)), "check_kprobe_res", "wrong kprobe res from probe read: %s:%u\n", diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c new file mode 100644 index 000000000000..b8538fc4fc3f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_bad_access.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "test_kmods/bpf_testmod.h" +#include "bpf_util.h" + +static void check_attach_reject(const struct bpf_insn *program, size_t prog_len) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts); + char error[4096]; + int bpf_fd, tp_fd; + + opts.log_level = 2; + opts.log_buf = error; + opts.log_size = sizeof(error); + + bpf_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, NULL, "GPL v2", + program, prog_len, &opts); + if (!ASSERT_GE(bpf_fd, 0, "prog_load")) + return; + + tp_fd = bpf_raw_tracepoint_open("bpf_testmod_test_writable_bare_tp", bpf_fd); + ASSERT_EQ(tp_fd, -EINVAL, "bpf_raw_tracepoint_open"); + if (tp_fd >= 0) + close(tp_fd); + + close(bpf_fd); +} + +void test_raw_tp_writable_reject_bad_access(void) +{ + const struct bpf_insn program[] = { + /* r6 is our tp buffer */ + BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0), + /* one byte beyond the end of the writable context */ + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6, + sizeof(struct bpf_testmod_test_writable_ctx)), + BPF_EXIT_INSN(), + }; + + const struct bpf_insn negative_var_off_program[] = { + BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0), + /* make var_off negative, but keep the effective access offset non-negative */ + BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, -8), + /* one byte beyond the end of the writable context */ + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6, + sizeof(struct bpf_testmod_test_writable_ctx) + 8), + BPF_EXIT_INSN(), + }; + + if (test__start_subtest("past_end")) + check_attach_reject(program, ARRAY_SIZE(program)); + + if (test__start_subtest("negative_var_off_past_end")) + check_attach_reject(negative_var_off_program, + ARRAY_SIZE(negative_var_off_program)); +} diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c deleted file mode 100644 index 216b0dfac0fe..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include <test_progs.h> -#include <linux/nbd.h> -#include "bpf_util.h" - -void test_raw_tp_writable_reject_nbd_invalid(void) -{ - __u32 duration = 0; - char error[4096]; - int bpf_fd = -1, tp_fd = -1; - - const struct bpf_insn program[] = { - /* r6 is our tp buffer */ - BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 0), - /* one byte beyond the end of the nbd_request struct */ - BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_6, - sizeof(struct nbd_request)), - BPF_EXIT_INSN(), - }; - - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .log_level = 2, - .log_buf = error, - .log_size = sizeof(error), - ); - - bpf_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, NULL, "GPL v2", - program, ARRAY_SIZE(program), - &opts); - if (CHECK(bpf_fd < 0, "bpf_raw_tracepoint_writable load", - "failed: %d errno %d\n", bpf_fd, errno)) - return; - - tp_fd = bpf_raw_tracepoint_open("nbd_send_request", bpf_fd); - if (CHECK(tp_fd >= 0, "bpf_raw_tracepoint_writable open", - "erroneously succeeded\n")) - goto out_bpffd; - - close(tp_fd); -out_bpffd: - close(bpf_fd); -} diff --git a/tools/testing/selftests/bpf/prog_tests/rbtree.c b/tools/testing/selftests/bpf/prog_tests/rbtree.c index d8f3d7a45fe9..a854fb38e418 100644 --- a/tools/testing/selftests/bpf/prog_tests/rbtree.c +++ b/tools/testing/selftests/bpf/prog_tests/rbtree.c @@ -9,6 +9,7 @@ #include "rbtree_btf_fail__wrong_node_type.skel.h" #include "rbtree_btf_fail__add_wrong_type.skel.h" #include "rbtree_search.skel.h" +#include "rbtree_search_kptr.skel.h" static void test_rbtree_add_nodes(void) { @@ -193,3 +194,8 @@ void test_rbtree_search(void) { RUN_TESTS(rbtree_search); } + +void test_rbtree_search_kptr(void) +{ + RUN_TESTS(rbtree_search_kptr); +} diff --git a/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c b/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c index 246eb259c08a..6a07b2b418d1 100644 --- a/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c @@ -34,6 +34,8 @@ static void test_success(void) bpf_program__set_autoload(skel->progs.rcu_read_lock_global_subprog, true); bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog_lock, true); bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog_unlock, true); + bpf_program__set_autoload(skel->progs.non_own_ref_untrusted_ld, true); + bpf_program__set_autoload(skel->progs.rcu_untrusted_union_ld, true); err = rcu_read_lock__load(skel); if (!ASSERT_OK(err, "skel_load")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c b/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c index d2c0542716a8..1737eba34323 100644 --- a/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c +++ b/tools/testing/selftests/bpf/prog_tests/refcounted_kptr.c @@ -57,6 +57,7 @@ void test_percpu_hash_refcounted_kptr_refcount_leak(void) .data_size_in = sizeof(pkt_v4), .repeat = 1, ); + LIBBPF_OPTS(bpf_test_run_opts, syscall_opts); cpu_nr = libbpf_num_possible_cpus(); if (!ASSERT_GT(cpu_nr, 0, "libbpf_num_possible_cpus")) @@ -87,8 +88,11 @@ void test_percpu_hash_refcounted_kptr_refcount_leak(void) if (!ASSERT_EQ(opts.retval, 2, "opts.retval")) goto out; - err = bpf_map__update_elem(map, &key, sizeof(key), values, values_sz, 0); - if (!ASSERT_OK(err, "bpf_map__update_elem")) + fd = bpf_program__fd(skel->progs.clear_percpu_hash_kptr); + err = bpf_prog_test_run_opts(fd, &syscall_opts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto out; + if (!ASSERT_EQ(syscall_opts.retval, 1, "syscall_opts.retval")) goto out; fd = bpf_program__fd(skel->progs.check_percpu_hash_refcount); diff --git a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c index cb8dd2f63296..7f170a69d1d8 100644 --- a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c +++ b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c @@ -478,6 +478,52 @@ static struct range range_refine_in_halves(enum num_t x_t, struct range x, } +static __always_inline u64 next_u32_block(u64 x) { return x + (1ULL << 32); } +static __always_inline u64 prev_u32_block(u64 x) { return x - (1ULL << 32); } + +/* Is v within the circular u64 range [base, base + len]? */ +static __always_inline bool u64_range_contains(u64 v, u64 base, u64 len) +{ + return v - base <= len; +} + +/* Is v within the circular u32 range [base, base + len]? */ +static __always_inline bool u32_range_contains(u32 v, u32 base, u32 len) +{ + return v - base <= len; +} + +static bool range64_range32_intersect(enum num_t a_t, + struct range a /* 64 */, + struct range b /* 32 */, + struct range *out /* 64 */) +{ + u64 b_len = (u32)(b.b - b.a); + u64 a_len = a.b - a.a; + u64 lo, hi; + + if (u32_range_contains((u32)a.a, (u32)b.a, b_len)) { + lo = a.a; + } else { + lo = swap_low32(a.a, (u32)b.a); + if (!u64_range_contains(lo, a.a, a_len)) + lo = next_u32_block(lo); + if (!u64_range_contains(lo, a.a, a_len)) + return false; + } + if (u32_range_contains(a.b, (u32)b.a, b_len)) { + hi = a.b; + } else { + hi = swap_low32(a.b, (u32)b.b); + if (!u64_range_contains(hi, a.a, a_len)) + hi = prev_u32_block(hi); + if (!u64_range_contains(hi, a.a, a_len)) + return false; + } + *out = range(a_t, lo, hi); + return true; +} + static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, struct range y) { struct range y_cast; @@ -500,23 +546,45 @@ static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, (s64)x.a >= S32_MIN && (s64)x.b <= S32_MAX) return range_intersection(x_t, x, y_cast); - /* the case when new range knowledge, *y*, is a 32-bit subregister - * range, while previous range knowledge, *x*, is a full register - * 64-bit range, needs special treatment to take into account upper 32 - * bits of full register range - */ + if (y_t == U32 && x_t == U64) { + u64 xmin_swap, xmax_swap, xmin_lower32, xmax_lower32; + + xmin_lower32 = x.a & 0xffffffff; + xmax_lower32 = x.b & 0xffffffff; + if (xmin_lower32 < y.a || xmin_lower32 > y.b) { + /* The 32 lower bits of the umin64 are outside the u32 + * range. Let's update umin64 to match the u32 range. + * We want to *increase* the umin64 to the *minimum* + * value that matches the u32 range. + */ + xmin_swap = swap_low32(x.a, y.a); + /* We should always only increase the minimum, so if + * the new value is lower than before, we need to + * increase the 32 upper bits by 1. + */ + if (xmin_swap < x.a) + xmin_swap += 0x100000000; + if (xmin_swap == x.b) + return range(x_t, x.b, x.b); + } else if (xmax_lower32 < y.a || xmax_lower32 > y.b) { + /* Same for the umax64, but we want to *decrease* + * umax64 to the *maximum* value that matches the u32 + * range. + */ + xmax_swap = swap_low32(x.b, y.b); + if (xmax_swap > x.b) + xmax_swap -= 0x100000000; + if (xmax_swap == x.a) + return range(x_t, x.a, x.a); + } + } + if (t_is_32(y_t) && !t_is_32(x_t)) { - struct range x_swap; + struct range x1; - /* some combinations of upper 32 bits and sign bit can lead to - * invalid ranges, in such cases it's easier to detect them - * after cast/swap than try to enumerate all the conditions - * under which transformation and knowledge transfer is valid - */ - x_swap = range(x_t, swap_low32(x.a, y_cast.a), swap_low32(x.b, y_cast.b)); - if (!is_valid_range(x_t, x_swap)) - return x; - return range_intersection(x_t, x, x_swap); + if (range64_range32_intersect(x_t, x, y, &x1)) + return x1; + return x; } /* otherwise, plain range cast and intersection works */ @@ -1217,7 +1285,23 @@ static int parse_range_cmp_log(const char *log_buf, struct case_spec spec, spec.compare_subregs ? "w0" : "r0", spec.compare_subregs ? "w" : "r", specs[i].reg_idx); - q = strstr(p, buf); + /* + * In the verifier log look for lines: + * 18: (bf) r0 = r6 ; R0=... R6=... + * Different verifier passes may print + * 18: (bf) r0 = r6 + * as well, but never followed by ';'. + */ + q = p; + while ((q = strstr(q, buf)) != NULL) { + const char *s = q + strlen(buf); + + while (*s == ' ' || *s == '\t') + s++; + if (*s == ';') + break; + q = s; + } if (!q) { *specs[i].state = (struct reg_state){.valid = false}; continue; @@ -1251,6 +1335,26 @@ static bool assert_range_eq(enum num_t t, struct range x, struct range y, return false; } +/* For a pair of signed/unsigned t1/t2 checks if r1/r2 intersect in two intervals. */ +static bool needs_two_arcs(enum num_t t1, struct range r1, + enum num_t t2, struct range r2) +{ + u64 lo = cast_t(t1, r2.a); + u64 hi = cast_t(t1, r2.b); + + /* does r2 wrap in t1's domain: [0, hi] ∪ [lo, MAX]? */ + return lo > hi && r1.a <= hi && r1.b >= lo; +} + +static bool reg_state_needs_two_arcs(struct reg_state *s) +{ + if (!s->valid) + return false; + + return needs_two_arcs(U64, s->r[U64], S64, s->r[S64]) || + needs_two_arcs(U32, s->r[U32], S32, s->r[S32]); +} + /* Validate that register states match, and print details if they don't */ static bool assert_reg_state_eq(struct reg_state *r, struct reg_state *e, const char *ctx) { @@ -1475,6 +1579,11 @@ static int verify_case_op(enum num_t init_t, enum num_t cond_t, !assert_reg_state_eq(&fr2, &fe2, "false_reg2") || !assert_reg_state_eq(&tr1, &te1, "true_reg1") || !assert_reg_state_eq(&tr2, &te2, "true_reg2")) { + if (reg_state_needs_two_arcs(&fe1) || reg_state_needs_two_arcs(&fe2) || + reg_state_needs_two_arcs(&te1) || reg_state_needs_two_arcs(&te2)) { + test__skip(); + return 0; + } failed = true; } @@ -2129,6 +2238,8 @@ static struct subtest_case crafted_cases[] = { {U64, S64, {0x7fffffff00000001ULL, 0xffffffff00000000ULL}, {0, 0}}, {U64, S64, {0, 0xffffffffULL}, {1, 1}}, {U64, S64, {0, 0xffffffffULL}, {0x7fffffff, 0x7fffffff}}, + {U64, S32, {0xfffffffe00000001, 0xffffffff00000000}, {S64_MIN, S64_MIN}}, + {U64, U32, {0xfffffffe00000000, U64_MAX - 1}, {U64_MAX, U64_MAX}}, {U64, U32, {0, 0x100000000}, {0, 0}}, {U64, U32, {0xfffffffe, 0x300000000}, {0x80000000, 0x80000000}}, diff --git a/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c b/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c index f0a8c828f8f1..7541f4966abc 100644 --- a/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c @@ -3,6 +3,8 @@ #include <test_progs.h> #include <network_helpers.h> #include <sys/sysinfo.h> +#include <sys/syscall.h> +#include <linux/perf_event.h> #include "res_spin_lock.skel.h" #include "res_spin_lock_fail.skel.h" @@ -102,11 +104,29 @@ end: void serial_test_res_spin_lock_stress(void) { + struct perf_event_attr attr = { + .size = sizeof(attr), + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + }; + int pmu_fd; + if (libbpf_num_possible_cpus() < 3) { test__skip(); return; } + pmu_fd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, 0); + if (pmu_fd < 0) { + if (errno == ENOENT || errno == EOPNOTSUPP) { + test__skip(); + return; + } + ASSERT_OK(-errno, "perf_event_open pmu probe"); + return; + } + close(pmu_fd); + ASSERT_OK(load_module("bpf_test_rqspinlock.ko", false), "load module AA"); sleep(5); unload_module("bpf_test_rqspinlock", false); diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c index 41dfaaabb73f..3f9949e8227d 100644 --- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c +++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c @@ -10,7 +10,25 @@ #include <linux/btf_ids.h> #include "test_progs.h" -static int duration; +#define BTF_DATA_FILE "resolve_btfids.test.o.BTF" + +#define DECL_TAG_FASTCALL "bpf_fastcall" +#define DECL_TAG_KFUNC "bpf_kfunc" +#define TYPE_ATTR_ARENA "address_space(1)" +#define ARENA_ARG(n) (1U << (n)) + +#ifndef KF_FASTCALL +#define KF_FASTCALL (1 << 12) +#endif +#ifndef KF_ARENA_RET +#define KF_ARENA_RET (1 << 13) +#endif +#ifndef KF_ARENA_ARG1 +#define KF_ARENA_ARG1 (1 << 14) +#endif +#ifndef KF_ARENA_ARG2 +#define KF_ARENA_ARG2 (1 << 15) +#endif struct symbol { const char *name; @@ -28,12 +46,50 @@ struct symbol test_symbols[] = { { "func", BTF_KIND_FUNC, -1 }, }; +struct kfunc_symbol { + const char *name; + s32 id; + u32 flags; + u32 arena_args; + bool arena_ret; +}; + +static struct kfunc_symbol kfunc_symbols[] = { + { "kfunc_a", -1, 0, 0, false }, + { "kfunc_b", -1, KF_FASTCALL, 0, false }, + { "kfunc_c", -1, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2, + ARENA_ARG(0) | ARENA_ARG(1), true }, + { "kfunc_d", -1, KF_ARENA_ARG2, ARENA_ARG(1), false }, + { "kfunc_e", -1, 0, ARENA_ARG(0) | ARENA_ARG(1) | ARENA_ARG(2) | + ARENA_ARG(3) | ARENA_ARG(4), false }, + { "kfunc_f", -1, 0, ARENA_ARG(1), false }, + { "kfunc_g", -1, KF_ARENA_RET, ARENA_ARG(0) | ARENA_ARG(1), true }, +}; + /* Align the .BTF_ids section to 4 bytes */ asm ( ".pushsection " BTF_IDS_SECTION " ,\"a\"; \n" ".balign 4, 0; \n" ".popsection; \n"); +/* + * test_list_local, test_set and test_kfunc_set are .local symbols placed + * in .BTF_ids by inline asm, and are read here directly by C name. To the + * compiler they are plain, default-visibility extern objects. + * + * When test_progs is linked as a position-independent executable (PIE), + * taking the address of such an extern is routed through the GOT. The + * GNU assembler on aarch64 unconditionally converts references to .local + * symbols into section + addend form (".BTF_ids + <offset>"), but a GOT + * slot cannot carry an addend (the AArch64 ELF spec mandates zero), so + * the linker resolves it to the .BTF_ids base. + * + * Mark them hidden so the compiler treats them as non-interposable and + * emits a direct, addend-preserving PC-relative access instead of a GOT + * load, in both PIE and non-PIE builds. test_list_global is .globl and + * not affected, so it is left at default visibility. + */ +#pragma GCC visibility push(hidden) BTF_ID_LIST(test_list_local) BTF_ID_UNUSED BTF_ID(typedef, S) @@ -43,24 +99,49 @@ BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) -extern __u32 test_list_global[]; -BTF_ID_LIST_GLOBAL(test_list_global, 1) -BTF_ID_UNUSED +BTF_SET_START(test_set) BTF_ID(typedef, S) BTF_ID(typedef, T) BTF_ID(typedef, U) BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) +BTF_SET_END(test_set) -BTF_SET_START(test_set) +BTF_KFUNCS_START(test_kfunc_set) +BTF_ID_FLAGS(func, kfunc_a) +BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_e) +BTF_ID_FLAGS(func, kfunc_f) +BTF_ID_FLAGS(func, kfunc_g, KF_ARENA_RET) +BTF_KFUNCS_END(test_kfunc_set) + +/* + * Same kfuncs in reverse declaration order, so resolve_btfids has to + * actually sort at least one of the two sets. + */ +BTF_KFUNCS_START(test_kfunc_set_rev) +BTF_ID_FLAGS(func, kfunc_g, KF_ARENA_RET) +BTF_ID_FLAGS(func, kfunc_f) +BTF_ID_FLAGS(func, kfunc_e) +BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_ID_FLAGS(func, kfunc_a) +BTF_KFUNCS_END(test_kfunc_set_rev) +#pragma GCC visibility pop + +extern __u32 test_list_global[]; +BTF_ID_LIST_GLOBAL(test_list_global, 1) +BTF_ID_UNUSED BTF_ID(typedef, S) BTF_ID(typedef, T) BTF_ID(typedef, U) BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) -BTF_SET_END(test_set) static int __resolve_symbol(struct btf *btf, int type_id) @@ -70,10 +151,10 @@ __resolve_symbol(struct btf *btf, int type_id) unsigned int i; type = btf__type_by_id(btf, type_id); - if (!type) { - PRINT_FAIL("Failed to get type for ID %d\n", type_id); + if (!ASSERT_OK_PTR(type, "btf__type_by_id")) return -1; - } + + str = btf__name_by_offset(btf, type->name_off); for (i = 0; i < ARRAY_SIZE(test_symbols); i++) { if (test_symbols[i].id >= 0) @@ -82,64 +163,118 @@ __resolve_symbol(struct btf *btf, int type_id) if (BTF_INFO_KIND(type->info) != test_symbols[i].type) continue; - str = btf__name_by_offset(btf, type->name_off); - if (!str) { - PRINT_FAIL("Failed to get name for BTF ID %d\n", type_id); - return -1; - } - if (!strcmp(str, test_symbols[i].name)) test_symbols[i].id = type_id; } + if (!btf_is_func(type)) + return 0; + + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + if (kfunc_symbols[i].id >= 0) + continue; + if (!strcmp(str, kfunc_symbols[i].name)) + kfunc_symbols[i].id = type_id; + } + return 0; } -static int resolve_symbols(void) +static int resolve_symbols(struct btf *btf) { - struct btf *btf; + __u32 nr = btf__type_cnt(btf); int type_id; - __u32 nr; - btf = btf__parse_raw("resolve_btfids.test.o.BTF"); - if (CHECK(libbpf_get_error(btf), "resolve", - "Failed to load BTF from resolve_btfids.test.o.BTF\n")) - return -1; + for (type_id = 1; type_id < nr; type_id++) { + if (__resolve_symbol(btf, type_id)) + return -1; + } + return 0; +} + +static bool btf_has_decl_tag(struct btf *btf, const char *tag_name, s32 target_id) +{ + const struct btf_type *t; + const char *name; + int nr, id; nr = btf__type_cnt(btf); + for (id = 1; id < nr; id++) { + t = btf__type_by_id(btf, id); + if (!btf_is_decl_tag(t)) + continue; + if (t->type != (__u32)target_id) + continue; + if (btf_decl_tag(t)->component_idx != -1) + continue; + name = btf__name_by_offset(btf, t->name_off); + if (strcmp(name, tag_name) == 0) + return true; + } + return false; +} - for (type_id = 1; type_id < nr; type_id++) { - if (__resolve_symbol(btf, type_id)) - break; +static void check_kfunc_set(struct btf_id_set8 *set) +{ + unsigned int i, j; + + ASSERT_EQ(set->flags, BTF_SET8_KFUNCS, "kfunc_set_flags"); + ASSERT_EQ(set->cnt, ARRAY_SIZE(kfunc_symbols), "kfunc_set_cnt"); + + for (i = 0; i < set->cnt; i++) { + for (j = 0; j < ARRAY_SIZE(kfunc_symbols); j++) { + if (kfunc_symbols[j].id == (s32)set->pairs[i].id) { + ASSERT_EQ(set->pairs[i].flags, + kfunc_symbols[j].flags, "kfunc_flags_check"); + break; + } + } + + ASSERT_TRUE(j < ARRAY_SIZE(kfunc_symbols), "kfunc_id_found"); + + if (i > 0) { + ASSERT_LE(set->pairs[i - 1].id, + set->pairs[i].id, "kfunc_sort_check"); + } } +} - btf__free(btf); - return 0; +/* True if @id is PTR -> TYPE_TAG(kflag=1, "address_space(1)") -> pointee */ +static bool is_arena_tagged_ptr(struct btf *btf, __u32 id) +{ + const struct btf_type *ptr, *tag; + const char *name; + + ptr = btf__type_by_id(btf, id); + if (!btf_is_ptr(ptr)) + return false; + tag = btf__type_by_id(btf, ptr->type); + if (!btf_is_type_tag(tag) || !btf_kflag(tag)) + return false; + name = btf__name_by_offset(btf, tag->name_off); + return strcmp(name, TYPE_ATTR_ARENA) == 0; } void test_resolve_btfids(void) { __u32 *test_list, *test_lists[] = { test_list_local, test_list_global }; unsigned int i, j; - int ret = 0; + struct btf *btf; - if (resolve_symbols()) + btf = btf__parse_raw(BTF_DATA_FILE); + if (!ASSERT_OK_PTR(btf, "btf_parse")) return; + if (resolve_symbols(btf)) + goto out; + /* Check BTF_ID_LIST(test_list_local) and * BTF_ID_LIST_GLOBAL(test_list_global) IDs */ for (j = 0; j < ARRAY_SIZE(test_lists); j++) { test_list = test_lists[j]; - for (i = 0; i < ARRAY_SIZE(test_symbols); i++) { - ret = CHECK(test_list[i] != test_symbols[i].id, - "id_check", - "wrong ID for %s (%d != %d)\n", - test_symbols[i].name, - test_list[i], test_symbols[i].id); - if (ret) - return; - } + for (i = 0; i < ARRAY_SIZE(test_symbols); i++) + ASSERT_EQ(test_list[i], test_symbols[i].id, test_symbols[i].name); } /* Check BTF_SET_START(test_set) IDs */ @@ -153,15 +288,60 @@ void test_resolve_btfids(void) break; } - ret = CHECK(!found, "id_check", - "ID %d not found in test_symbols\n", - test_set.ids[i]); - if (ret) + if (!ASSERT_TRUE(found, "id_in_test_symbols")) break; - if (i > 0) { - if (!ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check")) - return; + if (i > 0) + ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check"); + } + + check_kfunc_set(&test_kfunc_set); + check_kfunc_set(&test_kfunc_set_rev); + + /* Check resolve_btfids emitted a bpf_kfunc decl_tag for each kfunc */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + ASSERT_TRUE(btf_has_decl_tag(btf, DECL_TAG_KFUNC, + kfunc_symbols[i].id), + kfunc_symbols[i].name); + } + + /* Check resolve_btfids emitted bpf_fastcall for KF_FASTCALL kfuncs */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + if (kfunc_symbols[i].flags & KF_FASTCALL) { + ASSERT_TRUE(btf_has_decl_tag(btf, DECL_TAG_FASTCALL, + kfunc_symbols[i].id), + kfunc_symbols[i].name); } } + + /* + * Check resolve_btfids wrapped exactly the arena-flagged or suffixed + * return/args with the address_space(1) type attribute, and left other + * pointers/returns untouched. + */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + const struct btf_type *fn, *proto; + const struct btf_param *params; + const char *name = kfunc_symbols[i].name; + u32 arena_args = kfunc_symbols[i].arena_args; + __u32 nr; + + fn = btf__type_by_id(btf, kfunc_symbols[i].id); + if (!ASSERT_TRUE(btf_is_func(fn), name)) + continue; + proto = btf__type_by_id(btf, fn->type); + if (!ASSERT_TRUE(btf_is_func_proto(proto), name)) + continue; + params = btf_params(proto); + nr = btf_vlen(proto); + + ASSERT_EQ(is_arena_tagged_ptr(btf, proto->type), + kfunc_symbols[i].arena_ret, name); + for (j = 0; j < nr; j++) + ASSERT_EQ(is_arena_tagged_ptr(btf, params[j].type), + !!(arena_args & ARENA_ARG(j)), name); + } + +out: + btf__free(btf); } diff --git a/tools/testing/selftests/bpf/prog_tests/rhash.c b/tools/testing/selftests/bpf/prog_tests/rhash.c new file mode 100644 index 000000000000..0641bd5b0a9e --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/rhash.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> +#include <string.h> +#include <stdio.h> +#include "rhash.skel.h" +#include "bpf_iter_bpf_rhash_map.skel.h" +#include <linux/bpf.h> +#include <linux/perf_event.h> +#include <sys/syscall.h> + +static void rhash_run(const char *prog_name) +{ + struct rhash *skel; + struct bpf_program *prog; + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + skel = rhash__open(); + if (!ASSERT_OK_PTR(skel, "rhash__open")) + return; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name")) + goto cleanup; + bpf_program__set_autoload(prog, true); + + err = rhash__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(err, "prog run")) + goto cleanup; + + if (!ASSERT_OK(opts.retval, "prog retval")) + goto cleanup; + + if (!ASSERT_OK(skel->bss->err, "bss->err")) + goto cleanup; + +cleanup: + rhash__destroy(skel); +} + +static int rhash_map_create(__u32 max_entries, __u64 map_extra) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_NO_PREALLOC, + .map_extra = map_extra); + + return bpf_map_create(BPF_MAP_TYPE_RHASH, "rhash_extra", + sizeof(__u32), sizeof(__u64), max_entries, &opts); +} + +static void rhash_map_extra_presize(void) +{ + const __u32 max_entries = 1024; + const __u32 nelem_hint = 256; + struct bpf_map_info info = {}; + __u32 info_len = sizeof(info); + __u64 val = 0; + __u32 key; + int fd, i; + + fd = rhash_map_create(max_entries, nelem_hint); + if (!ASSERT_GE(fd, 0, "rhash_map_create presize")) + return; + + if (!ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &info_len), "info")) + goto close; + ASSERT_EQ(info.map_extra, nelem_hint, "info.map_extra"); + + for (i = 0; i < (int)nelem_hint; i++) { + key = i; + if (!ASSERT_OK(bpf_map_update_elem(fd, &key, &val, BPF_NOEXIST), + "update")) + goto close; + } +close: + close(fd); +} + +static void rhash_map_extra_too_big(void) +{ + int fd; + + fd = rhash_map_create(1U << 20, 0x10000); + if (!ASSERT_LT(fd, 0, "rhash_map_create hint > U16_MAX")) + close(fd); +} + +static void rhash_iter_test(void) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + struct bpf_iter_bpf_rhash_map *skel; + int err, i, len, map_fd, iter_fd; + union bpf_iter_link_info linfo; + u32 expected_key_sum = 0, key; + struct bpf_link *link; + u64 val = 0; + char buf[64]; + + skel = bpf_iter_bpf_rhash_map__open(); + if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_rhash_map__open")) + return; + + err = bpf_iter_bpf_rhash_map__load(skel); + if (!ASSERT_OK(err, "bpf_iter_bpf_rhash_map__load")) + goto out; + + map_fd = bpf_map__fd(skel->maps.rhashmap); + + /* Populate map with test data */ + for (i = 0; i < 64; i++) { + key = i + 1; + expected_key_sum += key; + + err = bpf_map_update_elem(map_fd, &key, &val, BPF_NOEXIST); + if (!ASSERT_OK(err, "map_update")) + goto out; + } + + memset(&linfo, 0, sizeof(linfo)); + linfo.map.map_fd = map_fd; + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(skel->progs.dump_bpf_rhash_map, &opts); + if (!ASSERT_OK_PTR(link, "attach_iter")) + goto out; + + iter_fd = bpf_iter_create(bpf_link__fd(link)); + if (!ASSERT_GE(iter_fd, 0, "create_iter")) + goto free_link; + + do { + len = read(iter_fd, buf, sizeof(buf)); + } while (len > 0); + + ASSERT_EQ(skel->bss->key_sum, expected_key_sum, "key_sum"); + ASSERT_EQ(skel->bss->elem_count, 64, "elem_count"); + + close(iter_fd); + +free_link: + bpf_link__destroy(link); +out: + bpf_iter_bpf_rhash_map__destroy(skel); +} + +void test_rhash(void) +{ + if (test__start_subtest("test_rhash_lookup_update")) + rhash_run("test_rhash_lookup_update"); + + if (test__start_subtest("test_rhash_update_delete")) + rhash_run("test_rhash_update_delete"); + + if (test__start_subtest("test_rhash_update_elements")) + rhash_run("test_rhash_update_elements"); + + if (test__start_subtest("test_rhash_update_exist")) + rhash_run("test_rhash_update_exist"); + + if (test__start_subtest("test_rhash_update_any")) + rhash_run("test_rhash_update_any"); + + if (test__start_subtest("test_rhash_noexist_duplicate")) + rhash_run("test_rhash_noexist_duplicate"); + + if (test__start_subtest("test_rhash_delete_nonexistent")) + rhash_run("test_rhash_delete_nonexistent"); + + if (test__start_subtest("test_rhash_kptr_update")) + rhash_run("test_rhash_kptr_update"); + + if (test__start_subtest("test_rhash_kptr_delete")) + rhash_run("test_rhash_kptr_delete"); + + if (test__start_subtest("test_rhash_map_extra_presize")) + rhash_map_extra_presize(); + + if (test__start_subtest("test_rhash_map_extra_too_big")) + rhash_map_extra_too_big(); + + if (test__start_subtest("test_rhash_iter")) + rhash_iter_test(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/rhash_timer.c b/tools/testing/selftests/bpf/prog_tests/rhash_timer.c new file mode 100644 index 000000000000..3aad9fc02e06 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/rhash_timer.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include <sched.h> + +#include <test_progs.h> +#include "rhash_timer.skel.h" + +#define MAX_ATTEMPTS 256 +#define RCU_SYNC_INTERVAL 64 + +static int pin_to_first_cpu(cpu_set_t *old_mask) +{ + cpu_set_t new_mask; + int cpu; + + if (sched_getaffinity(0, sizeof(*old_mask), old_mask)) + return -errno; + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) + if (CPU_ISSET(cpu, old_mask)) + break; + if (cpu == CPU_SETSIZE) + return -EINVAL; + + CPU_ZERO(&new_mask); + CPU_SET(cpu, &new_mask); + if (sched_setaffinity(0, sizeof(new_mask), &new_mask)) + return -errno; + return 0; +} + +static int update_timer_map(int map_fd, __u64 key) +{ + __u64 value[3] = {}; + + return bpf_map_update_elem(map_fd, &key, value, BPF_NOEXIST); +} + +static int run_prog(int prog_fd, struct bpf_test_run_opts *opts) +{ + int err; + + err = bpf_prog_test_run_opts(prog_fd, opts); + if (err) + return err; + return opts->retval; +} + +void test_rhash_timer(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct rhash_timer *skel = NULL; + cpu_set_t old_mask; + int map_fd = -1, arm_fd, cancel_fd; + bool affinity_set = false; + __u64 key = 1; + int attempt, err; + + err = pin_to_first_cpu(&old_mask); + if (!ASSERT_OK(err, "pin_to_first_cpu")) + return; + affinity_set = true; + + skel = rhash_timer__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto out; + + map_fd = bpf_map__fd(skel->maps.timer_map); + if (!ASSERT_GE(map_fd, 0, "timer_map fd")) + goto out; + arm_fd = bpf_program__fd(skel->progs.arm_deleted_timer); + if (!ASSERT_GE(arm_fd, 0, "arm_deleted_timer fd")) + goto out; + cancel_fd = bpf_program__fd(skel->progs.cancel_recycled_timer); + if (!ASSERT_GE(cancel_fd, 0, "cancel_recycled_timer fd")) + goto out; + + err = update_timer_map(map_fd, key); + if (!ASSERT_OK(err, "seed_timer_map")) + goto out; + + for (attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + err = run_prog(arm_fd, &opts); + if (err) { + ASSERT_OK(err, "arm_deleted_timer"); + goto out; + } + if (skel->bss->armed != attempt + 1) { + ASSERT_EQ(skel->bss->armed, attempt + 1, "armed"); + goto out; + } + if (skel->bss->timer_init_err) { + ASSERT_OK(skel->bss->timer_init_err, "timer_init_err"); + goto out; + } + if (skel->bss->timer_set_callback_err) { + ASSERT_OK(skel->bss->timer_set_callback_err, + "timer_set_callback_err"); + goto out; + } + if (skel->bss->timer_start_err) { + ASSERT_OK(skel->bss->timer_start_err, "timer_start_err"); + goto out; + } + + if ((attempt + 1) % RCU_SYNC_INTERVAL == 0) { + err = kern_sync_rcu(); + if (err) { + ASSERT_OK(err, "kern_sync_rcu"); + goto out; + } + } + + err = update_timer_map(map_fd, ++key); + if (err) { + ASSERT_OK(err, "replace_timer_map"); + goto out; + } + + err = run_prog(cancel_fd, &opts); + if (err) { + ASSERT_OK(err, "cancel_recycled_timer"); + goto out; + } + if (skel->bss->timer_cancel_err) { + ASSERT_OK(skel->bss->timer_cancel_err, "timer_cancel_err"); + goto out; + } + if (skel->bss->cancelled) + break; + } + + ASSERT_GT(skel->bss->cancelled, 0, "preserved timer"); +out: + if (map_fd >= 0) + bpf_map_delete_elem(map_fd, &key); + rhash_timer__destroy(skel); + if (affinity_set) + sched_setaffinity(0, sizeof(old_mask), &old_mask); +} diff --git a/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c b/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c index 77fe1bfb7504..4e91d9b615ce 100644 --- a/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c @@ -199,6 +199,83 @@ err_out: bpf_link__destroy(getsockopt_link); } +static int connect_to_v4mapped_v6_fd(int server_fd) +{ + struct sockaddr_storage addr; + struct sockaddr_in *addr4 = (void *)&addr; + socklen_t addrlen = sizeof(addr); + struct sockaddr_in6 addr6 = {}; + int fd = -1, v6only = 0, err; + + err = getsockname(server_fd, (struct sockaddr *)&addr, &addrlen); + if (!ASSERT_OK(err, "getsockname")) + return -1; + + fd = socket(AF_INET6, SOCK_STREAM, 0); + if (!ASSERT_GE(fd, 0, "socket")) + return -1; + + err = settimeo(fd, 0); + if (!ASSERT_OK(err, "settimeo")) + goto err_out; + + err = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &v6only, sizeof(v6only)); + if (!ASSERT_OK(err, "clear_v6only")) + goto err_out; + + addr6.sin6_family = AF_INET6; + addr6.sin6_port = addr4->sin_port; + addr6.sin6_addr.s6_addr[10] = 0xff; + addr6.sin6_addr.s6_addr[11] = 0xff; + memcpy(&addr6.sin6_addr.s6_addr[12], &addr4->sin_addr, sizeof(addr4->sin_addr)); + + err = connect(fd, (struct sockaddr *)&addr6, sizeof(addr6)); + if (!ASSERT_OK(err, "connect")) + goto err_out; + + return fd; + +err_out: + close(fd); + return -1; +} + +static void test_v4mapped_v6_ip_tos(void) +{ + struct setget_sockopt__bss *bss = skel->bss; + int sfd = -1, fd = -1, got = 0, exp = 0x1c; + socklen_t optlen; + + memset(bss, 0, sizeof(*bss)); + bss->v4mapped_v6_ip_tos_enable = 1; + bss->v4mapped_v6_ip_tos_ret = -1; + bss->v4mapped_v6_ip_tos_val = exp; + + sfd = start_server(AF_INET, SOCK_STREAM, addr4_str, 0, 0); + if (!ASSERT_GE(sfd, 0, "start_server")) + goto err_out; + + fd = connect_to_v4mapped_v6_fd(sfd); + if (!ASSERT_GE(fd, 0, "connect_to_v4mapped_v6_fd")) + goto err_out; + + ASSERT_GT(bss->v4mapped_v6_ip_tos_cnt, 0, "v4mapped_v6_ip_tos_cnt"); + ASSERT_EQ(bss->v4mapped_v6_ip_tos_ret, 0, "v4mapped_v6_ip_tos_ret"); + + optlen = sizeof(got); + if (!ASSERT_OK(getsockopt(fd, SOL_IP, IP_TOS, &got, &optlen), "getsockopt_ip_tos")) + goto err_out; + + ASSERT_EQ(got, exp, "ip_tos"); + +err_out: + bss->v4mapped_v6_ip_tos_enable = 0; + if (fd >= 0) + close(fd); + if (sfd >= 0) + close(sfd); +} + void test_setget_sockopt(void) { cg_fd = test__join_cgroup(CG_NAME); @@ -238,6 +315,7 @@ void test_setget_sockopt(void) test_ktls(AF_INET); test_nonstandard_opt(AF_INET); test_nonstandard_opt(AF_INET6); + test_v4mapped_v6_ip_tos(); done: setget_sockopt__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/sha256.c b/tools/testing/selftests/bpf/prog_tests/sha256.c index 604a0b1423d5..5edbc6194b07 100644 --- a/tools/testing/selftests/bpf/prog_tests/sha256.c +++ b/tools/testing/selftests/bpf/prog_tests/sha256.c @@ -25,10 +25,10 @@ void test_sha256(void) size_t i; data = malloc(MAX_LEN); - if (!ASSERT_OK_PTR(data, "malloc")) + if (!ASSERT_NEQ(data, NULL, "malloc")) goto out; digests = malloc((MAX_LEN + 1) * SHA256_DIGEST_LENGTH); - if (!ASSERT_OK_PTR(digests, "malloc")) + if (!ASSERT_NEQ(digests, NULL, "malloc")) goto out; /* Generate MAX_LEN bytes of "random" data deterministically. */ diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c new file mode 100644 index 000000000000..77381d345435 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -0,0 +1,1840 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <test_progs.h> +#include <sys/syscall.h> +#include <sys/mman.h> +#include <sys/wait.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <limits.h> +#include <linux/keyctl.h> +#include <linux/bpf.h> + +#include <bpf/btf.h> + +#include "bpf/libbpf_internal.h" /* for libbpf_sha256() */ +#include "bpf/skel_internal.h" /* for loader ctx layout (bpf_loader_ctx etc) */ + +#include "test_signed_loader.skel.h" +#include "test_signed_loader_map.skel.h" +#include "test_signed_loader_data.skel.h" +#include "test_signed_loader_lsm.skel.h" + +enum { + BPF_SIG_UNSIGNED = 0, + BPF_SIG_VERIFIED, +}; + +enum { + BPF_SIG_KEYRING_NONE = 0, + BPF_SIG_KEYRING_BUILTIN, + BPF_SIG_KEYRING_SECONDARY, + BPF_SIG_KEYRING_PLATFORM, + BPF_SIG_KEYRING_USER, +}; + +static int load_loader(const void *insns, __u32 insns_sz, int map_fd, + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt) +{ + union bpf_attr attr; + int fd; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + if (sig) { + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = keyring_id; + } + attr.fd_array_cnt = fd_array_cnt; + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + return fd < 0 ? -errno : fd; +} + +static int run_gen_loader(const void *insns, __u32 insns_sz, + const void *data, __u32 data_sz, + const void *excl, __u32 excl_sz, + const void *sig, __u32 sig_sz, + void *ctx, __u32 ctx_sz, bool *loader_ran) +{ + LIBBPF_OPTS(bpf_map_create_opts, mopts, + .excl_prog_hash = excl, + .excl_prog_hash_size = excl_sz); + __u32 key = 0; + union bpf_attr attr; + int map_fd, prog_fd, ret; + + *loader_ran = false; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", + 4, data_sz, 1, &mopts); + if (map_fd < 0) + return -errno; + if (bpf_map_update_elem(map_fd, &key, data, 0)) { + ret = -errno; + goto out_map; + } + if (bpf_map_freeze(map_fd)) { + ret = -errno; + goto out_map; + } + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + if (sig) { + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.fd_array_cnt = 1; + } + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + if (prog_fd < 0) { + ret = -errno; + goto out_map; + } + + memset(&attr, 0, sizeof(attr)); + attr.test.prog_fd = prog_fd; + attr.test.ctx_in = ptr_to_u64(ctx); + attr.test.ctx_size_in = ctx_sz; + if (syscall(__NR_bpf, BPF_PROG_RUN, &attr, + offsetofend(union bpf_attr, test)) < 0) { + ret = -errno; + goto out_prog; + } + *loader_ran = true; + ret = (int)attr.test.retval; +out_prog: + close(prog_fd); +out_map: + close(map_fd); + return ret; +} + +static void close_loader_ctx_fds(void *ctx, int nr_maps, int nr_progs) +{ + struct bpf_map_desc *md = (struct bpf_map_desc *)((char *)ctx + + sizeof(struct bpf_loader_ctx)); + struct bpf_prog_desc *pd = (struct bpf_prog_desc *)(md + nr_maps); + int i; + + for (i = 0; i < nr_maps; i++) + if (md[i].map_fd > 0) + close(md[i].map_fd); + for (i = 0; i < nr_progs; i++) + if (pd[i].prog_fd > 0) + close(pd[i].prog_fd); +} + +static int run_setup(const char *cmd, const char *dir) +{ + int pid, status; + + pid = fork(); + if (pid < 0) + return -errno; + if (pid == 0) { + execlp("./verify_sig_setup.sh", "./verify_sig_setup.sh", + cmd, dir, NULL); + exit(1); + } + if (waitpid(pid, &status, 0) < 0) + return -errno; + return (WIFEXITED(status) && + WEXITSTATUS(status) == 0) ? 0 : -EINVAL; +} + +static int sign_buf(const char *dir, const void *buf, __u32 len, + void *sig, __u32 *sig_sz) +{ + char data_tmpl[PATH_MAX], key[PATH_MAX]; + char sigpath[PATH_MAX + sizeof(".p7s")]; + int fd, pid, status, ret; + struct stat st; + + ret = snprintf(data_tmpl, sizeof(data_tmpl), "%s/dataXXXXXX", dir); + if (ret < 0 || ret >= (int)sizeof(data_tmpl)) + return -ENAMETOOLONG; + ret = 0; + + fd = mkstemp(data_tmpl); + if (fd < 0) + return -errno; + if (write(fd, buf, len) != (ssize_t)len) { + close(fd); + ret = -EIO; + goto out; + } + close(fd); + + pid = fork(); + if (pid < 0) { + ret = -errno; + goto out; + } + if (pid == 0) { + snprintf(key, sizeof(key), "%s/signing_key.pem", dir); + execlp("./sign-file", "./sign-file", "-d", "sha256", + key, key, data_tmpl, NULL); + exit(1); + } + if (waitpid(pid, &status, 0) < 0 || + !WIFEXITED(status) || WEXITSTATUS(status)) { + ret = -EINVAL; + goto out; + } + + snprintf(sigpath, sizeof(sigpath), "%s.p7s", data_tmpl); + if (stat(sigpath, &st) < 0) { + ret = -errno; + goto out; + } + if (st.st_size > (off_t)*sig_sz) { + ret = -E2BIG; + goto out_sig; + } + fd = open(sigpath, O_RDONLY); + if (fd < 0) { + ret = -errno; + goto out_sig; + } + if (read(fd, sig, st.st_size) != st.st_size) { + close(fd); + ret = -EIO; + goto out_sig; + } + close(fd); + *sig_sz = st.st_size; +out_sig: + unlink(sigpath); +out: + unlink(data_tmpl); + return ret; +} + +struct gen_loader_fixture { + struct test_signed_loader *skel; + struct gen_loader_opts gopts; + unsigned char *blob; + void *ctx; + __u32 data_sz; + __u32 ctx_sz; + int nr_maps; + int nr_progs; + __u8 excl[SHA256_DIGEST_LENGTH]; +}; + +static int gen_loader_fixture_init(struct gen_loader_fixture *f) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + int nr_maps = 0, nr_progs = 0; + struct bpf_program *p; + struct bpf_map *m; + + memset(f, 0, sizeof(*f)); + f->skel = test_signed_loader__open(); + if (!ASSERT_OK_PTR(f->skel, "skel_open")) + return -1; + if (!ASSERT_OK(bpf_object__gen_loader(f->skel->obj, &gopts), "gen_loader")) + return -1; + if (!ASSERT_OK(bpf_object__load(f->skel->obj), "gen_load")) + return -1; + f->gopts = gopts; + + bpf_object__for_each_program(p, f->skel->obj) + nr_progs++; + bpf_object__for_each_map(m, f->skel->obj) + nr_maps++; + f->nr_maps = nr_maps; + f->nr_progs = nr_progs; + f->ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + f->ctx = calloc(1, f->ctx_sz); + if (!ASSERT_OK_PTR(f->ctx, "ctx_alloc")) + return -1; + ((struct bpf_loader_ctx *)f->ctx)->sz = f->ctx_sz; + + f->data_sz = gopts.data_sz; + f->blob = malloc(f->data_sz); + if (!ASSERT_OK_PTR(f->blob, "blob_alloc")) + return -1; + memcpy(f->blob, gopts.data, f->data_sz); + + /* excl_prog_hash = SHA256(loader insns) == the loader's prog->digest. */ + libbpf_sha256(gopts.insns, gopts.insns_sz, f->excl); + return 0; +} + +static void gen_loader_fixture_fini(struct gen_loader_fixture *f) +{ + if (f->ctx) + close_loader_ctx_fds(f->ctx, f->nr_maps, f->nr_progs); + free(f->blob); + free(f->ctx); + test_signed_loader__destroy(f->skel); +} + +static void metadata_match(void) +{ + struct gen_loader_fixture f; + bool ran; + int r; + + if (gen_loader_fixture_init(&f) == 0) { + r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob, + f.data_sz, f.excl, sizeof(f.excl), NULL, 0, + f.ctx, f.ctx_sz, &ran); + ASSERT_TRUE(ran, "loader ran"); + ASSERT_EQ(r, 0, "honest loader retval"); + } + gen_loader_fixture_fini(&f); +} + +static void signature_enforced(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + struct gen_loader_fixture f; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * A present-but-invalid signature (the cert bytes are not a + * PKCS#7 signature) must be rejected at load: the signature + * path is honored, not ignored. (The valid path is covered by + * the signed lskels.) Pin -EBADMSG, the PKCS#7 parse failure: + * a looser fd < 0 check could also be satisfied by the sparse + * fd_array rejection (-EACCES) that the loader's map reference + * would trip even if the signature were silently ignored. + */ + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(fd, -EBADMSG, "invalid signature rejected at load"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +static void signed_nonexcl_fd_array_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + struct gen_loader_fixture f; + int map_fd, fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * A signed program may only bind exclusive maps through fd_array + * (their contents are folded into the signature). Binding a + * non-exclusive map is rejected, before the signature is even + * examined. + */ + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "nonexcl", 4, + f.data_sz, 1, NULL); + if (ASSERT_OK_FD(map_fd, "nonexcl_map")) { + if (ASSERT_OK(bpf_map_freeze(map_fd), "freeze")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, + map_fd, junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EPERM, + "non-exclusive map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + } + close(map_fd); + } + } + gen_loader_fixture_fini(&f); +} + +static void signed_unfrozen_fd_array_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + LIBBPF_OPTS(bpf_map_create_opts, mopts); + struct gen_loader_fixture f; + __u32 key = 0; + int map_fd, fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * The metadata map must be frozen before a signed load so the + * folded bytes cannot change afterwards. Bind an exclusive map + * with matching contents but skip the freeze: the load must be + * rejected by the frozen check with -EPERM. The exclusivity + * check right after it would pass, so the errno uniquely pins + * the freeze requirement. + */ + mopts.excl_prog_hash = f.excl; + mopts.excl_prog_hash_size = sizeof(f.excl); + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "unfrozen", 4, + f.data_sz, 1, &mopts); + if (ASSERT_OK_FD(map_fd, "unfrozen_map")) { + if (ASSERT_OK(bpf_map_update_elem(map_fd, &key, f.blob, 0), + "update")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, + map_fd, junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EPERM, + "unfrozen map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + } + close(map_fd); + } + } + gen_loader_fixture_fini(&f); +} + +static void signed_nonarray_fd_array_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + LIBBPF_OPTS(bpf_map_create_opts, mopts); + struct gen_loader_fixture f; + int map_fd, fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * Only a plain BPF_MAP_TYPE_ARRAY may be folded into the + * signature. An exclusive map of any other type is rejected + * (-EINVAL) rather than folded - this is the type gate that + * keeps arena maps (map_direct_value_addr() returns a user + * address) and insn-array maps (buffer smaller than value_size) + * out of the hashed region, where the old code would have + * memcpy()'d from them. A hash map stands in here: it is + * exclusive (bound to the loader digest) but not an array. + */ + mopts.excl_prog_hash = f.excl; + mopts.excl_prog_hash_size = sizeof(f.excl); + map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, "excl_hash", 4, 4, 1, + &mopts); + if (ASSERT_OK_FD(map_fd, "excl_hash_map")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, + junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EINVAL, + "non-array map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + close(map_fd); + } + } + gen_loader_fixture_fini(&f); +} + +static int setup_meta_map(const struct gen_loader_fixture *f); + +static void signed_btf_fd_array_rejected(void) +{ + char dir_tmpl[] = "/tmp/signed_loader_btfXXXXXX", *dir = NULL; + __u32 sig_sz = 8192; + int map_fd = -1, prog_fd = -1; + unsigned char *buf = NULL; + struct gen_loader_fixture f; + bool have_fixture = false; + struct btf *btf = NULL; + union bpf_attr attr; + int fds[2]; + __u8 sig[8192]; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + /* + * fd_array binds maps and BTFs alike, but only exclusive array maps are + * folded into the signature. Build an otherwise genuinely signed load - + * insns || metadata, exclusive frozen map at fd_array[0] - then smuggle + * an extra BTF into fd_array[1]. A signed program may not bind any BTF, + * so resolving the fd_array entries rejects the BTF with -EACCES (in + * __add_used_btf(), before the signature is even verified). + */ + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, sig, + &sig_sz), "sign insns||metadata")) + goto out; + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + goto out; + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new_empty")) + goto out; + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto out; + + fds[0] = map_fd; + fds[1] = btf__fd(btf); + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(f.gopts.insns); + attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(fds); + attr.fd_array_cnt = 2; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + ASSERT_EQ(prog_fd < 0 ? -errno : prog_fd, -EACCES, + "BTF in signed fd_array rejected"); + if (prog_fd >= 0) + close(prog_fd); +out: + if (btf) + btf__free(btf); + if (map_fd >= 0) + close(map_fd); + if (have_fixture) + gen_loader_fixture_fini(&f); + if (dir) + run_setup("cleanup", dir); + free(buf); +} + +static void signature_failure_logs(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + char log_buf[1024] = {}; + struct gen_loader_fixture f; + union bpf_attr attr; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * Signature verification now runs inside bpf_check(), so a + * failure is reported through the verifier log. A present-but- + * invalid signature is rejected and the log says why. + */ + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(f.gopts.insns); + attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(junk); + attr.signature_size = sizeof(junk); + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = sizeof(log_buf); + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + + fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + ASSERT_LT(fd, 0, "invalid signature rejected at load"); + if (fd >= 0) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "signature verification failed", + "verifier logs signature failure"); + } + gen_loader_fixture_fini(&f); +} + +static void signature_too_large(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * signature_size beyond the kernel's bound (KMALLOC_MAX_CACHE_SIZE) + * is rejected before the buffer is read. + */ + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + 64 << 20, KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(fd, -EINVAL, "oversized signature rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +static void signature_zero_size(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * A present signature with signature_size == 0 is rejected + * up front, before the keyring is resolved or the signature + * buffer is read. + */ + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + 0, KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(fd, -EINVAL, "zero-size signature rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +static void signature_bad_keyring(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * A present signature with a keyring_id that resolves to no key is + * rejected up front: bpf_prog_verify_signature() fails the keyring + * lookup (-EINVAL) before it ever looks at the signature bytes. A + * large positive serial takes the user-keyring path and won't exist. + */ + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), INT_MAX, 0); + ASSERT_EQ(fd, -EINVAL, "signature with bad keyring_id rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +/* + * A signed loader must ignore ctx-supplied map dimensions: the host cannot + * resize a signed program's maps via the loader ctx. Drive a one-map program + * through gen_loader, ask (via ctx) for every map to be resized to a bogus + * value, and confirm the created maps keep their attested size. + */ +#define GATING_BOGUS_MAX 0x4000 + +static void metadata_ctx_max_entries_ignored(void) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + struct test_signed_loader_map *skel; + __u8 excl[SHA256_DIGEST_LENGTH]; + int nr_maps = 0, nr_progs = 0, i, checked = 0, r; + struct bpf_program *p; + struct bpf_map *m; + struct bpf_map_desc *md; + unsigned char *blob; + __u32 ctx_sz, data_sz; + void *ctx; + bool ran; + + skel = test_signed_loader_map__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader")) + goto destroy; + if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load")) + goto destroy; + + bpf_object__for_each_program(p, skel->obj) + nr_progs++; + bpf_object__for_each_map(m, skel->obj) + nr_maps++; + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + ctx = calloc(1, ctx_sz); + if (!ASSERT_OK_PTR(ctx, "ctx_alloc")) + goto destroy; + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + + md = (struct bpf_map_desc *)((char *)ctx + sizeof(struct bpf_loader_ctx)); + for (i = 0; i < nr_maps; i++) + md[i].max_entries = GATING_BOGUS_MAX; + + libbpf_sha256(gopts.insns, gopts.insns_sz, excl); + data_sz = gopts.data_sz; + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(blob, "blob_alloc")) + goto free_ctx; + memcpy(blob, gopts.data, data_sz); + + r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz, + excl, sizeof(excl), NULL, 0, ctx, ctx_sz, &ran); + if (!ASSERT_TRUE(ran, "loader ran") || + !ASSERT_EQ(r, 0, "loader retval")) + goto free_blob; + + for (i = 0; i < nr_maps; i++) { + struct bpf_map_info info; + __u32 ilen = sizeof(info); + int fd = md[i].map_fd; + + if (fd <= 0) + continue; + memset(&info, 0, sizeof(info)); + if (ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &ilen), "map_info")) { + ASSERT_NEQ(info.max_entries, GATING_BOGUS_MAX, + "ctx max_entries ignored for signed loader"); + checked++; + } + } + ASSERT_GT(checked, 0, "inspected a created map"); + +free_blob: + free(blob); +free_ctx: + close_loader_ctx_fds(ctx, nr_maps, nr_progs); + free(ctx); +destroy: + test_signed_loader_map__destroy(skel); +} + +/* + * A signed loader must also ignore ctx-supplied initial_value: the host cannot + * re-seed a signed program's map contents through the loader ctx. Drive a + * program with one initialized global (a .data map) through gen_loader, point + * every map's ctx initial_value at an adversarial buffer, and confirm the + * created map still holds the attested value, never the ctx bytes. + */ +#define DATA_MAGIC 0x5eed1234abad1deaULL + +static void metadata_ctx_initial_value_ignored(void) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + struct test_signed_loader_data *skel; + __u8 excl[SHA256_DIGEST_LENGTH], evil[64]; + int nr_maps = 0, nr_progs = 0, i, found = 0, r; + struct bpf_program *p; + struct bpf_map *m; + struct bpf_map_desc *md; + unsigned char *blob; + __u32 ctx_sz, data_sz; + void *ctx; + bool ran; + + skel = test_signed_loader_data__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader")) + goto destroy; + if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load")) + goto destroy; + + bpf_object__for_each_program(p, skel->obj) + nr_progs++; + bpf_object__for_each_map(m, skel->obj) + nr_maps++; + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + ctx = calloc(1, ctx_sz); + if (!ASSERT_OK_PTR(ctx, "ctx_alloc")) + goto destroy; + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + + memset(evil, 0xAA, sizeof(evil)); + md = (struct bpf_map_desc *)((char *)ctx + sizeof(struct bpf_loader_ctx)); + for (i = 0; i < nr_maps; i++) + md[i].initial_value = ptr_to_u64(evil); + + libbpf_sha256(gopts.insns, gopts.insns_sz, excl); + data_sz = gopts.data_sz; + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(blob, "blob_alloc")) + goto free_ctx; + memcpy(blob, gopts.data, data_sz); + + r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz, + excl, sizeof(excl), NULL, 0, ctx, ctx_sz, &ran); + if (!ASSERT_TRUE(ran, "loader ran") || + !ASSERT_EQ(r, 0, "loader retval")) + goto free_blob; + + for (i = 0; i < nr_maps; i++) { + struct bpf_map_info info; + __u32 ilen = sizeof(info), key = 0; + __u8 value[64] = {}; + __u64 got; + int fd = md[i].map_fd; + + if (fd <= 0) + continue; + memset(&info, 0, sizeof(info)); + if (!ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &ilen), "map_info")) + continue; + if (info.value_size <= sizeof(value) && + bpf_map_lookup_elem(fd, &key, value) == 0) { + memcpy(&got, value, sizeof(got)); + /* attested .data survives; ctx bytes (0xAA..) ignored */ + if (got == DATA_MAGIC) + found = 1; + ASSERT_NEQ(got, 0xAAAAAAAAAAAAAAAAULL, + "ctx initial_value ignored for signed loader"); + } + } + ASSERT_EQ(found, 1, "attested .data value preserved"); + +free_blob: + free(blob); +free_ctx: + close_loader_ctx_fds(ctx, nr_maps, nr_progs); + free(ctx); +destroy: + test_signed_loader_data__destroy(skel); +} + +/* + * The load-time signature must authenticate the loader instructions: a valid + * signature loads, and the very same signature over one-byte-tampered insns is + * rejected. Uses ./verify_sig_setup.sh + ./sign-file at runtime, like + * verify_pkcs7_sig, and verifies against the session keyring the key was added + * to. (signature_enforced/_too_large only cover a malformed signature.) + */ +static void signature_authenticates_insns(void) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + struct test_signed_loader *skel = NULL; + __u8 excl[SHA256_DIGEST_LENGTH], sig[8192]; + __u32 sig_sz = sizeof(sig), insns_sz, data_sz, ctx_sz; + unsigned char *insns = NULL, *tampered = NULL, *blob = NULL; + unsigned char *signbuf = NULL; + int nr_maps = 0, nr_progs = 0, r; + struct bpf_program *p; + struct bpf_map *m; + void *ctx = NULL; + bool ran; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + skel = test_signed_loader__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader")) + goto cleanup; + if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load")) + goto cleanup; + + bpf_object__for_each_program(p, skel->obj) + nr_progs++; + bpf_object__for_each_map(m, skel->obj) + nr_maps++; + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + insns_sz = gopts.insns_sz; + data_sz = gopts.data_sz; + ctx = calloc(1, ctx_sz); + insns = malloc(insns_sz); + tampered = malloc(insns_sz); + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(ctx, "ctx") || + !ASSERT_OK_PTR(insns, "insns") || + !ASSERT_OK_PTR(tampered, "tampered") || + !ASSERT_OK_PTR(blob, "blob")) + goto cleanup; + memcpy(insns, gopts.insns, insns_sz); + memcpy(blob, gopts.data, data_sz); + libbpf_sha256(insns, insns_sz, excl); + + signbuf = malloc((size_t)insns_sz + data_sz); + if (!ASSERT_OK_PTR(signbuf, "signbuf")) + goto cleanup; + memcpy(signbuf, insns, insns_sz); + memcpy(signbuf + insns_sz, blob, data_sz); + if (!ASSERT_OK(sign_buf(dir, signbuf, insns_sz + data_sz, sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_TRUE(ran, "valid signature: loader loaded and ran"); + ASSERT_EQ(r, 0, "valid signature accepted"); + close_loader_ctx_fds(ctx, nr_maps, nr_progs); + + memcpy(tampered, insns, insns_sz); + tampered[insns_sz / 2] ^= 0xff; + /* + * Bind the metadata map to the tampered loader's own digest, so the + * verifier's exclusive-map check (excl_prog_sha == prog->digest) passes + * and the signature - verified after the maps are resolved - is what + * rejects the load. This is the attacker's best case: even after + * re-binding the exclusive map to their tampered loader, the signature + * over the original insns || metadata still fails. (Leaving the map + * bound to the original digest would instead trip the excl check first.) + */ + libbpf_sha256(tampered, insns_sz, excl); + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(tampered, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_FALSE(ran, "tampered loader rejected before run"); + ASSERT_EQ(r, -EKEYREJECTED, "signature is bound to the instructions"); +cleanup: + free(insns); + free(tampered); + free(blob); + free(signbuf); + free(ctx); + test_signed_loader__destroy(skel); + run_setup("cleanup", dir); +} + +static void signature_authenticates_metadata(void) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + struct test_signed_loader *skel = NULL; + __u8 excl[SHA256_DIGEST_LENGTH], sig[8192]; + __u32 sig_sz = sizeof(sig), insns_sz, data_sz, ctx_sz; + unsigned char *insns = NULL, *blob = NULL; + unsigned char *signbuf = NULL; + int nr_maps = 0, nr_progs = 0, r; + struct bpf_program *p; + struct bpf_map *m; + void *ctx = NULL; + bool ran; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + skel = test_signed_loader__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader")) + goto cleanup; + if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load")) + goto cleanup; + + bpf_object__for_each_program(p, skel->obj) + nr_progs++; + bpf_object__for_each_map(m, skel->obj) + nr_maps++; + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + insns_sz = gopts.insns_sz; + data_sz = gopts.data_sz; + ctx = calloc(1, ctx_sz); + insns = malloc(insns_sz); + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(ctx, "ctx") || + !ASSERT_OK_PTR(insns, "insns") || + !ASSERT_OK_PTR(blob, "blob")) + goto cleanup; + memcpy(insns, gopts.insns, insns_sz); + memcpy(blob, gopts.data, data_sz); + libbpf_sha256(insns, insns_sz, excl); + + signbuf = malloc((size_t)insns_sz + data_sz); + if (!ASSERT_OK_PTR(signbuf, "signbuf")) + goto cleanup; + memcpy(signbuf, insns, insns_sz); + memcpy(signbuf + insns_sz, blob, data_sz); + if (!ASSERT_OK(sign_buf(dir, signbuf, insns_sz + data_sz, sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_TRUE(ran, "valid signature: loader loaded and ran"); + ASSERT_EQ(r, 0, "valid signature accepted"); + close_loader_ctx_fds(ctx, nr_maps, nr_progs); + + /* + * Tamper the metadata after signing while leaving the instructions + * and thus the exclusive hash binding untouched: the map freezes + * fine and excl_prog_sha still matches the loader's digest, so the + * load reaches signature verification, which folds the live frozen + * map bytes into the checked payload and must reject the modified + * blob. A kernel folding anything but the map contents themselves + * would wrongly accept this load. + */ + blob[data_sz / 2] ^= 0xff; + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_FALSE(ran, "tampered metadata rejected before run"); + ASSERT_EQ(r, -EKEYREJECTED, "signature is bound to the metadata"); +cleanup: + free(insns); + free(blob); + free(signbuf); + free(ctx); + test_signed_loader__destroy(skel); + run_setup("cleanup", dir); +} + +static int make_excl_map(__u32 flags, __u32 value_size) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + __u8 hash[SHA256_DIGEST_LENGTH] = { 1 }; /* any 32-byte value */ + + opts.excl_prog_hash = hash; + opts.excl_prog_hash_size = sizeof(hash); + opts.map_flags = flags; + return bpf_map_create(BPF_MAP_TYPE_ARRAY, "md", 4, value_size, 1, &opts); +} + +static void hash_requires_frozen(void) +{ + __u8 hbuf[SHA256_DIGEST_LENGTH], val[64] = {}; + struct bpf_map_info info; + __u32 ilen, key = 0; + int fd; + + fd = make_excl_map(0, sizeof(val)); + if (!ASSERT_OK_FD(fd, "excl_map")) + return; + ASSERT_OK(bpf_map_update_elem(fd, &key, val, 0), "update"); + + memset(&info, 0, sizeof(info)); + info.hash = ptr_to_u64(hbuf); + info.hash_size = sizeof(hbuf); + ilen = sizeof(info); + ASSERT_EQ(bpf_map_get_info_by_fd(fd, &info, &ilen), -EPERM, + "hash of unfrozen map rejected"); + close(fd); +} + +static void no_update_after_freeze(void) +{ + __u8 val[64] = {}; + __u32 key = 0; + int fd; + + fd = make_excl_map(0, sizeof(val)); + if (!ASSERT_OK_FD(fd, "excl_map")) + return; + ASSERT_OK(bpf_map_update_elem(fd, &key, val, 0), "update"); + ASSERT_OK(bpf_map_freeze(fd), "freeze"); + ASSERT_EQ(bpf_map_update_elem(fd, &key, val, 0), -EPERM, + "update after freeze rejected"); + close(fd); +} + +static void freeze_writable_mmap(void) +{ + void *w; + int fd; + + fd = make_excl_map(BPF_F_MMAPABLE, 4096); + if (!ASSERT_OK_FD(fd, "excl_mmapable_map")) + return; + w = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (ASSERT_OK_PTR(w, "writable_mmap")) { + ASSERT_EQ(bpf_map_freeze(fd), -EBUSY, + "freeze rejected while writable mmap held"); + munmap(w, 4096); + } + close(fd); +} + +static void no_writable_mmap_frozen(void) +{ + void *w; + int fd; + + fd = make_excl_map(BPF_F_MMAPABLE, 4096); + if (!ASSERT_OK_FD(fd, "excl_mmapable_map")) + return; + ASSERT_OK(bpf_map_freeze(fd), "freeze"); + w = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + ASSERT_EQ(w, MAP_FAILED, "writable mmap of frozen map rejected"); + if (w != MAP_FAILED) + munmap(w, 4096); + close(fd); +} + +static void map_hash_matches_libbpf(void) +{ + __u8 kbuf[SHA256_DIGEST_LENGTH], lbuf[SHA256_DIGEST_LENGTH], val[64] = {}; + struct bpf_map_info info; + __u32 ilen, key = 0; + int fd, i; + + /* + * The signing scheme assumes the kernel's map hash equals what libbpf + * computes over the same bytes (gen_loader bakes libbpf_sha256(blob); + * the kernel recomputes via array_map_get_hash). Pin that they agree. + */ + for (i = 0; i < (int)sizeof(val); i++) + val[i] = i * 7 + 1; + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "h", 4, sizeof(val), 1, NULL); + if (!ASSERT_OK_FD(fd, "array_map")) + return; + ASSERT_OK(bpf_map_update_elem(fd, &key, val, 0), "update"); + ASSERT_OK(bpf_map_freeze(fd), "freeze"); + memset(&info, 0, sizeof(info)); + info.hash = ptr_to_u64(kbuf); + info.hash_size = sizeof(kbuf); + ilen = sizeof(info); + if (ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &ilen), "get_hash")) { + libbpf_sha256(val, sizeof(val), lbuf); + ASSERT_EQ(memcmp(kbuf, lbuf, sizeof(kbuf)), 0, + "kernel map hash matches libbpf_sha256"); + } + close(fd); +} + +static void map_hash_multi_element(void) +{ + const __u32 nr = 8, value_size = 64; + __u8 kbuf[SHA256_DIGEST_LENGTH], lbuf[SHA256_DIGEST_LENGTH]; + struct bpf_map_info info; + __u32 ilen, i, j; + __u8 *full; + int fd; + + /* + * array_map_get_hash() hashes elem_size * max_entries (the whole value + * area), not just element 0. With an 8-aligned value_size elem_size has + * no padding, so pin that a >1-entry array's kernel hash equals + * libbpf_sha256() over the full, concatenated element contents. + */ + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "h", 4, value_size, nr, NULL); + if (!ASSERT_OK_FD(fd, "array_map")) + return; + full = calloc(nr, value_size); + if (!ASSERT_OK_PTR(full, "buf")) + goto close_fd; + for (i = 0; i < nr; i++) { + __u8 *v = full + i * value_size; + + for (j = 0; j < value_size; j++) + v[j] = i * 31 + j * 7 + 1; + ASSERT_OK(bpf_map_update_elem(fd, &i, v, 0), "update"); + } + ASSERT_OK(bpf_map_freeze(fd), "freeze"); + memset(&info, 0, sizeof(info)); + info.hash = ptr_to_u64(kbuf); + info.hash_size = sizeof(kbuf); + ilen = sizeof(info); + if (ASSERT_OK(bpf_map_get_info_by_fd(fd, &info, &ilen), "get_hash")) { + libbpf_sha256(full, (size_t)nr * value_size, lbuf); + ASSERT_EQ(memcmp(kbuf, lbuf, sizeof(kbuf)), 0, + "kernel hash covers full multi-element value area"); + } + free(full); +close_fd: + close(fd); +} + +static void map_hash_bad_size(void) +{ + __u8 kbuf[SHA256_DIGEST_LENGTH], val[64] = {}; + struct bpf_map_info info; + __u32 ilen, key = 0; + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "h", 4, sizeof(val), 1, NULL); + if (!ASSERT_OK_FD(fd, "array_map")) + return; + ASSERT_OK(bpf_map_update_elem(fd, &key, val, 0), "update"); + ASSERT_OK(bpf_map_freeze(fd), "freeze"); + memset(&info, 0, sizeof(info)); + info.hash = ptr_to_u64(kbuf); + info.hash_size = sizeof(kbuf) / 2; + ilen = sizeof(info); + ASSERT_EQ(bpf_map_get_info_by_fd(fd, &info, &ilen), -EINVAL, + "wrong hash_size rejected"); + close(fd); +} + +static void map_hash_unsupported_type(void) +{ + __u8 kbuf[SHA256_DIGEST_LENGTH]; + struct bpf_map_info info; + __u32 ilen; + int fd; + + /* Only arrays implement map_get_hash; a hash map must be refused. */ + fd = bpf_map_create(BPF_MAP_TYPE_HASH, "h", 4, 8, 4, NULL); + if (!ASSERT_OK_FD(fd, "hash_map")) + return; + memset(&info, 0, sizeof(info)); + info.hash = ptr_to_u64(kbuf); + info.hash_size = sizeof(kbuf); + ilen = sizeof(info); + ASSERT_EQ(bpf_map_get_info_by_fd(fd, &info, &ilen), -EINVAL, + "hash unsupported for non-array map"); + close(fd); +} + +static int setup_meta_map(const struct gen_loader_fixture *f) +{ + LIBBPF_OPTS(bpf_map_create_opts, mopts, + .excl_prog_hash = f->excl, + .excl_prog_hash_size = sizeof(f->excl)); + __u32 key = 0; + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, + f->data_sz, 1, &mopts); + if (fd < 0) + return -errno; + if (bpf_map_update_elem(fd, &key, f->blob, 0) || bpf_map_freeze(fd)) { + close(fd); + return -errno; + } + return fd; +} + +static void lsm_signature_verdict(void) +{ + char dir_tmpl[] = "/tmp/signed_loader_lsmXXXXXX", *dir = NULL; + struct test_signed_loader_lsm *lsm = NULL; + __u32 sig_sz = 8192, msig_sz = 8192; + int map_fd = -1, prog_fd = -1; + bool have_fixture = false; + struct gen_loader_fixture f; + unsigned char *buf; + __s32 ses_serial; + __u8 sig[8192]; + + lsm = test_signed_loader_lsm__open_and_load(); + if (!ASSERT_OK_PTR(lsm, "lsm_skel_load")) + return; + lsm->bss->monitored_tid = sys_gettid(); + if (!ASSERT_OK(test_signed_loader_lsm__attach(lsm), "lsm_attach")) + goto out; + + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map_unsigned")) + goto out; + lsm->bss->seen = 0; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, NULL, 0, 0, 0); + close(map_fd); + map_fd = -1; + if (!ASSERT_OK_FD(prog_fd, "unsigned loader load")) + goto out; + close(prog_fd); + prog_fd = -1; + if (!ASSERT_NEQ(lsm->bss->seen, 0, "bpf LSM in the active LSM set")) + goto out; + ASSERT_EQ(lsm->bss->seen, 1, "unsigned: one observed load"); + ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_UNSIGNED, "unsigned verdict"); + ASSERT_EQ(lsm->bss->sig_keyring_type, BPF_SIG_KEYRING_NONE, "unsigned keyring type"); + ASSERT_EQ(lsm->bss->sig_keyring_serial, 0, "unsigned: no keyring serial"); + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + goto out; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + dir = NULL; + goto out; + } + if (!ASSERT_OK(sign_buf(dir, f.gopts.insns, f.gopts.insns_sz, sig, + &sig_sz), "sign-file")) + goto out; + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map_signed")) + goto out; + lsm->bss->seen = 0; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, + sig_sz, KEY_SPEC_SESSION_KEYRING, 0); + close(map_fd); + map_fd = -1; + ASSERT_EQ(prog_fd, -EACCES, "unfolded metadata rejected"); + if (prog_fd >= 0) + close(prog_fd); + prog_fd = -1; + + ses_serial = syscall(__NR_keyctl, KEYCTL_GET_KEYRING_ID, + KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(lsm->bss->seen, 1, "signed: one observed load"); + ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, + "admission saw a valid signature"); + ASSERT_EQ(lsm->bss->sig_keyring_type, BPF_SIG_KEYRING_USER, "signed keyring type"); + ASSERT_GT(ses_serial, 0, "session keyring serial resolved"); + ASSERT_EQ(lsm->bss->sig_keyring_serial, ses_serial, + "signed: validated against session keyring"); + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "meta_signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, + sig, &msig_sz), "sign insns||metadata")) { + free(buf); + goto out; + } + free(buf); + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map_bound")) + goto out; + lsm->bss->seen = 0; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, + msig_sz, KEY_SPEC_SESSION_KEYRING, 1); + close(map_fd); + map_fd = -1; + if (!ASSERT_OK_FD(prog_fd, "metadata-bound loader load")) + goto out; + close(prog_fd); + prog_fd = -1; + ASSERT_EQ(lsm->bss->seen, 1, "metadata: one observed load"); + ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, + "metadata-bound verdict"); +out: + if (map_fd >= 0) + close(map_fd); + if (prog_fd >= 0) + close(prog_fd); + if (have_fixture) + gen_loader_fixture_fini(&f); + if (dir) + run_setup("cleanup", dir); + test_signed_loader_lsm__destroy(lsm); +} + +/* + * Load-time metadata verification: the kernel folds the frozen metadata map + * into the signature (insns || metadata) and checks it at BPF_PROG_LOAD via + * fd_array_cnt, rather than the loader checking from within BPF. Sign that + * concatenation, hand the kernel the map, and confirm the signed loader loads, + * runs, and installs its target. + */ +static int loadtime_drive(const char *dir, const void *insns, __u32 insns_sz, + const void *data, __u32 data_sz, const __u8 *excl, + void *ctx, __u32 ctx_sz, int *load_ret, bool *ran) +{ + LIBBPF_OPTS(bpf_map_create_opts, mopts, + .excl_prog_hash = excl, + .excl_prog_hash_size = SHA256_DIGEST_LENGTH); + __u32 sig_sz = 8192, key = 0; + unsigned char *buf = NULL; + int map_fd, prog_fd, ret = 0; + union bpf_attr attr; + __u8 sig[8192]; + + *ran = false; + *load_ret = 0; + + /* + * Metadata map, bound to the loader digest and frozen, exactly as + * skel_internal.h's bpf_load_and_run() sets it up. + */ + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, + data_sz, 1, &mopts); + if (map_fd < 0) { + ret = -errno; + goto out_load; + } + if (bpf_map_update_elem(map_fd, &key, data, 0) || bpf_map_freeze(map_fd)) { + ret = -errno; + goto out_load; + } + + /* Sign insns || metadata, the same bytes the kernel reconstructs. */ + buf = malloc((size_t)insns_sz + data_sz); + if (!buf) { + ret = -ENOMEM; + goto out_load; + } + memcpy(buf, insns, insns_sz); + memcpy(buf + insns_sz, data, data_sz); + ret = sign_buf(dir, buf, insns_sz + data_sz, sig, &sig_sz); + if (ret) + goto out_load; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.fd_array_cnt = 1; + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + if (prog_fd < 0) { + ret = -errno; + goto out_load; + } + + memset(&attr, 0, sizeof(attr)); + attr.test.prog_fd = prog_fd; + attr.test.ctx_in = ptr_to_u64(ctx); + attr.test.ctx_size_in = ctx_sz; + if (syscall(__NR_bpf, BPF_PROG_RUN, &attr, + offsetofend(union bpf_attr, test)) < 0) { + ret = -errno; + goto out_prog; + } + *ran = true; + ret = (int)attr.test.retval; +out_prog: + close(prog_fd); + goto out_map; +out_load: + *load_ret = ret; +out_map: + free(buf); + if (map_fd >= 0) + close(map_fd); + return ret; +} + +static void loadtime_verify(struct bpf_object *obj, int expect_maps) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + char dir_tmpl[] = "/tmp/signed_loader_ltXXXXXX", *dir = NULL; + int nr_maps = 0, nr_progs = 0, load_ret = 0, r; + __u8 excl[SHA256_DIGEST_LENGTH]; + struct bpf_prog_desc *pd; + struct bpf_map_desc *md; + unsigned char *blob = NULL; + struct bpf_program *p; + struct bpf_map *m; + __u32 ctx_sz, data_sz; + void *ctx = NULL; + bool ran = false; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + if (!ASSERT_OK(bpf_object__gen_loader(obj, &gopts), "gen_loader")) + goto out; + if (!ASSERT_OK(bpf_object__load(obj), "gen_load")) + goto out; + + bpf_object__for_each_program(p, obj) + nr_progs++; + bpf_object__for_each_map(m, obj) + nr_maps++; + if (!ASSERT_EQ(nr_maps, expect_maps, "fixture map count")) + goto out; + + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + ctx = calloc(1, ctx_sz); + if (!ASSERT_OK_PTR(ctx, "ctx_alloc")) + goto out; + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + + data_sz = gopts.data_sz; + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(blob, "blob_alloc")) + goto out; + memcpy(blob, gopts.data, data_sz); + + /* excl_prog_hash = SHA256(loader insns) == the loader's prog->digest. */ + libbpf_sha256(gopts.insns, gopts.insns_sz, excl); + + r = loadtime_drive(dir, gopts.insns, gopts.insns_sz, blob, data_sz, + excl, ctx, ctx_sz, &load_ret, &ran); + ASSERT_OK(load_ret, "signed loader loaded (insns || metadata)"); + ASSERT_TRUE(ran, "loader ran"); + ASSERT_EQ(r, 0, "loader installed its target"); + + md = (struct bpf_map_desc *)((char *)ctx + sizeof(struct bpf_loader_ctx)); + pd = (struct bpf_prog_desc *)(md + nr_maps); + ASSERT_GT(pd[0].prog_fd, 0, "target program installed"); + if (nr_maps) + ASSERT_GT(md[0].map_fd, 0, "target map installed"); + + close_loader_ctx_fds(ctx, nr_maps, nr_progs); +out: + free(blob); + free(ctx); + if (dir) + run_setup("cleanup", dir); +} + +static void loadtime_no_map(void) +{ + struct test_signed_loader *skel = test_signed_loader__open(); + + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + loadtime_verify(skel->obj, 0); + test_signed_loader__destroy(skel); +} + +static void loadtime_with_map(void) +{ + struct test_signed_loader_map *skel = test_signed_loader_map__open(); + + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + loadtime_verify(skel->obj, 1); + test_signed_loader_map__destroy(skel); +} + +/* + * A signed program need not bind any map. A plain BPF_PROG_TYPE_SYSCALL + * program with no fd_array is signed over its instructions alone: the kernel + * verifies the signature, folds no metadata, and the program loads. Exercise + * the fd_array == NULL / fd_array_cnt == 0 path, and confirm the signature + * still authenticates the instructions (a tampered copy is rejected). + */ +static void signed_no_fd_array(void) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int prog_fd, err; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + /* No metadata map: the signed payload is the instructions alone. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + /* fd_array and fd_array_cnt deliberately left NULL/0. */ + memcpy(attr.prog_name, "signed_nomap", sizeof("signed_nomap")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + if (!ASSERT_GE(prog_fd, 0, "map-less signed program loaded")) { + if (prog_fd >= 0) + close(prog_fd); + goto cleanup; + } + close(prog_fd); + + /* The signature covers the instructions, so tampering must be rejected. */ + insns[0].imm = 1; + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EKEYREJECTED, "tampered map-less program rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +} + +/* + * A signed program may reach maps only through fd_array indices, so the kernel + * folds (and thus attests) them. A direct BPF_PSEUDO_MAP_FD reference - a raw, + * unfolded fd baked into the signed instructions - is rejected by the verifier. + */ +static void signed_map_by_fd_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_LD_MAP_FD(BPF_REG_1, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int map_fd, prog_fd, err; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "sig_mapfd", 4, 4, 1, NULL); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + insns[0].imm = map_fd; /* bake the raw map fd into the ld_imm64 */ + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + goto out_map; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + goto out_map; + } + + /* Sign the instructions, raw map fd and all. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + /* No fd_array: the map is reached by a raw fd in the instructions. */ + memcpy(attr.prog_name, "signed_mapfd", sizeof("signed_mapfd")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EINVAL, "signed program referencing a map by fd rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +out_map: + close(map_fd); +} + +/* + * A signed program may reach maps only through the continuous fd_array, so the + * kernel folds (and thus attests) them. Referencing a map by fd_array *index* + * while leaving fd_array_cnt at 0 selects the sparse path, which resolves a map + * the signature never covered; the verifier rejects it up front with -EACCES. + */ +static void signed_sparse_fd_array_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_IDX, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loader_spXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int map_fd, prog_fd, err; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "sig_sparse", 4, 4, 1, NULL); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + goto out_map; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + goto out_map; + } + + /* Sign the instructions alone; the sparse map is not folded. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + attr.fd_array_cnt = 0; /* sparse: force lazy map resolution */ + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "signed_sparse", sizeof("signed_sparse")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EACCES, "signed program binding a sparse fd_array map rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +out_map: + close(map_fd); +} + +static void signed_module_kfunc_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 1, 1), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loader_kfnXXXXXX", *dir; + int prog_fd, err, fds[2]; + struct btf *btf = NULL; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new_empty")) + goto cleanup; + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto cleanup; + fds[0] = -1; + fds[1] = btf__fd(btf); + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(fds); + attr.fd_array_cnt = 0; /* sparse: force lazy kfunc BTF resolution */ + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "signed_kfunc", sizeof("signed_kfunc")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + if (prog_fd >= 0) + close(prog_fd); + + ASSERT_EQ(err, -EACCES, "module kfunc BTF in signed program rejected"); +cleanup: + if (btf) + btf__free(btf); + run_setup("cleanup", dir); +} + +void test_signed_loader(void) +{ + if (test__start_subtest("loadtime_no_map")) + loadtime_no_map(); + if (test__start_subtest("loadtime_with_map")) + loadtime_with_map(); + if (test__start_subtest("metadata_match")) + metadata_match(); + if (test__start_subtest("signature_enforced")) + signature_enforced(); + if (test__start_subtest("signed_nonexcl_fd_array_rejected")) + signed_nonexcl_fd_array_rejected(); + if (test__start_subtest("signed_unfrozen_fd_array_rejected")) + signed_unfrozen_fd_array_rejected(); + if (test__start_subtest("signed_nonarray_fd_array_rejected")) + signed_nonarray_fd_array_rejected(); + if (test__start_subtest("signed_btf_fd_array_rejected")) + signed_btf_fd_array_rejected(); + if (test__start_subtest("signed_module_kfunc_rejected")) + signed_module_kfunc_rejected(); + if (test__start_subtest("signature_failure_logs")) + signature_failure_logs(); + if (test__start_subtest("signature_too_large")) + signature_too_large(); + if (test__start_subtest("signature_zero_size")) + signature_zero_size(); + if (test__start_subtest("signature_bad_keyring")) + signature_bad_keyring(); + if (test__start_subtest("metadata_ctx_max_entries_ignored")) + metadata_ctx_max_entries_ignored(); + if (test__start_subtest("metadata_ctx_initial_value_ignored")) + metadata_ctx_initial_value_ignored(); + if (test__start_subtest("signature_authenticates_insns")) + signature_authenticates_insns(); + if (test__start_subtest("signature_authenticates_metadata")) + signature_authenticates_metadata(); + if (test__start_subtest("hash_requires_frozen")) + hash_requires_frozen(); + if (test__start_subtest("no_update_after_freeze")) + no_update_after_freeze(); + if (test__start_subtest("freeze_writable_mmap")) + freeze_writable_mmap(); + if (test__start_subtest("no_writable_mmap_frozen")) + no_writable_mmap_frozen(); + if (test__start_subtest("map_hash_matches_libbpf")) + map_hash_matches_libbpf(); + if (test__start_subtest("map_hash_multi_element")) + map_hash_multi_element(); + if (test__start_subtest("map_hash_bad_size")) + map_hash_bad_size(); + if (test__start_subtest("map_hash_unsupported_type")) + map_hash_unsupported_type(); + if (test__start_subtest("lsm_signature_verdict")) + lsm_signature_verdict(); + if (test__start_subtest("signed_no_fd_array")) + signed_no_fd_array(); + if (test__start_subtest("signed_map_by_fd_rejected")) + signed_map_by_fd_rejected(); + if (test__start_subtest("signed_sparse_fd_array_rejected")) + signed_sparse_fd_array_rejected(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sleepable_tracepoints.c b/tools/testing/selftests/bpf/prog_tests/sleepable_tracepoints.c new file mode 100644 index 000000000000..19500b785ee3 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/sleepable_tracepoints.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include <unistd.h> +#include "test_sleepable_tracepoints.skel.h" +#include "test_sleepable_tracepoints_fail.skel.h" + +static void run_test(struct test_sleepable_tracepoints *skel) +{ + char buf[PATH_MAX] = "/"; + + skel->bss->target_pid = getpid(); + skel->bss->prog_triggered = 0; + skel->bss->err = 0; + skel->bss->copied_byte = 0; + + syscall(__NR_getcwd, buf, sizeof(buf)); + + ASSERT_EQ(skel->bss->prog_triggered, 1, "prog_triggered"); + ASSERT_EQ(skel->bss->err, 0, "err"); + ASSERT_EQ(skel->bss->copied_byte, '/', "copied_byte"); +} + +static void run_auto_attach_test(struct bpf_program *prog, + struct test_sleepable_tracepoints *skel) +{ + struct bpf_link *link; + + link = bpf_program__attach(prog); + if (!ASSERT_OK_PTR(link, "prog_attach")) + return; + + run_test(skel); + bpf_link__destroy(link); +} + +static void test_attach_only(struct bpf_program *prog) +{ + struct bpf_link *link; + + link = bpf_program__attach(prog); + if (ASSERT_OK_PTR(link, "attach")) + bpf_link__destroy(link); +} + +static void test_attach_reject(struct bpf_program *prog) +{ + struct bpf_link *link; + + link = bpf_program__attach(prog); + if (!ASSERT_ERR_PTR(link, "attach_should_fail")) + bpf_link__destroy(link); +} + +static void test_raw_tp_bare(struct test_sleepable_tracepoints *skel) +{ + struct bpf_link *link; + + link = bpf_program__attach_raw_tracepoint(skel->progs.handle_raw_tp_bare, + "sys_enter"); + if (ASSERT_OK_PTR(link, "attach")) + bpf_link__destroy(link); +} + +static void test_tp_bare(struct test_sleepable_tracepoints *skel) +{ + struct bpf_link *link; + + link = bpf_program__attach_tracepoint(skel->progs.handle_tp_bare, + "syscalls", "sys_enter_getcwd"); + if (ASSERT_OK_PTR(link, "attach")) + bpf_link__destroy(link); +} + +static void test_test_run(struct test_sleepable_tracepoints *skel) +{ + __u64 args[2] = {0x1234ULL, 0x5678ULL}; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + ); + int fd, err; + + fd = bpf_program__fd(skel->progs.handle_test_run); + err = bpf_prog_test_run_opts(fd, &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, args[0] + args[1], "test_run_retval"); +} + +static void test_test_run_on_cpu_reject(struct test_sleepable_tracepoints *skel) +{ + __u64 args[2] = {}; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + int fd, err; + + fd = bpf_program__fd(skel->progs.handle_test_run); + err = bpf_prog_test_run_opts(fd, &topts); + ASSERT_ERR(err, "test_run_on_cpu_reject"); +} + +void test_sleepable_tracepoints(void) +{ + struct test_sleepable_tracepoints *skel; + + skel = test_sleepable_tracepoints__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + if (test__start_subtest("tp_btf")) + run_auto_attach_test(skel->progs.handle_sys_enter_tp_btf, skel); + if (test__start_subtest("raw_tp")) + run_auto_attach_test(skel->progs.handle_sys_enter_raw_tp, skel); + if (test__start_subtest("tracepoint")) + run_auto_attach_test(skel->progs.handle_sys_enter_tp, skel); + if (test__start_subtest("sys_exit")) + run_auto_attach_test(skel->progs.handle_sys_exit_tp, skel); + if (test__start_subtest("tracepoint_alias")) + test_attach_only(skel->progs.handle_sys_enter_tp_alias); + if (test__start_subtest("raw_tracepoint_alias")) + test_attach_only(skel->progs.handle_sys_enter_raw_tp_alias); + if (test__start_subtest("raw_tp_bare")) + test_raw_tp_bare(skel); + if (test__start_subtest("tp_bare")) + test_tp_bare(skel); + if (test__start_subtest("test_run")) + test_test_run(skel); + if (test__start_subtest("test_run_on_cpu_reject")) + test_test_run_on_cpu_reject(skel); + if (test__start_subtest("raw_tp_non_faultable")) + test_attach_reject(skel->progs.handle_raw_tp_non_faultable); + if (test__start_subtest("tp_non_syscall")) + test_attach_reject(skel->progs.handle_tp_non_syscall); + if (test__start_subtest("tp_btf_non_faultable_reject")) + RUN_TESTS(test_sleepable_tracepoints_fail); + + test_sleepable_tracepoints__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf.c b/tools/testing/selftests/bpf/prog_tests/snprintf.c index 594441acb707..4e4a82d54f79 100644 --- a/tools/testing/selftests/bpf/prog_tests/snprintf.c +++ b/tools/testing/selftests/bpf/prog_tests/snprintf.c @@ -114,7 +114,8 @@ static void test_snprintf_negative(void) ASSERT_ERR(load_single_snprintf("%--------"), "invalid specifier 5"); ASSERT_ERR(load_single_snprintf("%lc"), "invalid specifier 6"); ASSERT_ERR(load_single_snprintf("%llc"), "invalid specifier 7"); - ASSERT_ERR(load_single_snprintf("\x80"), "non ascii character"); + ASSERT_OK(load_single_snprintf("\x80"), "non ascii plain text"); + ASSERT_ERR(load_single_snprintf("%\x80"), "non ascii in specifier"); ASSERT_ERR(load_single_snprintf("\x1"), "non printable character"); ASSERT_ERR(load_single_snprintf("%p%"), "invalid specifier 8"); ASSERT_ERR(load_single_snprintf("%s%"), "invalid specifier 9"); diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c b/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c index dd41b826be30..edce9c1b54fb 100644 --- a/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c +++ b/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> #include <linux/btf.h> +#include <bpf/btf.h> #include "netif_receive_skb.skel.h" +#include "snprintf_btf_void.skel.h" /* Demonstrate that bpf_snprintf_btf succeeds and that various data types * are formatted correctly. @@ -58,3 +60,80 @@ void serial_test_snprintf_btf(void) cleanup: netif_receive_skb__destroy(skel); } + +/* + * bpf_snprintf_btf() renders a type_id taken straight from the vmlinux BTF. + * Two such type_ids used to NULL-deref in the BTF show path: + * - a "const void" (a modifier resolving to void) in btf_modifier_show() + * - a BTF_KIND_VAR in btf_var_show() (base BTF has no resolved_ids) + * A fixed kernel renders both without crashing. + */ +static long run(struct snprintf_btf_void *skel, __u32 type_id) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char ctx[8] = {}; + + skel->bss->type_id = type_id; + topts.ctx_in = ctx; + topts.ctx_size_in = sizeof(ctx); + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.dump_type), + &topts), "test_run")) + return -1; + return skel->bss->ret; +} + +void test_snprintf_btf_void(void) +{ + const struct btf_type *t; + struct snprintf_btf_void *skel; + int i, n, cv = 0, var = 0; + struct btf *btf; + + btf = btf__parse("/sys/kernel/btf/vmlinux", NULL); + if (!btf) { + test__skip(); + return; + } + + skel = snprintf_btf_void__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + goto out_btf; + + n = btf__type_cnt(btf); + for (i = 1; i < n && !(cv && var); i++) { + t = btf__type_by_id(btf, i); + if (!cv && btf_kind(t) == BTF_KIND_CONST && t->type == 0) + cv = i; + /* Pick a VAR small enough to render from the program's buffer. */ + if (!var && btf_kind(t) == BTF_KIND_VAR) { + long sz = btf__resolve_size(btf, t->type); + + if (sz > 0 && sz <= (long)sizeof(skel->bss->obj)) + var = i; + } + } + + /* "const void" renders the "<unsupported kind:0>" placeholder. */ + if (test__start_subtest("const_void")) { + if (cv) { + ASSERT_EQ(run(skel, cv), + sizeof("<unsupported kind:0>") - 1, "ret"); + ASSERT_STREQ(skel->bss->out, "<unsupported kind:0>", + "placeholder"); + } else { + test__skip(); + } + } + + /* A BTF_KIND_VAR must resolve and render without error. */ + if (test__start_subtest("var")) { + if (var) + ASSERT_GT(run(skel, var), 0, "ret"); + else + test__skip(); + } + + snprintf_btf_void__destroy(skel); +out_btf: + btf__free(btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sock_destroy.c b/tools/testing/selftests/bpf/prog_tests/sock_destroy.c index 9c11938fe597..78d642a02bdb 100644 --- a/tools/testing/selftests/bpf/prog_tests/sock_destroy.c +++ b/tools/testing/selftests/bpf/prog_tests/sock_destroy.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include <poll.h> #include <test_progs.h> #include <bpf/bpf_endian.h> @@ -110,6 +111,122 @@ cleanup: close(serv); } +static void test_tcp_listen_pending(struct sock_destroy_prog *skel) +{ + int serv = -1, clien = -1, accept_serv = -1, n, serv_port; + struct pollfd pfd = { .events = POLLIN }; + char buf[1]; + + serv = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_GE(serv, 0, "start_server")) + goto cleanup; + serv_port = get_socket_local_port(serv); + if (!ASSERT_GE(serv_port, 0, "get_sock_local_port")) + goto cleanup; + skel->bss->serv_port = (__be16)serv_port; + + /* + * Connect but never accept, so the child sits in the accept queue + * of the listener. Wait until it's actually there. + */ + clien = connect_to_fd(serv, 0); + if (!ASSERT_GE(clien, 0, "connect_to_fd")) + goto cleanup; + pfd.fd = serv; + if (!ASSERT_EQ(poll(&pfd, 1, -1), 1, "poll listener")) + goto cleanup; + + /* Run iterator program that destroys server sockets. */ + start_iter_sockets(skel->progs.iter_tcp6_server); + + accept_serv = accept(serv, NULL, NULL); + if (!ASSERT_LT(accept_serv, 0, "accept on destroyed listener")) + goto cleanup; + ASSERT_EQ(errno, EINVAL, "error code on destroyed listener"); + + /* The unaccepted child was reset along with the listener. */ + n = recv(clien, buf, sizeof(buf), 0); + if (!ASSERT_LT(n, 0, "client recv on reset child")) + goto cleanup; + ASSERT_EQ(errno, ECONNRESET, "error code on reset child"); + +cleanup: + if (clien != -1) + close(clien); + if (accept_serv != -1) + close(accept_serv); + if (serv != -1) + close(serv); +} + +static void test_tcp_timewait(struct sock_destroy_prog *skel) +{ + int serv = -1, clien = -1, accept_serv = -1, n; + struct timeval tv = {}; + char buf[1]; + + serv = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_GE(serv, 0, "start_server")) + goto cleanup; + + clien = connect_to_fd(serv, 0); + if (!ASSERT_GE(clien, 0, "connect_to_fd")) + goto cleanup; + + accept_serv = accept(serv, NULL, NULL); + if (!ASSERT_GE(accept_serv, 0, "serv accept")) + goto cleanup; + + /* + * Active close from the client, then close the server side. Once + * recv() sees EOF the server FIN has been processed and the client + * sock is in TIME_WAIT. Block without timeout so a loaded CI box + * can't race us. + */ + if (!ASSERT_OK(setsockopt(clien, SOL_SOCKET, SO_RCVTIMEO, &tv, + sizeof(tv)), "clear rcvtimeo")) + goto cleanup; + if (!ASSERT_OK(shutdown(clien, SHUT_WR), "client shutdown")) + goto cleanup; + + /* + * Make sure the server has seen the client FIN before it closes, + * so the two FINs never cross. + */ + n = recv(accept_serv, buf, sizeof(buf), 0); + if (!ASSERT_EQ(n, 0, "server recv EOF")) + goto cleanup; + + close(accept_serv); + accept_serv = -1; + + /* block until return EOF */ + n = recv(clien, buf, sizeof(buf), 0); + if (!ASSERT_EQ(n, 0, "client recv EOF")) + goto cleanup; + + /* Run iterator program that destroys the timewait client sock. */ + skel->bss->tw_found = 0; + start_iter_sockets(skel->progs.iter_tcp6_timewait); + if (!ASSERT_EQ(skel->bss->tw_found, 1, "timewait sock found")) + goto cleanup; + + ASSERT_OK(skel->bss->tw_destroy_err, "destroy timewait sock"); + + /* The destroyed timewait sock must be gone. */ + skel->bss->tw_found = 0; + start_iter_sockets(skel->progs.iter_tcp6_timewait); + ASSERT_EQ(skel->bss->tw_found, 0, "timewait sock destroyed"); + +cleanup: + if (clien != -1) + close(clien); + if (accept_serv != -1) + close(accept_serv); + if (serv != -1) + close(serv); +} + static void test_udp_client(struct sock_destroy_prog *skel) { int serv = -1, clien = -1, n = 0; @@ -204,6 +321,10 @@ void test_sock_destroy(void) test_tcp_client(skel); if (test__start_subtest("tcp_server")) test_tcp_server(skel); + if (test__start_subtest("tcp_listen_pending")) + test_tcp_listen_pending(skel); + if (test__start_subtest("tcp_timewait")) + test_tcp_timewait(skel); if (test__start_subtest("udp_client")) test_udp_client(skel); if (test__start_subtest("udp_server")) diff --git a/tools/testing/selftests/bpf/prog_tests/sock_ops_get_sk.c b/tools/testing/selftests/bpf/prog_tests/sock_ops_get_sk.c new file mode 100644 index 000000000000..343d92c4df30 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/sock_ops_get_sk.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "cgroup_helpers.h" +#include "network_helpers.h" +#include "sock_ops_get_sk.skel.h" + +/* See progs/sock_ops_get_sk.c for the bug description. */ +static void run_sock_ops_test(int cgroup_fd, int prog_fd) +{ + int server_fd, client_fd, err; + + err = bpf_prog_attach(prog_fd, cgroup_fd, BPF_CGROUP_SOCK_OPS, 0); + if (!ASSERT_OK(err, "prog_attach")) + return; + + server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); + if (!ASSERT_OK_FD(server_fd, "start_server")) + goto detach; + + /* Trigger TCP handshake which causes TCP_NEW_SYN_RECV state where + * is_fullsock == 0 and is_locked_tcp_sock == 0. + */ + client_fd = connect_to_fd(server_fd, 0); + if (!ASSERT_OK_FD(client_fd, "connect_to_fd")) + goto close_server; + + close(client_fd); + +close_server: + close(server_fd); +detach: + bpf_prog_detach(cgroup_fd, BPF_CGROUP_SOCK_OPS); +} + +void test_ns_sock_ops_get_sk(void) +{ + struct sock_ops_get_sk *skel; + int cgroup_fd; + + cgroup_fd = test__join_cgroup("/sock_ops_get_sk"); + if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup")) + return; + + skel = sock_ops_get_sk__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_load")) + goto close_cgroup; + + /* Test SOCK_OPS_GET_SK with same src/dst register */ + if (test__start_subtest("get_sk")) { + run_sock_ops_test(cgroup_fd, + bpf_program__fd(skel->progs.sock_ops_get_sk_same_reg)); + ASSERT_EQ(skel->bss->null_seen, 1, "null_seen"); + ASSERT_EQ(skel->bss->bug_detected, 0, "bug_not_detected"); + } + + /* Test SOCK_OPS_GET_FIELD with same src/dst register */ + if (test__start_subtest("get_field")) { + run_sock_ops_test(cgroup_fd, + bpf_program__fd(skel->progs.sock_ops_get_field_same_reg)); + ASSERT_EQ(skel->bss->field_null_seen, 1, "field_null_seen"); + ASSERT_EQ(skel->bss->field_bug_detected, 0, "field_bug_not_detected"); + } + + /* Test SOCK_OPS_GET_SK with different src/dst register */ + if (test__start_subtest("get_sk_diff_reg")) { + run_sock_ops_test(cgroup_fd, + bpf_program__fd(skel->progs.sock_ops_get_sk_diff_reg)); + ASSERT_EQ(skel->bss->diff_reg_null_seen, 1, "diff_reg_null_seen"); + ASSERT_EQ(skel->bss->diff_reg_bug_detected, 0, "diff_reg_bug_not_detected"); + } + + sock_ops_get_sk__destroy(skel); +close_cgroup: + close(cgroup_fd); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sock_xattr.c b/tools/testing/selftests/bpf/prog_tests/sock_xattr.c new file mode 100644 index 000000000000..b5816e90f01a --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/sock_xattr.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Christian Brauner */ + +#include <errno.h> +#include <string.h> +#include <unistd.h> +#include <sys/xattr.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <test_progs.h> + +#include "sock_read_xattr.skel.h" + +static const char xattr_value[] = "bpf_sock_value"; +static const char xattr_name[] = "user.bpf_test"; + +static void test_read_sock_xattr(void) +{ + struct sockaddr_in addr = {}; + struct sock_read_xattr *skel = NULL; + struct bpf_link *link = NULL; + int sock_fd = -1, err; + + sock_fd = socket(AF_INET, SOCK_STREAM, 0); + if (!ASSERT_OK_FD(sock_fd, "socket")) + return; + + err = fsetxattr(sock_fd, xattr_name, xattr_value, sizeof(xattr_value), 0); + if (!ASSERT_OK(err, "fsetxattr")) + goto out; + + skel = sock_read_xattr__open_and_load(); + if (!ASSERT_OK_PTR(skel, "sock_read_xattr__open_and_load")) + goto out; + + skel->bss->monitored_pid = sys_gettid(); + + /* Only attach the functional program; the verifier-only programs + * above are not pid-gated and would clobber the shared globals. + */ + link = bpf_program__attach(skel->progs.read_sock_xattr); + if (!ASSERT_OK_PTR(link, "attach read_sock_xattr")) + goto out; + + addr.sin_family = AF_INET; + addr.sin_port = htons(1234); + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + /* Only the lsm/socket_connect hook matters; the connect may fail. */ + connect(sock_fd, (struct sockaddr *)&addr, sizeof(addr)); + + ASSERT_EQ(skel->data->read_ret, sizeof(xattr_value), "read_ret"); + ASSERT_STREQ(skel->bss->value, xattr_value, "value"); + +out: + bpf_link__destroy(link); + if (sock_fd >= 0) + close(sock_fd); + sock_read_xattr__destroy(skel); +} + +void test_sock_xattr(void) +{ + RUN_TESTS(sock_read_xattr); + + if (test__start_subtest("read_sock_xattr")) + test_read_sock_xattr(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index dd3c757859f6..1fef6ec2ba7a 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -7,13 +7,13 @@ #include "test_progs.h" #include "test_skmsg_load_helpers.skel.h" -#include "test_sockmap_update.skel.h" #include "test_sockmap_invalid_update.skel.h" #include "test_sockmap_skb_verdict_attach.skel.h" #include "test_sockmap_progs_query.skel.h" #include "test_sockmap_pass_prog.skel.h" #include "test_sockmap_drop_prog.skel.h" #include "test_sockmap_change_tail.skel.h" +#include "test_sockmap_msg_pop_data.skel.h" #include "bpf_iter_sockmap.skel.h" #include "sockmap_helpers.h" @@ -234,53 +234,6 @@ out: test_skmsg_load_helpers__destroy(skel); } -static void test_sockmap_update(enum bpf_map_type map_type) -{ - int err, prog, src; - struct test_sockmap_update *skel; - struct bpf_map *dst_map; - const __u32 zero = 0; - char dummy[14] = {0}; - LIBBPF_OPTS(bpf_test_run_opts, topts, - .data_in = dummy, - .data_size_in = sizeof(dummy), - .repeat = 1, - ); - __s64 sk; - - sk = connected_socket_v4(); - if (!ASSERT_NEQ(sk, -1, "connected_socket_v4")) - return; - - skel = test_sockmap_update__open_and_load(); - if (!ASSERT_OK_PTR(skel, "open_and_load")) - goto close_sk; - - prog = bpf_program__fd(skel->progs.copy_sock_map); - src = bpf_map__fd(skel->maps.src); - if (map_type == BPF_MAP_TYPE_SOCKMAP) - dst_map = skel->maps.dst_sock_map; - else - dst_map = skel->maps.dst_sock_hash; - - err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST); - if (!ASSERT_OK(err, "update_elem(src)")) - goto out; - - err = bpf_prog_test_run_opts(prog, &topts); - if (!ASSERT_OK(err, "test_run")) - goto out; - if (!ASSERT_NEQ(topts.retval, 0, "test_run retval")) - goto out; - - compare_cookies(skel->maps.src, dst_map); - -out: - test_sockmap_update__destroy(skel); -close_sk: - close(sk); -} - static void test_sockmap_invalid_update(void) { struct test_sockmap_invalid_update *skel; @@ -666,6 +619,51 @@ out: test_sockmap_change_tail__destroy(skel); } +static void test_sockmap_msg_verdict_pop_data(void) +{ + struct test_sockmap_msg_pop_data *skel; + int err, map, verdict; + int c1 = -1, p1 = -1, sent; + int zero = 0; + char *buf; + const size_t len = 32 * 1024; + + skel = test_sockmap_msg_pop_data__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + verdict = bpf_program__fd(skel->progs.prog_msg_pop_data); + map = bpf_map__fd(skel->maps.sock_map); + + err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0); + if (!ASSERT_OK(err, "bpf_prog_attach")) + goto out; + + err = create_pair(AF_INET, SOCK_STREAM, &c1, &p1); + if (!ASSERT_OK(err, "create_pair")) + goto out; + + err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out_close; + + buf = calloc(len, 1); + if (!ASSERT_OK_PTR(buf, "calloc")) + goto out_close; + + sent = xsend(c1, buf, len, 0); + ASSERT_EQ(sent, (ssize_t)len, "xsend"); + ASSERT_EQ(skel->data->pop_data_ret, -EINVAL, "pop_data_rejects overflow"); + + free(buf); + +out_close: + close(c1); + close(p1); +out: + test_sockmap_msg_pop_data__destroy(skel); +} + static void test_sockmap_skb_verdict_peek_helper(int map) { int err, c1, p1, zero = 0, sent, recvd, avail; @@ -807,7 +805,7 @@ static void test_sockmap_many_socket(void) return; } - udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); + udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK); if (udp < 0) { close(dgram); close(tcp); @@ -876,7 +874,7 @@ static void test_sockmap_many_maps(void) return; } - udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); + udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK); if (udp < 0) { close(dgram); close(tcp); @@ -947,7 +945,7 @@ static void test_sockmap_same_sock(void) return; } - udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); + udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK); if (udp < 0) { close(dgram); close(tcp); @@ -1298,10 +1296,23 @@ static void test_sockmap_multi_channels(int sotype) avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC); ASSERT_EQ(avail, expected, "ioctl(FIONREAD) full return"); - recvd = recv_timeout(p1, rcv, sizeof(rcv), MSG_DONTWAIT, 1); - if (!ASSERT_EQ(recvd, sizeof(buf), "recv_timeout(p1)") || + recvd = recv_timeout(p1, rcv, expected, MSG_DONTWAIT, 1); + if (!ASSERT_EQ(recvd, expected, "recv_timeout(p1)") || !ASSERT_OK(memcmp(buf, rcv, recvd), "data mismatch")) goto end; + + /* process remaining data for udp if secondary data is available */ + expected = sizeof(buf) - expected; + if (expected) { + avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC); + ASSERT_EQ(avail, expected, "second ioctl(FIONREAD) full return"); + + recvd = recv_timeout(p1, rcv, expected, MSG_DONTWAIT, 1); + if (!ASSERT_EQ(recvd, expected, "second recv_timeout(p1)") || + !ASSERT_OK(memcmp(buf + sizeof(buf) - expected, rcv, recvd), + "second data mismatch")) + goto end; + } end: if (c0 >= 0) close(c0); @@ -1314,6 +1325,43 @@ end: test_sockmap_pass_prog__destroy(skel); } +/* A socket in a sockmap without a verdict program keeps its ingress data + * in sk_receive_queue: FIONREAD must account for it. + */ +static void test_sockmap_no_verdict_fionread(void) +{ + int err, map, zero = 0, sent, avail; + int c0 = -1, c1 = -1, p0 = -1, p1 = -1; + struct test_sockmap_pass_prog *skel; + char buf[256] = "0123456789"; + + skel = test_sockmap_pass_prog__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + map = bpf_map__fd(skel->maps.sock_map_rx); + + err = create_socket_pairs(AF_INET, SOCK_STREAM, &c0, &c1, &p0, &p1); + if (!ASSERT_OK(err, "create_socket_pairs()")) + goto out; + + err = bpf_map_update_elem(map, &zero, &c1, BPF_NOEXIST); + if (!ASSERT_OK(err, "bpf_map_update_elem(c1)")) + goto out_close; + + sent = xsend(p1, &buf, sizeof(buf), 0); + ASSERT_EQ(sent, sizeof(buf), "xsend(p1)"); + avail = wait_for_fionread(c1, sizeof(buf), IO_TIMEOUT_SEC); + ASSERT_EQ(avail, sizeof(buf), "ioctl(FIONREAD)"); + +out_close: + close(c0); + close(p0); + close(c1); + close(p1); +out: + test_sockmap_pass_prog__destroy(skel); +} + void test_sockmap_basic(void) { if (test__start_subtest("sockmap create_update_free")) @@ -1326,10 +1374,6 @@ void test_sockmap_basic(void) test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP); if (test__start_subtest("sockhash sk_msg load helpers")) test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH); - if (test__start_subtest("sockmap update")) - test_sockmap_update(BPF_MAP_TYPE_SOCKMAP); - if (test__start_subtest("sockhash update")) - test_sockmap_update(BPF_MAP_TYPE_SOCKHASH); if (test__start_subtest("sockmap update in unsafe context")) test_sockmap_invalid_update(); if (test__start_subtest("sockmap copy")) @@ -1356,10 +1400,14 @@ void test_sockmap_basic(void) test_sockmap_skb_verdict_shutdown(); if (test__start_subtest("sockmap skb_verdict fionread")) test_sockmap_skb_verdict_fionread(true); + if (test__start_subtest("sockmap no_verdict fionread")) + test_sockmap_no_verdict_fionread(); if (test__start_subtest("sockmap skb_verdict fionread on drop")) test_sockmap_skb_verdict_fionread(false); if (test__start_subtest("sockmap skb_verdict change tail")) test_sockmap_skb_verdict_change_tail(); + if (test__start_subtest("sockmap msg_verdict pop_data overflow")) + test_sockmap_msg_verdict_pop_data(); if (test__start_subtest("sockmap skb_verdict msg_f_peek")) test_sockmap_skb_verdict_peek(); if (test__start_subtest("sockmap skb_verdict msg_f_peek with link")) diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c index b87e7f39e15a..34737e8df6ea 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c @@ -9,7 +9,6 @@ #include "test_progs.h" #include "sockmap_helpers.h" #include "test_skmsg_load_helpers.skel.h" -#include "test_sockmap_ktls.skel.h" #define MAX_TEST_NAME 80 #define TCP_ULP 31 @@ -117,6 +116,68 @@ close: close(s); } +static void test_sockmap_ktls_enable_fails_when_in_sockmap(int family, int map) +{ + struct tls12_crypto_info_aes_gcm_128 crypto = { + .info = { + .version = TLS_1_2_VERSION, + .cipher_type = TLS_CIPHER_AES_GCM_128, + }, + }; + struct sockaddr_storage addr = {}; + socklen_t len = sizeof(addr); + struct sockaddr_in6 *v6; + struct sockaddr_in *v4; + int err, s, zero = 0; + + switch (family) { + case AF_INET: + v4 = (struct sockaddr_in *)&addr; + v4->sin_family = AF_INET; + break; + case AF_INET6: + v6 = (struct sockaddr_in6 *)&addr; + v6->sin6_family = AF_INET6; + break; + default: + PRINT_FAIL("unsupported socket family %d", family); + return; + } + + s = socket(family, SOCK_STREAM, 0); + if (!ASSERT_GE(s, 0, "socket")) + return; + + err = bind(s, (struct sockaddr *)&addr, len); + if (!ASSERT_OK(err, "bind")) + goto close; + + err = getsockname(s, (struct sockaddr *)&addr, &len); + if (!ASSERT_OK(err, "getsockname")) + goto close; + + err = connect(s, (struct sockaddr *)&addr, len); + if (!ASSERT_OK(err, "connect")) + goto close; + + /* Add the socket to the sockmap, attaching a psock. */ + err = bpf_map_update_elem(map, &zero, &s, BPF_ANY); + if (!ASSERT_OK(err, "sockmap update elem")) + goto close; + + /* Installing the TLS ULP is allowed, it does not touch the datapath. */ + err = setsockopt(s, IPPROTO_TCP, TCP_ULP, "tls", strlen("tls")); + if (!ASSERT_OK(err, "setsockopt(TCP_ULP)")) + goto close; + + /* Enabling the TLS crypto datapath must be rejected. */ + err = setsockopt(s, SOL_TLS, TLS_TX, &crypto, sizeof(crypto)); + ASSERT_ERR(err, "setsockopt(TLS_TX)"); + +close: + close(s); +} + static const char *fmt_test_name(const char *subtest_name, int family, enum bpf_map_type map_type) { @@ -160,249 +221,6 @@ out: close(p); } -static void test_sockmap_ktls_tx_cork(int family, int sotype, bool push) -{ - int err, off; - int i, j; - int start_push = 0, push_len = 0; - int c = 0, p = 0, one = 1, sent, recvd; - int prog_fd, map_fd; - char msg[12] = "hello world\0"; - char rcv[20] = {0}; - struct test_sockmap_ktls *skel; - - skel = test_sockmap_ktls__open_and_load(); - if (!ASSERT_TRUE(skel, "open ktls skel")) - return; - - err = create_pair(family, sotype, &c, &p); - if (!ASSERT_OK(err, "create_pair()")) - goto out; - - prog_fd = bpf_program__fd(skel->progs.prog_sk_policy); - map_fd = bpf_map__fd(skel->maps.sock_map); - - err = bpf_prog_attach(prog_fd, map_fd, BPF_SK_MSG_VERDICT, 0); - if (!ASSERT_OK(err, "bpf_prog_attach sk msg")) - goto out; - - err = bpf_map_update_elem(map_fd, &one, &c, BPF_NOEXIST); - if (!ASSERT_OK(err, "bpf_map_update_elem(c)")) - goto out; - - err = init_ktls_pairs(c, p); - if (!ASSERT_OK(err, "init_ktls_pairs(c, p)")) - goto out; - - skel->bss->cork_byte = sizeof(msg); - if (push) { - start_push = 1; - push_len = 2; - } - skel->bss->push_start = start_push; - skel->bss->push_end = push_len; - - off = sizeof(msg) / 2; - sent = send(c, msg, off, 0); - if (!ASSERT_EQ(sent, off, "send(msg)")) - goto out; - - recvd = recv_timeout(p, rcv, sizeof(rcv), MSG_DONTWAIT, 1); - if (!ASSERT_EQ(-1, recvd, "expected no data")) - goto out; - - /* send remaining msg */ - sent = send(c, msg + off, sizeof(msg) - off, 0); - if (!ASSERT_EQ(sent, sizeof(msg) - off, "send remaining data")) - goto out; - - recvd = recv_timeout(p, rcv, sizeof(rcv), MSG_DONTWAIT, 1); - if (!ASSERT_OK(err, "recv(msg)") || - !ASSERT_EQ(recvd, sizeof(msg) + push_len, "check length mismatch")) - goto out; - - for (i = 0, j = 0; i < recvd;) { - /* skip checking the data that has been pushed in */ - if (i >= start_push && i <= start_push + push_len - 1) { - i++; - continue; - } - if (!ASSERT_EQ(rcv[i], msg[j], "data mismatch")) - goto out; - i++; - j++; - } -out: - if (c) - close(c); - if (p) - close(p); - test_sockmap_ktls__destroy(skel); -} - -static void test_sockmap_ktls_tx_no_buf(int family, int sotype, bool push) -{ - int c = -1, p = -1, one = 1, two = 2; - struct test_sockmap_ktls *skel; - unsigned char *data = NULL; - struct msghdr msg = {0}; - struct iovec iov[2]; - int prog_fd, map_fd; - int txrx_buf = 1024; - int iov_length = 8192; - int err; - - skel = test_sockmap_ktls__open_and_load(); - if (!ASSERT_TRUE(skel, "open ktls skel")) - return; - - err = create_pair(family, sotype, &c, &p); - if (!ASSERT_OK(err, "create_pair()")) - goto out; - - err = setsockopt(c, SOL_SOCKET, SO_RCVBUFFORCE, &txrx_buf, sizeof(int)); - err |= setsockopt(p, SOL_SOCKET, SO_SNDBUFFORCE, &txrx_buf, sizeof(int)); - if (!ASSERT_OK(err, "set buf limit")) - goto out; - - prog_fd = bpf_program__fd(skel->progs.prog_sk_policy_redir); - map_fd = bpf_map__fd(skel->maps.sock_map); - - err = bpf_prog_attach(prog_fd, map_fd, BPF_SK_MSG_VERDICT, 0); - if (!ASSERT_OK(err, "bpf_prog_attach sk msg")) - goto out; - - err = bpf_map_update_elem(map_fd, &one, &c, BPF_NOEXIST); - if (!ASSERT_OK(err, "bpf_map_update_elem(c)")) - goto out; - - err = bpf_map_update_elem(map_fd, &two, &p, BPF_NOEXIST); - if (!ASSERT_OK(err, "bpf_map_update_elem(p)")) - goto out; - - skel->bss->apply_bytes = 1024; - - err = init_ktls_pairs(c, p); - if (!ASSERT_OK(err, "init_ktls_pairs(c, p)")) - goto out; - - data = calloc(iov_length, sizeof(char)); - if (!data) - goto out; - - iov[0].iov_base = data; - iov[0].iov_len = iov_length; - iov[1].iov_base = data; - iov[1].iov_len = iov_length; - msg.msg_iov = iov; - msg.msg_iovlen = 2; - - for (;;) { - err = sendmsg(c, &msg, MSG_DONTWAIT); - if (err <= 0) - break; - } - -out: - if (data) - free(data); - if (c != -1) - close(c); - if (p != -1) - close(p); - - test_sockmap_ktls__destroy(skel); -} - -static void test_sockmap_ktls_tx_pop(int family, int sotype) -{ - char msg[37] = "0123456789abcdefghijklmnopqrstuvwxyz\0"; - int c = 0, p = 0, one = 1, sent, recvd; - struct test_sockmap_ktls *skel; - int prog_fd, map_fd; - char rcv[50] = {0}; - int err; - int i, m, r; - - skel = test_sockmap_ktls__open_and_load(); - if (!ASSERT_TRUE(skel, "open ktls skel")) - return; - - err = create_pair(family, sotype, &c, &p); - if (!ASSERT_OK(err, "create_pair()")) - goto out; - - prog_fd = bpf_program__fd(skel->progs.prog_sk_policy); - map_fd = bpf_map__fd(skel->maps.sock_map); - - err = bpf_prog_attach(prog_fd, map_fd, BPF_SK_MSG_VERDICT, 0); - if (!ASSERT_OK(err, "bpf_prog_attach sk msg")) - goto out; - - err = bpf_map_update_elem(map_fd, &one, &c, BPF_NOEXIST); - if (!ASSERT_OK(err, "bpf_map_update_elem(c)")) - goto out; - - err = init_ktls_pairs(c, p); - if (!ASSERT_OK(err, "init_ktls_pairs(c, p)")) - goto out; - - struct { - int pop_start; - int pop_len; - } pop_policy[] = { - /* trim the start */ - {0, 2}, - {0, 10}, - {1, 2}, - {1, 10}, - /* trim the end */ - {35, 2}, - /* New entries should be added before this line */ - {-1, -1}, - }; - - i = 0; - while (pop_policy[i].pop_start >= 0) { - skel->bss->pop_start = pop_policy[i].pop_start; - skel->bss->pop_end = pop_policy[i].pop_len; - - sent = send(c, msg, sizeof(msg), 0); - if (!ASSERT_EQ(sent, sizeof(msg), "send(msg)")) - goto out; - - recvd = recv_timeout(p, rcv, sizeof(rcv), MSG_DONTWAIT, 1); - if (!ASSERT_EQ(recvd, sizeof(msg) - pop_policy[i].pop_len, "pop len mismatch")) - goto out; - - /* verify the data - * msg: 0123456789a bcdefghij klmnopqrstuvwxyz - * | | - * popped data - */ - for (m = 0, r = 0; m < sizeof(msg);) { - /* skip checking the data that has been popped */ - if (m >= pop_policy[i].pop_start && - m <= pop_policy[i].pop_start + pop_policy[i].pop_len - 1) { - m++; - continue; - } - - if (!ASSERT_EQ(msg[m], rcv[r], "data mismatch")) - goto out; - m++; - r++; - } - i++; - } -out: - if (c) - close(c); - if (p) - close(p); - test_sockmap_ktls__destroy(skel); -} - static void run_tests(int family, enum bpf_map_type map_type) { int map; @@ -414,6 +232,9 @@ static void run_tests(int family, enum bpf_map_type map_type) if (test__start_subtest(fmt_test_name("update_fails_when_sock_has_ulp", family, map_type))) test_sockmap_ktls_update_fails_when_sock_has_ulp(family, map); + if (test__start_subtest(fmt_test_name("enable_fails_when_in_sockmap", family, map_type))) + test_sockmap_ktls_enable_fails_when_in_sockmap(family, map); + close(map); } @@ -421,14 +242,6 @@ static void run_ktls_test(int family, int sotype) { if (test__start_subtest("tls simple offload")) test_sockmap_ktls_offload(family, sotype); - if (test__start_subtest("tls tx cork")) - test_sockmap_ktls_tx_cork(family, sotype, false); - if (test__start_subtest("tls tx cork with push")) - test_sockmap_ktls_tx_cork(family, sotype, true); - if (test__start_subtest("tls tx egress with no buf")) - test_sockmap_ktls_tx_no_buf(family, sotype, true); - if (test__start_subtest("tls tx with pop")) - test_sockmap_ktls_tx_pop(family, sotype); } void test_sockmap_ktls(void) diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c index cc0c68bab907..1c96a3cf4b97 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c @@ -53,8 +53,8 @@ static void test_insert_opened(struct test_sockmap_listen *skel __always_unused, int family, int sotype, int mapfd) { u32 key = 0; - u64 value; int err, s; + u64 value; s = xsocket(family, sotype, 0); if (s == -1) @@ -63,11 +63,8 @@ static void test_insert_opened(struct test_sockmap_listen *skel __always_unused, errno = 0; value = s; err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); - if (sotype == SOCK_STREAM) { - if (!err || errno != EOPNOTSUPP) - FAIL_ERRNO("map_update: expected EOPNOTSUPP"); - } else if (err) - FAIL_ERRNO("map_update: expected success"); + ASSERT_ERR(err, "map_update"); + ASSERT_EQ(errno, EOPNOTSUPP, "errno"); xclose(s); } @@ -77,8 +74,8 @@ static void test_insert_bound(struct test_sockmap_listen *skel __always_unused, struct sockaddr_storage addr; socklen_t len = 0; u32 key = 0; - u64 value; int err, s; + u64 value; init_addr_loopback(family, &addr, &len); @@ -93,8 +90,12 @@ static void test_insert_bound(struct test_sockmap_listen *skel __always_unused, errno = 0; value = s; err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); - if (!err || errno != EOPNOTSUPP) - FAIL_ERRNO("map_update: expected EOPNOTSUPP"); + if (sotype == SOCK_STREAM) { + ASSERT_ERR(err, "map_update"); + ASSERT_EQ(errno, EOPNOTSUPP, "errno"); + } else { + ASSERT_OK(err, "map_update"); + } close: xclose(s); } @@ -1289,7 +1290,7 @@ static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map, /* insert */ TEST(test_insert_invalid), TEST(test_insert_opened), - TEST(test_insert_bound, SOCK_STREAM), + TEST(test_insert_bound), TEST(test_insert), /* delete */ TEST(test_delete_after_insert), diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c b/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c index 621b3b71888e..1d7231728eaf 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_strp.c @@ -431,6 +431,35 @@ out: test_sockmap_strp__destroy(strp); } +static void test_sockmap_strp_parser_reject(void) +{ + struct test_sockmap_strp *strp = NULL; + int parser_mod, parser_ro, link; + int err, map; + + strp = test_sockmap_strp__open_and_load(); + if (!ASSERT_OK_PTR(strp, "test_sockmap_strp__open_and_load")) + return; + + map = bpf_map__fd(strp->maps.sock_map); + parser_mod = bpf_program__fd(strp->progs.prog_skb_parser_resize); + parser_ro = bpf_program__fd(strp->progs.prog_skb_parser); + + err = bpf_prog_attach(parser_mod, map, BPF_SK_SKB_STREAM_PARSER, 0); + ASSERT_ERR(err, "bpf_prog_attach parser_mod"); + + link = bpf_link_create(parser_ro, map, BPF_SK_SKB_STREAM_PARSER, NULL); + if (!ASSERT_GE(link, 0, "bpf_link_create parser_ro")) + goto out; + + err = bpf_link_update(link, parser_mod, NULL); + ASSERT_ERR(err, "bpf_link_update parser_mod"); +out: + if (link >= 0) + close(link); + test_sockmap_strp__destroy(strp); +} + void test_sockmap_strp(void) { if (test__start_subtest("sockmap strp tcp pass")) @@ -451,4 +480,6 @@ void test_sockmap_strp(void) test_sockmap_strp_multiple_pkt(AF_INET, SOCK_STREAM); if (test__start_subtest("sockmap strp tcp dispatch")) test_sockmap_strp_dispatch_pkt(AF_INET, SOCK_STREAM); + if (test__start_subtest("sockmap strp parser reject pkt mod")) + test_sockmap_strp_parser_reject(); } diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt.c b/tools/testing/selftests/bpf/prog_tests/sockopt.c index eaac83a7f388..6c96f2d9fccf 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/tcp.h> #include <test_progs.h> #include <io_uring/mini_liburing.h> #include "cgroup_helpers.h" @@ -284,6 +285,27 @@ static struct sockopt_test { .io_uring_support = true, }, { + .descr = "getsockopt: deny negative ctx->optlen in TCP_ZEROCOPY_RECEIVE", + .insns = { + /* ctx->optlen = -1 */ + BPF_MOV64_IMM(BPF_REG_0, -1), + BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, + offsetof(struct bpf_sockopt, optlen)), + + /* return 1 */ + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_EXIT_INSN(), + }, + .attach_type = BPF_CGROUP_GETSOCKOPT, + .expected_attach_type = BPF_CGROUP_GETSOCKOPT, + + .get_level = IPPROTO_TCP, + .get_optname = TCP_ZEROCOPY_RECEIVE, + .get_optlen = sizeof(struct tcp_zerocopy_receive), + + .error = EFAULT_GETSOCKOPT, + }, + { .descr = "getsockopt: ignore >PAGE_SIZE optlen", .insns = { /* write 0xFF to the first optval byte */ diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c index 53637431ec5d..3a41c517b918 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c @@ -190,7 +190,7 @@ static int getsetsockopt(void) fd = socket(AF_NETLINK, SOCK_RAW, 0); if (fd < 0) { log_err("Failed to create AF_NETLINK socket"); - return -1; + goto err; } buf.u32 = 1; @@ -211,6 +211,21 @@ static int getsetsockopt(void) } ASSERT_EQ(optlen, 8, "Unexpected NETLINK_LIST_MEMBERSHIPS value"); + /* Trick bpf_tcp_sock() with IPPROTO_TCP */ + close(fd); + fd = socket(AF_INET, SOCK_RAW, IPPROTO_TCP); + if (!ASSERT_OK_FD(fd, "socket")) + goto err; + + /* The BPF prog intercepts this before the kernel sees it, any + * optlen works. Go with 4 bytes for simplicity. + */ + buf.u32 = 1; + optlen = sizeof(buf.u32); + err = setsockopt(fd, SOL_TCP, TCP_SAVED_SYN, &buf, optlen); + if (!ASSERT_ERR(err, "setsockopt(TCP_SAVED_SYN)")) + goto err; + free(big_buf); close(fd); return 0; diff --git a/tools/testing/selftests/bpf/prog_tests/spin_lock.c b/tools/testing/selftests/bpf/prog_tests/spin_lock.c index 254fbfeab06a..e368370262c8 100644 --- a/tools/testing/selftests/bpf/prog_tests/spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/spin_lock.c @@ -13,8 +13,9 @@ static struct { const char *err_msg; } spin_lock_fail_tests[] = { { "lock_id_kptr_preserve", - "5: (bf) r1 = r0 ; R0=ptr_foo(id=2,ref_obj_id=2) " - "R1=ptr_foo(id=2,ref_obj_id=2) refs=2\n6: (85) call bpf_this_cpu_ptr#154\n" + "[0-9]\\+: (bf) r1 = r0 ; R0=ptr_foo(id=2)" + " R1=ptr_foo(id=2) refs=2\n" + "[0-9]\\+: (85) call bpf_this_cpu_ptr#154\n" "R1 type=ptr_ expected=percpu_ptr_" }, { "lock_id_global_zero", "; R1=map_value(map=.data.A,ks=4,vs=4)\n2: (85) call bpf_this_cpu_ptr#154\n" @@ -53,6 +54,8 @@ static struct { { "lock_global_sleepable_helper_subprog", "global function calls are not allowed while holding a lock" }, { "lock_global_sleepable_kfunc_subprog", "global function calls are not allowed while holding a lock" }, { "lock_global_sleepable_subprog_indirect", "global function calls are not allowed while holding a lock" }, + { "callback_value_lock_identity", "bpf_spin_unlock of different lock" }, + { "callback_inner_map_value_lock_identity", "bpf_spin_unlock of different lock" }, }; static int match_regex(const char *pattern, const char *string) diff --git a/tools/testing/selftests/bpf/prog_tests/stack_arg.c b/tools/testing/selftests/bpf/prog_tests/stack_arg.c new file mode 100644 index 000000000000..57193543f260 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/stack_arg.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include <network_helpers.h> +#include "stack_arg.skel.h" +#include "stack_arg_kfunc.skel.h" + +static void run_subtest(struct bpf_program *prog, int expected) +{ + int err, prog_fd; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .repeat = 1, + ); + + prog_fd = bpf_program__fd(prog); + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, expected, "retval"); +} + +static void test_global_many(void) +{ + struct stack_arg *skel; + + skel = stack_arg__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + if (!skel->rodata->has_stack_arg) { + test__skip(); + goto out; + } + + if (!ASSERT_OK(stack_arg__load(skel), "load")) + goto out; + + run_subtest(skel->progs.test_global_many_args, 55); + +out: + stack_arg__destroy(skel); +} + +static void test_async_cb_many(void) +{ + struct stack_arg *skel; + + skel = stack_arg__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + if (!skel->rodata->has_stack_arg) { + test__skip(); + goto out; + } + + if (!ASSERT_OK(stack_arg__load(skel), "load")) + goto out; + + run_subtest(skel->progs.test_async_cb_many_args, 0); + + /* Wait for the timer callback to fire and verify the result. + * 10+20+30+40+50+60+70+80+90+100 = 550 + */ + usleep(50); + ASSERT_EQ(skel->bss->timer_result, 550, "timer_result"); + +out: + stack_arg__destroy(skel); +} + +static void test_bpf2bpf(void) +{ + struct stack_arg *skel; + + skel = stack_arg__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + if (!skel->rodata->has_stack_arg) { + test__skip(); + goto out; + } + + if (!ASSERT_OK(stack_arg__load(skel), "load")) + goto out; + + run_subtest(skel->progs.test_bpf2bpf_ptr_stack_arg, 75); + run_subtest(skel->progs.test_bpf2bpf_mix_stack_args, 66); + run_subtest(skel->progs.test_bpf2bpf_nesting_stack_arg, 84); + run_subtest(skel->progs.test_bpf2bpf_dynptr_stack_arg, 99); + run_subtest(skel->progs.test_two_callees, 133); + +out: + stack_arg__destroy(skel); +} + +static void test_kfunc(void) +{ + struct stack_arg_kfunc *skel; + + skel = stack_arg_kfunc__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + if (!skel->rodata->has_stack_arg) { + test__skip(); + goto out; + } + + if (!ASSERT_OK(stack_arg_kfunc__load(skel), "load")) + goto out; + + run_subtest(skel->progs.test_stack_arg_scalar, 55); + run_subtest(skel->progs.test_stack_arg_ptr, 75); + run_subtest(skel->progs.test_stack_arg_mix, 66); + run_subtest(skel->progs.test_stack_arg_dynptr, 99); + run_subtest(skel->progs.test_stack_arg_mem, 151); + run_subtest(skel->progs.test_stack_arg_iter, 145); + run_subtest(skel->progs.test_stack_arg_const_str, 45); + run_subtest(skel->progs.test_stack_arg_timer, 45); + +out: + stack_arg_kfunc__destroy(skel); +} + +void test_stack_arg(void) +{ + if (test__start_subtest("global_many_args")) + test_global_many(); + if (test__start_subtest("async_cb_many_args")) + test_async_cb_many(); + if (test__start_subtest("bpf2bpf")) + test_bpf2bpf(); + if (test__start_subtest("kfunc")) + test_kfunc(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/stack_arg_fail.c b/tools/testing/selftests/bpf/prog_tests/stack_arg_fail.c new file mode 100644 index 000000000000..090af1330953 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/stack_arg_fail.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include "stack_arg_fail.skel.h" + +void test_stack_arg_fail(void) +{ + RUN_TESTS(stack_arg_fail); +} diff --git a/tools/testing/selftests/bpf/prog_tests/stack_arg_precision.c b/tools/testing/selftests/bpf/prog_tests/stack_arg_precision.c new file mode 100644 index 000000000000..1ab041d66de3 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/stack_arg_precision.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include "stack_arg_precision.skel.h" + +void test_stack_arg_precision(void) +{ + RUN_TESTS(stack_arg_precision); +} diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index c3cce5c292bd..e4e9374309e2 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c @@ -103,6 +103,12 @@ void test_stream_arena_fault_address(void) test_address(skel->progs.stream_arena_read_fault, &skel->bss->fault_addr); if (test__start_subtest("write_fault")) test_address(skel->progs.stream_arena_write_fault, &skel->bss->fault_addr); + if (test__start_subtest("load_acquire_fault")) + test_address(skel->progs.stream_arena_load_acquire_fault, &skel->bss->fault_addr); + if (test__start_subtest("xchg_fault")) + test_address(skel->progs.stream_arena_xchg_fault, &skel->bss->fault_addr); + if (test__start_subtest("cmpxchg_fault")) + test_address(skel->progs.stream_arena_cmpxchg_fault, &skel->bss->fault_addr); stream__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c index d42123a0fb16..98db9bafa44b 100644 --- a/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c +++ b/tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c @@ -5,6 +5,7 @@ #include "struct_ops_private_stack_fail.skel.h" #include "struct_ops_private_stack_recur.skel.h" +#if defined(__x86_64__) || defined(__aarch64__) || defined(__powerpc64__) static void test_private_stack(void) { struct struct_ops_private_stack *skel; @@ -15,11 +16,6 @@ static void test_private_stack(void) if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack__open")) return; - if (skel->data->skip) { - test__skip(); - goto cleanup; - } - err = struct_ops_private_stack__load(skel); if (!ASSERT_OK(err, "struct_ops_private_stack__load")) goto cleanup; @@ -48,15 +44,9 @@ static void test_private_stack_fail(void) if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_fail__open")) return; - if (skel->data->skip) { - test__skip(); - goto cleanup; - } - err = struct_ops_private_stack_fail__load(skel); ASSERT_ERR(err, "struct_ops_private_stack_fail__load"); -cleanup: struct_ops_private_stack_fail__destroy(skel); } @@ -70,11 +60,6 @@ static void test_private_stack_recur(void) if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_recur__open")) return; - if (skel->data->skip) { - test__skip(); - goto cleanup; - } - err = struct_ops_private_stack_recur__load(skel); if (!ASSERT_OK(err, "struct_ops_private_stack_recur__load")) goto cleanup; @@ -93,7 +78,7 @@ cleanup: struct_ops_private_stack_recur__destroy(skel); } -void test_struct_ops_private_stack(void) +static void __test_struct_ops_private_stack(void) { if (test__start_subtest("private_stack")) test_private_stack(); @@ -102,3 +87,14 @@ void test_struct_ops_private_stack(void) if (test__start_subtest("private_stack_recur")) test_private_stack_recur(); } +#else +static void __test_struct_ops_private_stack(void) +{ + test__skip(); +} +#endif + +void test_struct_ops_private_stack(void) +{ + __test_struct_ops_private_stack(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/summarization.c b/tools/testing/selftests/bpf/prog_tests/summarization.c index 5dd6c120a838..6951786044ca 100644 --- a/tools/testing/selftests/bpf/prog_tests/summarization.c +++ b/tools/testing/selftests/bpf/prog_tests/summarization.c @@ -58,7 +58,7 @@ static void test_aux(const char *main_prog_name, * this particular combination can be enabled. */ if (!strcmp("might_sleep", replacement) && err) { - ASSERT_HAS_SUBSTR(log, "helper call might sleep in a non-sleepable prog", "error log"); + ASSERT_HAS_SUBSTR(log, "sleepable helper bpf_copy_from_user#", "error log"); ASSERT_EQ(err, -EINVAL, "err"); test__skip(); goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index 7d534fde0af9..c5c9d6c359bb 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -8,7 +8,13 @@ #include "tailcall_freplace.skel.h" #include "tc_bpf2bpf.skel.h" #include "tailcall_fail.skel.h" +#include "tailcall_cgrp_storage_owner.skel.h" +#include "tailcall_cgrp_storage_no_storage.skel.h" +#include "tailcall_cgrp_storage.skel.h" #include "tailcall_sleepable.skel.h" +#include "tailcall_callback.skel.h" +#include "tailcall_bpf2bpf2.skel.h" +#include "tailcall_bpf2bpf_fexit.skel.h" /* test_tailcall_1 checks basic functionality by patching multiple locations * in a single program for a single tail call slot with nop->jmp, jmp->nop @@ -1654,6 +1660,179 @@ static void test_tailcall_failure() RUN_TESTS(tailcall_fail); } +static void test_tailcall_cgrp_storage(void) +{ + struct tailcall_cgrp_storage_owner *owner_skel = NULL; + struct tailcall_cgrp_storage *skel = NULL; + int err, key = 0, prog_array_fd, prog_fd, storage_map_fd; + + owner_skel = tailcall_cgrp_storage_owner__open_and_load(); + if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) + return; + + prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); + storage_map_fd = bpf_map__fd(owner_skel->maps.storage_map); + + skel = tailcall_cgrp_storage__open(); + if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage__open")) + goto out; + + err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd); + if (!ASSERT_OK(err, "reuse_prog_array")) + goto out; + + err = bpf_map__reuse_fd(skel->maps.storage_map, storage_map_fd); + if (!ASSERT_OK(err, "reuse_storage_map")) + goto out; + + err = bpf_object__load(skel->obj); + if (!ASSERT_OK(err, "tailcall_cgrp_storage__load")) + goto out; + + prog_fd = bpf_program__fd(skel->progs.callee_prog); + err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); + ASSERT_OK(err, "update_prog_array"); +out: + tailcall_cgrp_storage__destroy(skel); + tailcall_cgrp_storage_owner__destroy(owner_skel); +} + +static void test_tailcall_cgrp_storage_diff_storage(void) +{ + struct tailcall_cgrp_storage_owner *owner_skel = NULL; + struct tailcall_cgrp_storage *skel = NULL; + int err, prog_array_fd; + + owner_skel = tailcall_cgrp_storage_owner__open_and_load(); + if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) + return; + + prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); + + skel = tailcall_cgrp_storage__open(); + if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage__open")) + goto out; + + err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd); + if (!ASSERT_OK(err, "reuse_prog_array")) + goto out; + + err = bpf_object__load(skel->obj); + ASSERT_ERR(err, "tailcall_cgrp_storage__load"); +out: + tailcall_cgrp_storage__destroy(skel); + tailcall_cgrp_storage_owner__destroy(owner_skel); +} + +static void test_tailcall_cgrp_storage_no_storage(void) +{ + struct tailcall_cgrp_storage_owner *owner_skel = NULL; + struct tailcall_cgrp_storage_no_storage *skel = NULL; + int err, prog_array_fd; + + owner_skel = tailcall_cgrp_storage_owner__open_and_load(); + if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) + return; + + prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); + + skel = tailcall_cgrp_storage_no_storage__open(); + if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage_no_storage__open")) + goto out; + + err = bpf_map__reuse_fd(skel->maps.prog_array, prog_array_fd); + if (!ASSERT_OK(err, "reuse_prog_array")) + goto out; + + err = bpf_object__load(skel->obj); + ASSERT_ERR(err, "tailcall_cgrp_storage_no_storage__load"); +out: + tailcall_cgrp_storage_no_storage__destroy(skel); + tailcall_cgrp_storage_owner__destroy(owner_skel); +} + +static void test_tailcall_cgrp_storage_no_storage_leaf(void) +{ + struct tailcall_cgrp_storage_owner *owner_skel = NULL; + struct tailcall_cgrp_storage_no_storage *skel = NULL; + int err, key = 0, prog_array_fd, prog_fd; + + owner_skel = tailcall_cgrp_storage_owner__open_and_load(); + if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) + return; + + prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); + + skel = tailcall_cgrp_storage_no_storage__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tailcall_cgrp_storage_no_storage__open_and_load")) + goto out; + + prog_fd = bpf_program__fd(skel->progs.leaf_prog); + err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); + if (!ASSERT_OK(err, "update_prog_array_leaf")) + goto out; + + prog_fd = bpf_program__fd(skel->progs.caller_prog); + err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); + ASSERT_ERR(err, "update_prog_array_bridge"); +out: + tailcall_cgrp_storage_no_storage__destroy(skel); + tailcall_cgrp_storage_owner__destroy(owner_skel); +} + +static void test_tailcall_cgrp_storage_no_storage_bridge(void) +{ + struct tailcall_cgrp_storage_owner *owner_skel = NULL; + struct tailcall_cgrp_storage_no_storage *bridge_skel = NULL; + struct tailcall_cgrp_storage *callee_skel = NULL; + int err, key = 0, prog_array_fd, prog_fd, storage_map_fd; + + owner_skel = tailcall_cgrp_storage_owner__open_and_load(); + if (!ASSERT_OK_PTR(owner_skel, "owner_open_and_load")) + return; + + prog_array_fd = bpf_map__fd(owner_skel->maps.prog_array); + storage_map_fd = bpf_map__fd(owner_skel->maps.storage_map); + + callee_skel = tailcall_cgrp_storage__open(); + if (!ASSERT_OK_PTR(callee_skel, "tailcall_cgrp_storage__open")) + goto out; + + bpf_program__set_autoload(callee_skel->progs.caller_prog, false); + + err = bpf_map__reuse_fd(callee_skel->maps.prog_array, prog_array_fd); + if (!ASSERT_OK(err, "reuse_prog_array")) + goto out; + + err = bpf_map__reuse_fd(callee_skel->maps.storage_map, storage_map_fd); + if (!ASSERT_OK(err, "reuse_storage_map")) + goto out; + + err = bpf_object__load(callee_skel->obj); + if (!ASSERT_OK(err, "tailcall_cgrp_storage__load")) + goto out; + + prog_fd = bpf_program__fd(callee_skel->progs.callee_prog); + err = bpf_map_update_elem(prog_array_fd, &key, &prog_fd, BPF_ANY); + if (!ASSERT_OK(err, "update_prog_array")) + goto out; + + bridge_skel = tailcall_cgrp_storage_no_storage__open(); + if (!ASSERT_OK_PTR(bridge_skel, "tailcall_cgrp_storage_no_storage__open")) + goto out; + + err = bpf_map__reuse_fd(bridge_skel->maps.prog_array, prog_array_fd); + if (!ASSERT_OK(err, "reuse_prog_array")) + goto out; + + err = bpf_object__load(bridge_skel->obj); + ASSERT_ERR(err, "tailcall_cgrp_storage_no_storage_bridge__load"); +out: + tailcall_cgrp_storage_no_storage__destroy(bridge_skel); + tailcall_cgrp_storage__destroy(callee_skel); + tailcall_cgrp_storage_owner__destroy(owner_skel); +} + noinline void uprobe_sleepable_trigger(void) { asm volatile (""); @@ -1725,6 +1904,55 @@ out: tailcall_sleepable__destroy(skel); } +static void test_tailcall_callback(void) +{ + RUN_TESTS(tailcall_callback); +} + +static void test_tailcall_bpf2bpf_fexit_links(void) +{ + struct tailcall_bpf2bpf_fexit *skel1 = NULL, *skel2 = NULL; + struct tailcall_bpf2bpf2 *skel_tc; + int err, prog_fd; + + skel_tc = tailcall_bpf2bpf2__open_and_load(); + if (!ASSERT_OK_PTR(skel_tc, "tailcall_bpf2bpf2__open_and_load")) + return; + + skel1 = tailcall_bpf2bpf_fexit__open(); + if (!ASSERT_OK_PTR(skel1, "tailcall_bpf2bpf_fexit__open")) + goto out; + + prog_fd = bpf_program__fd(skel_tc->progs.classifier_0); + err = bpf_program__set_attach_target(skel1->progs.fexit, prog_fd, "subprog_tail"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = tailcall_bpf2bpf_fexit__load(skel1); + if (!ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load")) + goto out; + + skel1->links.fexit = bpf_program__attach_trace(skel1->progs.fexit); + if (!ASSERT_OK_PTR(skel1->links.fexit, "bpf_program__attach_trace")) + goto out; + + skel2 = tailcall_bpf2bpf_fexit__open(); + if (!ASSERT_OK_PTR(skel2, "tailcall_bpf2bpf_fexit__open")) + goto out; + + err = bpf_program__set_attach_target(skel2->progs.fexit, prog_fd, "subprog_tail"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = tailcall_bpf2bpf_fexit__load(skel2); + ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load"); + +out: + tailcall_bpf2bpf_fexit__destroy(skel1); + tailcall_bpf2bpf_fexit__destroy(skel2); + tailcall_bpf2bpf2__destroy(skel_tc); +} + void test_tailcalls(void) { if (test__start_subtest("tailcall_1")) @@ -1781,4 +2009,17 @@ void test_tailcalls(void) test_tailcall_failure(); if (test__start_subtest("tailcall_sleepable")) test_tailcall_sleepable(); + if (test__start_subtest("tailcall_cgrp_storage")) + test_tailcall_cgrp_storage(); + if (test__start_subtest("tailcall_cgrp_storage_diff_storage")) + test_tailcall_cgrp_storage_diff_storage(); + if (test__start_subtest("tailcall_cgrp_storage_no_storage")) + test_tailcall_cgrp_storage_no_storage(); + if (test__start_subtest("tailcall_cgrp_storage_no_storage_leaf")) + test_tailcall_cgrp_storage_no_storage_leaf(); + if (test__start_subtest("tailcall_cgrp_storage_no_storage_bridge")) + test_tailcall_cgrp_storage_no_storage_bridge(); + test_tailcall_callback(); + if (test__start_subtest("tailcall_bpf2bpf_fexit_links")) + test_tailcall_bpf2bpf_fexit_links(); } diff --git a/tools/testing/selftests/bpf/prog_tests/task_kfunc.c b/tools/testing/selftests/bpf/prog_tests/task_kfunc.c index 83b90335967a..30d403028f98 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_kfunc.c +++ b/tools/testing/selftests/bpf/prog_tests/task_kfunc.c @@ -68,6 +68,36 @@ cleanup: task_kfunc_success__destroy(skel); } +static void run_syscall_success_test(const char *prog_name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct task_kfunc_success *skel; + struct bpf_program *prog; + int err; + + skel = open_load_task_kfunc_skel(); + if (!ASSERT_OK_PTR(skel, "open_load_skel")) + return; + + if (!ASSERT_OK(skel->bss->err, "pre_run_err")) + goto cleanup; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto cleanup; + if (!ASSERT_EQ(opts.retval, 0, "retval")) + goto cleanup; + + ASSERT_OK(skel->bss->err, "post_run_err"); + +cleanup: + task_kfunc_success__destroy(skel); +} + static int run_vpid_test(void *prog_name) { struct task_kfunc_success *skel; @@ -140,17 +170,28 @@ static const char * const success_tests[] = { "test_task_acquire_release_argument", "test_task_acquire_release_current", "test_task_acquire_leave_in_map", - "test_task_xchg_release", "test_task_map_acquire_release", "test_task_current_acquire_release", "test_task_from_pid_arg", "test_task_from_pid_current", "test_task_from_pid_invalid", "task_kfunc_acquire_trusted_walked", + "task_kfunc_acquire_after_spin_unlock_non_sleepable", + "task_kfunc_acquire_after_spin_unlock_explicit_rcu", + "task_kfunc_acquire_after_spin_unlock_preempt_disabled", + "task_kfunc_acquire_after_spin_unlock_irq_disabled", + "task_kfunc_acquire_after_rcu_unlock_preempt_disabled", + "task_kfunc_acquire_after_rcu_unlock_irq_disabled", + "task_kfunc_acquire_after_preempt_enable_explicit_rcu", + "task_kfunc_acquire_after_irq_restore_explicit_rcu", "test_task_kfunc_flavor_relo", "test_task_kfunc_flavor_relo_not_found", }; +static const char * const syscall_success_tests[] = { + "test_task_xchg_release", +}; + static const char * const vpid_success_tests[] = { "test_task_from_vpid_current", "test_task_from_vpid_invalid", @@ -167,6 +208,13 @@ void test_task_kfunc(void) run_success_test(success_tests[i]); } + for (i = 0; i < ARRAY_SIZE(syscall_success_tests); i++) { + if (!test__start_subtest(syscall_success_tests[i])) + continue; + + run_syscall_success_test(syscall_success_tests[i]); + } + for (i = 0; i < ARRAY_SIZE(vpid_success_tests); i++) { if (!test__start_subtest(vpid_success_tests[i])) continue; diff --git a/tools/testing/selftests/bpf/prog_tests/task_local_data.h b/tools/testing/selftests/bpf/prog_tests/task_local_data.h index 8342e2fe5260..8ae4fb2027f7 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_local_data.h +++ b/tools/testing/selftests/bpf/prog_tests/task_local_data.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ #ifndef __TASK_LOCAL_DATA_H #define __TASK_LOCAL_DATA_H @@ -22,14 +22,17 @@ /* * OPTIONS * - * Define the option before including the header + * Define the option before including the header. Using different options in + * different translation units is strongly discouraged. * * TLD_FREE_DATA_ON_THREAD_EXIT - Frees memory on thread exit automatically * * Thread-specific memory for storing TLD is allocated lazily on the first call to * tld_get_data(). The thread that calls it must also call tld_free() on thread exit * to prevent memory leak. Pthread will be included if the option is defined. A pthread - * key will be registered with a destructor that calls tld_free(). + * key will be registered with a destructor that calls tld_free(). Enabled only when + * the option is defined and TLD_DEFINE_KEY/tld_create_key() is called in the same + * translation unit. * * * TLD_DYN_DATA_SIZE - The maximum size of memory allocated for TLDs created dynamically @@ -47,19 +50,16 @@ * TLD_NAME_LEN - The maximum length of the name of a TLD (default: 62) * * Setting TLD_NAME_LEN will affect the maximum number of TLDs a process can store, - * TLD_MAX_DATA_CNT. + * TLD_MAX_DATA_CNT. Must be consistent with task_local_data.bpf.h. * * - * TLD_DATA_USE_ALIGNED_ALLOC - Always use aligned_alloc() instead of malloc() + * TLD_DONT_ROUND_UP_DATA_SIZE - Don't round up memory size allocated for data if + * the memory allocator has low overhead aligned_alloc() implementation. * - * When allocating the memory for storing TLDs, we need to make sure there is a memory - * region of the X bytes within a page. This is due to the limit posed by UPTR: memory - * pinned to the kernel cannot exceed a page nor can it cross the page boundary. The - * library normally calls malloc(2*X) given X bytes of total TLDs, and only uses - * aligned_alloc(PAGE_SIZE, X) when X >= PAGE_SIZE / 2. This is to reduce memory wastage - * as not all memory allocator can use the exact amount of memory requested to fulfill - * aligned_alloc(). For example, some may round the size up to the alignment. Enable the - * option to always use aligned_alloc() if the implementation has low memory overhead. + * For some memory allocators, when calling aligned_alloc(alignment, size), size + * does not need to be an integral multiple of alignment and it can be fulfilled + * without using round_up(size, alignment) bytes of memory. Enable this option to + * reduce memory usage. */ #define TLD_PAGE_SIZE getpagesize() @@ -68,7 +68,7 @@ #define TLD_ROUND_MASK(x, y) ((__typeof__(x))((y) - 1)) #define TLD_ROUND_UP(x, y) ((((x) - 1) | TLD_ROUND_MASK(x, y)) + 1) -#define TLD_READ_ONCE(x) (*(volatile typeof(x) *)&(x)) +#define TLD_ROUND_UP_POWER_OF_TWO(x) (1UL << (sizeof(x) * 8 - __builtin_clzl(x - 1))) #ifndef TLD_DYN_DATA_SIZE #define TLD_DYN_DATA_SIZE 64 @@ -90,7 +90,7 @@ typedef struct { struct tld_metadata { char name[TLD_NAME_LEN]; - _Atomic __u16 size; + _Atomic __u16 size; /* size of tld_data_u->data */ }; struct tld_meta_u { @@ -99,27 +99,34 @@ struct tld_meta_u { struct tld_metadata metadata[]; }; +/* + * The unused field ensures map_val.start > 0. On the BPF side, __tld_fetch_key() + * calculates off by summing map_val.start and tld_key_t.off and treats off == 0 + * as key not cached. + */ struct tld_data_u { - __u64 start; /* offset of tld_data_u->data in a page */ - char data[]; + __u64 unused; + char data[] __attribute__((aligned(8))); }; struct tld_map_value { void *data; struct tld_meta_u *meta; + __u16 start; /* offset of tld_data_u->data in a page */ }; struct tld_meta_u * _Atomic tld_meta_p __attribute__((weak)); __thread struct tld_data_u *tld_data_p __attribute__((weak)); -__thread void *tld_data_alloc_p __attribute__((weak)); #ifdef TLD_FREE_DATA_ON_THREAD_EXIT +bool _Atomic tld_pthread_key_init __attribute__((weak)); pthread_key_t tld_pthread_key __attribute__((weak)); static void tld_free(void); static void __tld_thread_exit_handler(void *unused) { + (void)unused; tld_free(); } #endif @@ -143,20 +150,16 @@ static int __tld_init_meta_p(void) goto out; } -#ifdef TLD_FREE_DATA_ON_THREAD_EXIT - pthread_key_create(&tld_pthread_key, __tld_thread_exit_handler); -#endif out: return err; } static int __tld_init_data_p(int map_fd) { - bool use_aligned_alloc = false; struct tld_map_value map_val; struct tld_data_u *data; - void *data_alloc = NULL; int err, tid_fd = -1; + size_t size, size_pot; tid_fd = syscall(SYS_pidfd_open, sys_gettid(), O_EXCL); if (tid_fd < 0) { @@ -164,47 +167,37 @@ static int __tld_init_data_p(int map_fd) goto out; } -#ifdef TLD_DATA_USE_ALIGNED_ALLOC - use_aligned_alloc = true; -#endif - /* * tld_meta_p->size = TLD_DYN_DATA_SIZE + * total size of TLDs defined via TLD_DEFINE_KEY() */ - data_alloc = (use_aligned_alloc || tld_meta_p->size * 2 >= TLD_PAGE_SIZE) ? - aligned_alloc(TLD_PAGE_SIZE, tld_meta_p->size) : - malloc(tld_meta_p->size * 2); - if (!data_alloc) { + size = tld_meta_p->size + sizeof(struct tld_data_u); + size_pot = TLD_ROUND_UP_POWER_OF_TWO(size); +#ifdef TLD_DONT_ROUND_UP_DATA_SIZE + data = (struct tld_data_u *)aligned_alloc(size_pot, size); +#else + data = (struct tld_data_u *)aligned_alloc(size_pot, size_pot); +#endif + if (!data) { err = -ENOMEM; goto out; } /* * Always pass a page-aligned address to UPTR since the size of tld_map_value::data - * is a page in BTF. If data_alloc spans across two pages, use the page that contains large - * enough memory. + * is a page in BTF. */ - if (TLD_PAGE_SIZE - (~TLD_PAGE_MASK & (intptr_t)data_alloc) >= tld_meta_p->size) { - map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data_alloc); - data = data_alloc; - data->start = (~TLD_PAGE_MASK & (intptr_t)data_alloc) + - offsetof(struct tld_data_u, data); - } else { - map_val.data = (void *)(TLD_ROUND_UP((intptr_t)data_alloc, TLD_PAGE_SIZE)); - data = (void *)(TLD_ROUND_UP((intptr_t)data_alloc, TLD_PAGE_SIZE)); - data->start = offsetof(struct tld_data_u, data); - } - map_val.meta = TLD_READ_ONCE(tld_meta_p); + map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data); + map_val.start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u); + map_val.meta = tld_meta_p; err = bpf_map_update_elem(map_fd, &tid_fd, &map_val, 0); if (err) { - free(data_alloc); + free(data); goto out; } tld_data_p = data; - tld_data_alloc_p = data_alloc; #ifdef TLD_FREE_DATA_ON_THREAD_EXIT pthread_setspecific(tld_pthread_key, (void *)1); #endif @@ -217,15 +210,24 @@ out: static tld_key_t __tld_create_key(const char *name, size_t size, bool dyn_data) { int err, i, sz, off = 0; + bool uninit = false; __u16 cnt; - if (!TLD_READ_ONCE(tld_meta_p)) { + if (!tld_meta_p) { err = __tld_init_meta_p(); if (err) - return (tld_key_t){err}; + return (tld_key_t){(__s16)err}; + } + +#ifdef TLD_FREE_DATA_ON_THREAD_EXIT + if (atomic_compare_exchange_strong(&tld_pthread_key_init, &uninit, true)) { + err = pthread_key_create(&tld_pthread_key, __tld_thread_exit_handler); + if (err) + return (tld_key_t){(__s16)err}; } +#endif - for (i = 0; i < TLD_MAX_DATA_CNT; i++) { + for (i = 0; i < (int)TLD_MAX_DATA_CNT; i++) { retry: cnt = atomic_load(&tld_meta_p->cnt); if (i < cnt) { @@ -245,7 +247,8 @@ retry: * TLD_DYN_DATA_SIZE is allocated for tld_create_key() */ if (dyn_data) { - if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size) + if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size || + tld_meta_p->size > TLD_PAGE_SIZE - sizeof(struct tld_data_u)) return (tld_key_t){-E2BIG}; } else { if (off + TLD_ROUND_UP(size, 8) > TLD_PAGE_SIZE - sizeof(struct tld_data_u)) @@ -290,7 +293,7 @@ retry: #define TLD_DEFINE_KEY(key, name, size) \ tld_key_t key; \ \ -__attribute__((constructor)) \ +__attribute__((constructor(101))) \ void __tld_define_key_##key(void) \ { \ key = __tld_create_key(name, size, false); \ @@ -350,7 +353,7 @@ static inline int tld_key_err_or_zero(tld_key_t key) __attribute__((unused)) static void *tld_get_data(int map_fd, tld_key_t key) { - if (!TLD_READ_ONCE(tld_meta_p)) + if (!tld_meta_p) return NULL; /* tld_data_p is allocated on the first invocation of tld_get_data() */ @@ -367,14 +370,14 @@ static void *tld_get_data(int map_fd, tld_key_t key) * * Users must call tld_free() on thread exit to prevent memory leak. Alternatively, * define TLD_FREE_DATA_ON_THREAD_EXIT and a thread exit handler will be registered - * to free the memory automatically. + * to free the memory automatically. Calling tld_free() before thread exit is + * undefined behavior, which may lead to null-pointer dereference. */ __attribute__((unused)) static void tld_free(void) { - if (tld_data_alloc_p) { - free(tld_data_alloc_p); - tld_data_alloc_p = NULL; + if (tld_data_p) { + free(tld_data_p); tld_data_p = NULL; } } diff --git a/tools/testing/selftests/bpf/prog_tests/task_local_storage.c b/tools/testing/selftests/bpf/prog_tests/task_local_storage.c index 7bee33797c71..5b2b56cc3a4f 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_local_storage.c +++ b/tools/testing/selftests/bpf/prog_tests/task_local_storage.c @@ -25,24 +25,31 @@ static void test_sys_enter_exit(void) { struct task_local_storage *skel; + pid_t pid = sys_gettid(); int err; skel = task_local_storage__open_and_load(); if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) return; - skel->bss->target_pid = sys_gettid(); - err = task_local_storage__attach(skel); if (!ASSERT_OK(err, "skel_attach")) goto out; + /* Set target_pid after attach so that syscalls made during + * attach are not counted. + */ + skel->bss->target_pid = pid; + sys_gettid(); sys_gettid(); - /* 3x syscalls: 1x attach and 2x gettid */ - ASSERT_EQ(skel->bss->enter_cnt, 3, "enter_cnt"); - ASSERT_EQ(skel->bss->exit_cnt, 3, "exit_cnt"); + skel->bss->target_pid = 0; + + /* 2x gettid syscalls */ + ASSERT_EQ(skel->bss->update_err, 0, "update_err"); + ASSERT_EQ(skel->bss->enter_cnt, 2, "enter_cnt"); + ASSERT_EQ(skel->bss->exit_cnt, 2, "exit_cnt"); ASSERT_EQ(skel->bss->mismatch_cnt, 0, "mismatch_cnt"); out: task_local_storage__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/tc_change_tail_pmtu.c b/tools/testing/selftests/bpf/prog_tests/tc_change_tail_pmtu.c new file mode 100644 index 000000000000..7acdbd5757a9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tc_change_tail_pmtu.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <netinet/tcp.h> + +#include "test_progs.h" +#include "network_helpers.h" +#include "test_tc_change_tail_pmtu.skel.h" + +#define CLIENT_NS "tc-change-tail-cli-ns" +#define SERVER_NS "tc-change-tail-srv-ns" +#define CLIENT_IP "192.168.1.1" +#define SERVER_IP "192.168.1.2" + +#define TEST_PMTU 1000 +#define TEST_MSS_MAX (TEST_PMTU - 20 - 20) +#define TIMEOUT_MS 3000 +#define XFER_BYTES 8192 + +void test_tc_change_tail_pmtu(void) +{ + LIBBPF_OPTS(bpf_tcx_opts, tcx_opts); + int mss_before = 0, mss_after = 0, ifindex, port; + int srv_fd = -1, srv_conn_fd = -1, cli_fd = -1; + struct test_tc_change_tail_pmtu *skel = NULL; + struct nstoken *nstoken = NULL; + static char buf[XFER_BYTES]; + socklen_t optlen; + ssize_t bytes; + size_t total; + + if (!ASSERT_OK(make_netns(CLIENT_NS), "make client ns")) + return; + if (!ASSERT_OK(make_netns(SERVER_NS), "make server ns")) + goto out_client_ns; + + nstoken = open_netns(CLIENT_NS); + if (!ASSERT_OK_PTR(nstoken, "open client ns")) + goto out; + SYS(out, "ip link add veth1 type veth peer name veth2 netns " SERVER_NS); + SYS(out, "ip -4 addr add " CLIENT_IP "/24 dev veth1"); + SYS(out, "ip link set veth1 up"); + ifindex = if_nametoindex("veth1"); + if (!ASSERT_NEQ(ifindex, 0, "if_nametoindex")) + goto out; + close_netns(nstoken); + nstoken = NULL; + + nstoken = open_netns(SERVER_NS); + if (!ASSERT_OK_PTR(nstoken, "open server ns")) + goto out; + SYS(out, "ip -4 addr add " SERVER_IP "/24 dev veth2"); + SYS(out, "ip link set veth2 up"); + srv_fd = start_server(AF_INET, SOCK_STREAM, SERVER_IP, 0, TIMEOUT_MS); + if (!ASSERT_OK_FD(srv_fd, "start server")) + goto out; + close_netns(nstoken); + nstoken = NULL; + + skel = test_tc_change_tail_pmtu__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load skeleton")) + goto out; + + port = get_socket_local_port(srv_fd); + if (!ASSERT_GE(port, 0, "get server port")) + goto out; + + skel->bss->server_port = port; + skel->bss->pmtu = TEST_PMTU; + + nstoken = open_netns(CLIENT_NS); + if (!ASSERT_OK_PTR(nstoken, "open client ns")) + goto out; + + skel->links.change_tail_icmp = + bpf_program__attach_tcx(skel->progs.change_tail_icmp, ifindex, + &tcx_opts); + if (!ASSERT_OK_PTR(skel->links.change_tail_icmp, "attach tcx")) + goto out; + + cli_fd = connect_to_fd(srv_fd, TIMEOUT_MS); + if (!ASSERT_OK_FD(cli_fd, "connect to server")) + goto out; + srv_conn_fd = accept(srv_fd, NULL, NULL); + if (!ASSERT_OK_FD(srv_conn_fd, "accept connection")) + goto out; + if (!ASSERT_OK(settimeo(srv_conn_fd, TIMEOUT_MS), "set server timeout")) + goto out; + + optlen = sizeof(mss_before); + if (!ASSERT_OK(getsockopt(cli_fd, IPPROTO_TCP, TCP_MAXSEG, &mss_before, + &optlen), "get mss before")) + goto out; + + bytes = send(cli_fd, buf, sizeof(buf), 0); + if (!ASSERT_EQ(bytes, (ssize_t)sizeof(buf), "send data")) + goto out; + + for (total = 0; total < sizeof(buf); total += bytes) { + bytes = recv(srv_conn_fd, buf, sizeof(buf), 0); + if (bytes <= 0) + break; + } + + ASSERT_EQ(total, sizeof(buf), "receive data"); + ASSERT_OK(skel->data->change_tail_ret, "change tail"); + ASSERT_OK(skel->bss->adjust_room_ret, "adjust room"); + ASSERT_TRUE(skel->bss->icmp_sent, "icmp sent"); + + optlen = sizeof(mss_after); + if (!ASSERT_OK(getsockopt(cli_fd, IPPROTO_TCP, TCP_MAXSEG, &mss_after, + &optlen), "get mss after")) + goto out; + + ASSERT_LT(mss_after, mss_before, "mss reduced"); + ASSERT_LE(mss_after, TEST_MSS_MAX, "mss below pmtu"); +out: + close(srv_conn_fd); + close(cli_fd); + close(srv_fd); + test_tc_change_tail_pmtu__destroy(skel); + close_netns(nstoken); + remove_netns(SERVER_NS); +out_client_ns: + remove_netns(CLIENT_NS); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tc_qevent.c b/tools/testing/selftests/bpf/prog_tests/tc_qevent.c new file mode 100644 index 000000000000..67e1d17567ab --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tc_qevent.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <network_helpers.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <unistd.h> +#include <string.h> + +#include "test_tc_qevent.skel.h" + +#define NS_TX "tc_qevent_tx" +#define NS_RX "tc_qevent_rx" +#define IP_TX "10.255.0.1" +#define IP_RX "10.255.0.2" +#define PIN_PATH "/sys/fs/bpf/tc_qevent_redirect" + +static void blast_udp(void) +{ + struct sockaddr_in dst = {}; + char buf[1400] = {}; + int fd, i; + + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!ASSERT_GE(fd, 0, "udp socket")) + return; + + dst.sin_family = AF_INET; + dst.sin_port = htons(12345); + inet_pton(AF_INET, IP_RX, &dst.sin_addr); + + /* + * Push far more than the RED queue can hold. Once qavg crosses qth_min + * every further packet hits the congestion_drop / early_drop qevent. + */ + for (i = 0; i < 50000; i++) + sendto(fd, buf, sizeof(buf), MSG_DONTWAIT, + (struct sockaddr *)&dst, sizeof(dst)); + + close(fd); +} + +static void run_qevent_redirect(struct bpf_program *prog, __u64 *counter) +{ + struct nstoken *tok = NULL; + int err; + + SYS_NOFAIL("ip netns del %s", NS_TX); + SYS_NOFAIL("ip netns del %s", NS_RX); + unlink(PIN_PATH); + + err = bpf_program__pin(prog, PIN_PATH); + if (!ASSERT_OK(err, "pin prog")) + return; + + SYS(unpin, "ip netns add %s", NS_TX); + SYS(del_tx, "ip netns add %s", NS_RX); + SYS(del_rx, "ip -n %s link add veth0 type veth peer name veth1 netns %s", NS_TX, NS_RX); + SYS(del_rx, "ip -n %s addr add %s/24 dev veth0", NS_TX, IP_TX); + SYS(del_rx, "ip -n %s link set veth0 up", NS_TX); + SYS(del_rx, "ip -n %s addr add %s/24 dev veth1", NS_RX, IP_RX); + SYS(del_rx, "ip -n %s link set veth1 up", NS_RX); + + tok = open_netns(NS_TX); + if (!ASSERT_OK_PTR(tok, "open_netns")) + goto del_rx; + + SYS(close_ns, "tc qdisc add dev veth0 root handle 1: htb default 1"); + SYS(close_ns, "tc class add dev veth0 parent 1: classid 1:1 htb rate 1mbit ceil 1mbit"); + + if (system("tc qdisc add dev veth0 parent 1:1 handle 11: red " + "limit 500000 avpkt 1000 probability 1 min 5000 max 6000 " + "burst 6 qevent early_drop block 10 2>/dev/null")) { + test__skip(); + goto close_ns; + } + + if (system("tc filter add block 10 bpf da object-pinned " + PIN_PATH " 2>/dev/null")) { + test__skip(); + goto close_ns; + } + + blast_udp(); + ASSERT_GT(*counter, 0, "qevent classifier ran"); +close_ns: + close_netns(tok); +del_rx: + SYS_NOFAIL("ip netns del %s", NS_RX); +del_tx: + SYS_NOFAIL("ip netns del %s", NS_TX); +unpin: + bpf_program__unpin(prog, PIN_PATH); +} + +void test_tc_qevent(void) +{ + struct test_tc_qevent *skel; + + skel = test_tc_qevent__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + if (test__start_subtest("redirect_verdict")) + run_qevent_redirect(skel->progs.qevent_redirect_verdict, + &skel->bss->verdict_calls); + if (test__start_subtest("redirect_helper")) + run_qevent_redirect(skel->progs.qevent_redirect_helper, + &skel->bss->helper_calls); + + test_tc_qevent__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c index 64fbda082309..af8968b89ad7 100644 --- a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c +++ b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c @@ -192,6 +192,8 @@ static int create_netkit(int mode, char *prim, char *peer) req.n.nlmsg_len += sizeof(struct ifinfomsg); addattr_l(&req.n, sizeof(req), IFLA_IFNAME, peer, strlen(peer)); addattr_nest_end(&req.n, peer_info); + addattr32(&req.n, sizeof(req), IFLA_NETKIT_SCRUB, + NETKIT_SCRUB_NONE); addattr_nest_end(&req.n, data); addattr_nest_end(&req.n, linkinfo); @@ -405,6 +407,24 @@ fail: return -1; } +static struct bpf_link *netns_attach_nk(const char *ns, int ifindex, + struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_netkit_opts, optl); + struct nstoken *nstoken = NULL; + struct bpf_link *link = NULL; + + nstoken = open_netns(ns); + if (!ASSERT_OK_PTR(nstoken, "setns")) + goto cleanup; + + link = bpf_program__attach_netkit(prog, ifindex, &optl); +cleanup: + if (nstoken) + close_netns(nstoken); + return link; +} + static void test_tcp(int family, const char *addr, __u16 port) { int listen_fd = -1, accept_fd = -1, client_fd = -1; @@ -1082,6 +1102,53 @@ done: close_netns(nstoken); } +static void test_tc_redirect_peer_ing(struct netns_setup_result *setup_result) +{ + struct test_tc_peer *skel; + struct nstoken *nstoken; + int err; + + nstoken = open_netns(NS_FWD); + if (!ASSERT_OK_PTR(nstoken, "setns fwd")) + return; + + skel = test_tc_peer__open(); + if (!ASSERT_OK_PTR(skel, "test_tc_peer__open")) + goto done; + + skel->rodata->IFINDEX_SRC = setup_result->ifindex_src_fwd; + skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd; + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_src_ing, + BPF_NETKIT_PRIMARY), 0, "src_prog_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_dst_ing, + BPF_NETKIT_PRIMARY), 0, "dst_prog_attach_type"); + + err = test_tc_peer__load(skel); + if (!ASSERT_OK(err, "test_tc_peer__load")) + goto done; + + skel->links.tc_src_ing = netns_attach_nk(NS_SRC, + setup_result->ifindex_src, + skel->progs.tc_src_ing); + if (!ASSERT_OK_PTR(skel->links.tc_src_ing, "attach_src")) + goto done; + skel->links.tc_dst_ing = netns_attach_nk(NS_DST, + setup_result->ifindex_dst, + skel->progs.tc_dst_ing); + if (!ASSERT_OK_PTR(skel->links.tc_dst_ing, "attach_dst")) + goto done; + + if (!ASSERT_OK(set_forwarding(false), "disable forwarding")) + goto done; + + test_connectivity(); + +done: + if (skel) + test_tc_peer__destroy(skel); + close_netns(nstoken); +} + static int tun_open(char *name) { struct ifreq ifr; @@ -1280,6 +1347,7 @@ static void *test_tc_redirect_run_tests(void *arg) RUN_TEST(tc_redirect_peer, MODE_VETH); RUN_TEST(tc_redirect_peer, MODE_NETKIT); + RUN_TEST(tc_redirect_peer_ing, MODE_NETKIT); RUN_TEST(tc_redirect_peer_l3, MODE_VETH); RUN_TEST(tc_redirect_peer_l3, MODE_NETKIT); RUN_TEST(tc_redirect_neigh, MODE_VETH); diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c index 56685fc03c7e..80e6315da2a5 100644 --- a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c +++ b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c @@ -507,6 +507,10 @@ static void misc(void) ASSERT_EQ(misc_skel->bss->nr_hwtstamp, 0, "nr_hwtstamp"); + ASSERT_TRUE(misc_skel->bss->nodelay_est_ok, "nodelay_est_ok"); + ASSERT_TRUE(misc_skel->bss->nodelay_hdr_len_reject, "nodelay_hdr_len_reject"); + ASSERT_TRUE(misc_skel->bss->nodelay_write_hdr_reject, "nodelay_write_hdr_reject"); + check_linum: ASSERT_FALSE(check_error_linum(&sk_fds), "check_error_linum"); sk_fds_close(&sk_fds); diff --git a/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c b/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c index de22734abc4d..40d38280c091 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c +++ b/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c @@ -131,8 +131,10 @@ static bool get_smc_nl_family_id(void) goto fail; ret = recv(fd, &msg, sizeof(msg), 0); - if (!ASSERT_FALSE(msg.n.nlmsg_type == NLMSG_ERROR || ret < 0 || - !NLMSG_OK(&msg.n, ret), "nl_family response")) + if (msg.n.nlmsg_type == NLMSG_ERROR) + goto fail; + if (!ASSERT_FALSE(ret < 0 || !NLMSG_OK(&msg.n, ret), + "nl_family response")) goto fail; nl = (struct nlattr *)GENLMSG_DATA(&msg); diff --git a/tools/testing/selftests/bpf/prog_tests/test_dst_clear.c b/tools/testing/selftests/bpf/prog_tests/test_dst_clear.c new file mode 100644 index 000000000000..7c35ca6f4539 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_dst_clear.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <sys/types.h> +#include <sys/socket.h> +#include <net/if.h> + +#include "test_progs.h" +#include "network_helpers.h" +#include "test_dst_clear.skel.h" + +#define IPV4_IFACE_ADDR "1.0.0.1" +#define UDP_TEST_PORT 7777 + +void test_ns_dst_clear(void) +{ + LIBBPF_OPTS(bpf_tcx_opts, tcx_opts); + struct test_dst_clear *skel; + struct sockaddr_in addr; + struct bpf_link *link; + socklen_t addrlen; + char buf[128] = {}; + int sockfd, err; + + skel = test_dst_clear__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + + SYS(fail, "ip addr add %s/8 dev lo", IPV4_IFACE_ADDR); + + link = bpf_program__attach_tcx(skel->progs.dst_clear, + if_nametoindex("lo"), &tcx_opts); + if (!ASSERT_OK_PTR(link, "attach_tcx")) + goto fail; + skel->links.dst_clear = link; + + addrlen = sizeof(addr); + err = make_sockaddr(AF_INET, IPV4_IFACE_ADDR, UDP_TEST_PORT, + (void *)&addr, &addrlen); + if (!ASSERT_OK(err, "make_sockaddr")) + goto fail; + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (!ASSERT_NEQ(sockfd, -1, "socket")) + goto fail; + err = sendto(sockfd, buf, sizeof(buf), 0, (void *)&addr, addrlen); + close(sockfd); + if (!ASSERT_EQ(err, sizeof(buf), "send")) + goto fail; + + ASSERT_TRUE(skel->bss->had_dst, "had_dst"); + ASSERT_TRUE(skel->bss->dst_cleared, "dst_cleared"); + +fail: + test_dst_clear__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c b/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c index e905cbaf6b3d..500446808908 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c +++ b/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c @@ -18,6 +18,7 @@ #include "test_global_func15.skel.h" #include "test_global_func16.skel.h" #include "test_global_func17.skel.h" +#include "test_global_func_deep_stack.skel.h" #include "test_global_func_ctx_args.skel.h" #include "bpf/libbpf_internal.h" @@ -155,6 +156,7 @@ void test_test_global_funcs(void) RUN_TESTS(test_global_func15); RUN_TESTS(test_global_func16); RUN_TESTS(test_global_func17); + RUN_TESTS(test_global_func_deep_stack); RUN_TESTS(test_global_func_ctx_args); if (test__start_subtest("ctx_arg_rewrite")) diff --git a/tools/testing/selftests/bpf/prog_tests/test_lsm.c b/tools/testing/selftests/bpf/prog_tests/test_lsm.c index bdc4fc06bc5a..d7495efd4a56 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_lsm.c +++ b/tools/testing/selftests/bpf/prog_tests/test_lsm.c @@ -5,36 +5,14 @@ */ #include <test_progs.h> -#include <sys/mman.h> #include <sys/wait.h> #include <unistd.h> -#include <malloc.h> -#include <stdlib.h> #include "lsm.skel.h" #include "lsm_tailcall.skel.h" char *CMD_ARGS[] = {"true", NULL}; -#define GET_PAGE_ADDR(ADDR, PAGE_SIZE) \ - (char *)(((unsigned long) (ADDR + PAGE_SIZE)) & ~(PAGE_SIZE-1)) - -int stack_mprotect(void) -{ - void *buf; - long sz; - int ret; - - sz = sysconf(_SC_PAGESIZE); - if (sz < 0) - return sz; - - buf = alloca(sz * 3); - ret = mprotect(GET_PAGE_ADDR(buf, sz), sz, - PROT_READ | PROT_WRITE | PROT_EXEC); - return ret; -} - int exec_cmd(int *monitored_pid) { int child_pid, child_status; diff --git a/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c new file mode 100644 index 000000000000..d0ba2ca587b0 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> + +#include "map_kptr.skel.h" + +void test_map_uninit_mem_exposure(void) +{ + size_t value_sz, slot_sz, lookup_sz, tail_sz; + int err, key, nr_cpus, cpu, map_fd; + __u8 *value = NULL, *zero = NULL; + struct bpf_program *prog; + struct map_kptr *skel; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + skel = map_kptr__open(); + if (!ASSERT_OK_PTR(skel, "map_kptr__open")) + return; + + bpf_object__for_each_program(prog, skel->obj) { + err = bpf_program__set_autoload(prog, false); + if (!ASSERT_OK(err, "bpf_program__set_autoload")) + goto out; + } + + err = map_kptr__load(skel); + if (!ASSERT_OK(err, "map_kptr__load")) + goto out; + + value_sz = bpf_map__value_size((skel)->maps.pcpu_array); + slot_sz = roundup(value_sz, 8); + tail_sz = slot_sz - value_sz; + if (!ASSERT_NEQ(tail_sz, 0, "tail_sz")) + goto out; + + lookup_sz = slot_sz * nr_cpus; + map_fd = bpf_map__fd(skel->maps.pcpu_array); + + value = malloc(lookup_sz); + zero = calloc(1, tail_sz); + if (!ASSERT_OK_PTR(value, "malloc value") || !ASSERT_OK_PTR(zero, "calloc zero")) + goto out; + + key = 0; + memset(value, 0x2B, lookup_sz); + err = bpf_map_update_elem(map_fd, &key, value, BPF_ANY); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out; + + memset(value, 0xFF, lookup_sz); + err = bpf_map_lookup_elem(map_fd, &key, value); + if (!ASSERT_OK(err, "bpf_map_lookup_elem")) + goto out; + + for (cpu = 0; cpu < nr_cpus; cpu++) { + __u8 *tail = value + cpu * slot_sz + value_sz; + + if (!ASSERT_MEMEQ(tail, zero, tail_sz, "zeroed tail bytes")) + goto out; + } + +out: + free(zero); + free(value); + map_kptr__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c new file mode 100644 index 000000000000..7f9f54ba3fbe --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> + +#include "struct_ops_arena.skel.h" +#include "struct_ops_arena_attach.skel.h" +#include "struct_ops_arena_fail.skel.h" + +#if defined(__x86_64__) || defined(__aarch64__) +/* + * Attach callbacks with __arena and __arena__nullable arguments and drive + * them through the bpf_testmod_ops3_call_test_arena*() kfuncs. + */ +static void arena_arg(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct struct_ops_arena *skel; + struct bpf_link *link = NULL; + int err; + + skel = struct_ops_arena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_arena); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.trigger), + &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + +out: + bpf_link__destroy(link); + struct_ops_arena__destroy(skel); +} + +/* + * A program with no arena cannot attach to a member with an __arena + * argument. + */ +static void arena_arg_fail(void) +{ + struct struct_ops_arena_fail *skel; + + skel = struct_ops_arena_fail__open_and_load(); + if (ASSERT_ERR_PTR(skel, "struct_ops_arena_fail__open_and_load")) + return; + + struct_ops_arena_fail__destroy(skel); +} + +static void arena_arg_attach_one(int target_fd, const char *prog_name) +{ + struct struct_ops_arena_attach *skel; + struct bpf_program *prog, *pos; + char log_buf[64 * 1024]; + int err; + + skel = struct_ops_arena_attach__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_attach__open")) + return; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, prog_name)) + goto out; + + bpf_object__for_each_program(pos, skel->obj) + bpf_program__set_autoload(pos, pos == prog); + + err = bpf_program__set_attach_target(prog, target_fd, "test_arena_cb"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + log_buf[0] = '\0'; + bpf_program__set_log_buf(prog, log_buf, sizeof(log_buf)); + err = struct_ops_arena_attach__load(skel); + + ASSERT_EQ(err, -EOPNOTSUPP, prog_name); + ASSERT_HAS_SUBSTR(log_buf, "Cannot attach to a target with arena context arguments", + "verifier_log"); + +out: + struct_ops_arena_attach__destroy(skel); +} + +static void arena_arg_attach(void) +{ + struct struct_ops_arena *skel; + int target_fd; + + skel = struct_ops_arena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena__open_and_load")) + return; + + target_fd = bpf_program__fd(skel->progs.test_arena_cb); + arena_arg_attach_one(target_fd, "fentry_test_arena"); + arena_arg_attach_one(target_fd, "fexit_test_arena"); + arena_arg_attach_one(target_fd, "freplace_test_arena"); + + struct_ops_arena__destroy(skel); +} +#endif + +/* + * Serialized because it attaches the singleton bpf_testmod_ops3, which + * test_struct_ops_private_stack also attaches; registering it twice fails + * with -EEXIST. + */ +void serial_test_struct_ops_arena(void) +{ + /* + * Arena struct_ops arguments need JIT support, currently x86-64 and + * arm64 only. Elsewhere verification fails with "JIT does not support + * arena arguments", so the programs cannot even load. + */ +#if defined(__x86_64__) || defined(__aarch64__) + if (test__start_subtest("arena_arg")) + arena_arg(); + if (test__start_subtest("arena_arg_fail")) + arena_arg_fail(); + if (test__start_subtest("arena_arg_attach")) + arena_arg_attach(); +#else + test__skip(); +#endif +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c new file mode 100644 index 000000000000..0f321e889862 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_multi_args.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "struct_ops_multi_args.skel.h" + +void test_struct_ops_multi_args(void) +{ + RUN_TESTS(struct_ops_multi_args); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c b/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c index 9556ad3d986f..6a5806b36113 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c +++ b/tools/testing/selftests/bpf/prog_tests/test_task_local_data.c @@ -3,8 +3,14 @@ #include <bpf/btf.h> #include <test_progs.h> +/* + * Only a page is pinned to kernel, so the maximum amount of dynamic data + * allowed is page_size - sizeof(struct tld_data_u) - static TLD fields. + */ +#define TLD_DYN_DATA_SIZE_MAX (getpagesize() - sizeof(struct tld_data_u) - 8) + #define TLD_FREE_DATA_ON_THREAD_EXIT -#define TLD_DYN_DATA_SIZE (getpagesize() - 8) +#define TLD_DYN_DATA_SIZE TLD_DYN_DATA_SIZE_MAX #include "task_local_data.h" struct test_tld_struct { @@ -24,12 +30,12 @@ TLD_DEFINE_KEY(value0_key, "value0", sizeof(int)); * sequentially. Users of task local data library should not touch * library internal. */ -static void reset_tld(void) +static void reset_tld(__u16 dyn_data_size) { - if (TLD_READ_ONCE(tld_meta_p)) { + if (tld_meta_p) { /* Remove TLDs created by tld_create_key() */ tld_meta_p->cnt = 1; - tld_meta_p->size = TLD_DYN_DATA_SIZE; + tld_meta_p->size = dyn_data_size + 8; memset(&tld_meta_p->metadata[1], 0, (TLD_MAX_DATA_CNT - 1) * sizeof(struct tld_metadata)); } @@ -127,7 +133,7 @@ static void test_task_local_data_basic(void) tld_key_t key; int i, err; - reset_tld(); + reset_tld(TLD_DYN_DATA_SIZE_MAX); ASSERT_OK(pthread_mutex_init(&global_mutex, NULL), "pthread_mutex_init"); @@ -147,11 +153,13 @@ static void test_task_local_data_basic(void) /* * Shouldn't be able to store data exceed a page. Create a TLD just big - * enough to exceed a page. TLDs already created are int value0, int - * value1, and struct test_tld_struct value2. + * enough to exceed a page. Data already contains struct tld_data_u, + * value0 and value1 of int type, and value 2 of struct test_tld_struct. */ - key = tld_create_key("value_not_exist", - TLD_PAGE_SIZE - 2 * sizeof(int) - sizeof(struct test_tld_struct) + 1); + key = tld_create_key("value_not_exist", TLD_PAGE_SIZE + 1 - + sizeof(struct tld_data_u) - + TLD_ROUND_UP(sizeof(int), 8) * 2 - + TLD_ROUND_UP(sizeof(struct test_tld_struct), 8)); ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key"); key = tld_create_key("value2", sizeof(struct test_tld_struct)); @@ -239,7 +247,7 @@ static void test_task_local_data_race(void) tld_keys[0] = value0_key; for (j = 0; j < 100; j++) { - reset_tld(); + reset_tld(TLD_DYN_DATA_SIZE_MAX); for (i = 0; i < TEST_RACE_THREAD_NUM; i++) { /* @@ -288,10 +296,80 @@ out: test_task_local_data__destroy(skel); } +static void test_task_local_data_dyn_size(__u16 dyn_data_size) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct test_task_local_data *skel; + int max_keys, i, err, fd, *data; + char name[TLD_NAME_LEN]; + tld_key_t key; + + reset_tld(dyn_data_size); + + skel = test_task_local_data__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + tld_keys = calloc(TLD_MAX_DATA_CNT, sizeof(tld_key_t)); + if (!ASSERT_OK_PTR(tld_keys, "calloc tld_keys")) + goto out; + + fd = bpf_map__fd(skel->maps.tld_data_map); + + /* Create as many int-sized TLDs as the dynamic data size allows */ + max_keys = dyn_data_size / TLD_ROUND_UP(sizeof(int), 8); + for (i = 0; i < max_keys; i++) { + snprintf(name, TLD_NAME_LEN, "value_%d", i); + tld_keys[i] = tld_create_key(name, sizeof(int)); + if (!ASSERT_FALSE(tld_key_is_err(tld_keys[i]), "tld_create_key")) + goto out; + + data = tld_get_data(fd, tld_keys[i]); + if (!ASSERT_OK_PTR(data, "tld_get_data")) + goto out; + *data = i; + } + + /* The next key should fail with E2BIG */ + key = tld_create_key("overflow", sizeof(int)); + ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key overflow"); + + /* Verify data for value_i do not overlap */ + for (i = 0; i < max_keys; i++) { + data = tld_get_data(fd, tld_keys[i]); + if (!ASSERT_OK_PTR(data, "tld_get_data")) + goto out; + + ASSERT_EQ(*data, i, "tld_get_data value_i"); + } + + /* Verify BPF side can still read the static key */ + data = tld_get_data(fd, value0_key); + if (!ASSERT_OK_PTR(data, "tld_get_data value0")) + goto out; + *data = 0xdeadbeef; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.task_main), &opts); + ASSERT_OK(err, "run task_main"); + ASSERT_EQ(skel->bss->test_value0, 0xdeadbeef, "tld_get_data value0"); + +out: + if (tld_keys) { + free(tld_keys); + tld_keys = NULL; + } + tld_free(); + test_task_local_data__destroy(skel); +} + void test_task_local_data(void) { if (test__start_subtest("task_local_data_basic")) test_task_local_data_basic(); if (test__start_subtest("task_local_data_race")) test_task_local_data_race(); + if (test__start_subtest("task_local_data_dyn_size_small")) + test_task_local_data_dyn_size(64); + if (test__start_subtest("task_local_data_dyn_size_zero")) + test_task_local_data_dyn_size(0); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c b/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c index 7fc4d7dd70ef..67ba27d69347 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c +++ b/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c @@ -168,7 +168,7 @@ static int check_server_rx_data(struct subtest_cfg *cfg, static struct connection *connect_client_to_server(struct subtest_cfg *cfg) { - struct network_helper_opts opts = {.timeout_ms = 500}; + struct network_helper_opts opts = {.timeout_ms = 1000}; int family = cfg->ipproto == 6 ? AF_INET6 : AF_INET; struct connection *conn = NULL; int client_fd, server_fd; @@ -206,18 +206,13 @@ static void disconnect_client_from_server(struct subtest_cfg *cfg, free(conn); } -static int send_and_test_data(struct subtest_cfg *cfg, bool must_succeed) +static int send_and_test_data(struct subtest_cfg *cfg) { struct connection *conn; int err, res = -1; conn = connect_client_to_server(cfg); - if (!must_succeed && !ASSERT_ERR_PTR(conn, "connection that must fail")) - goto end; - else if (!must_succeed) - return 0; - - if (!ASSERT_OK_PTR(conn, "connection that must succeed")) + if (!ASSERT_OK_PTR(conn, "connect to server")) return -1; err = send(conn->client_fd, tx_buffer, DEFAULT_TEST_DATA_SIZE, 0); @@ -391,7 +386,7 @@ static void run_test(struct subtest_cfg *cfg) goto fail; /* Basic communication must work */ - if (!ASSERT_OK(send_and_test_data(cfg, true), "connect without any encap")) + if (!ASSERT_OK(send_and_test_data(cfg), "connect without any encap")) goto fail; /* Attach encapsulation program to client */ @@ -403,7 +398,7 @@ static void run_test(struct subtest_cfg *cfg) if (!ASSERT_OK(configure_kernel_decapsulation(cfg), "configure kernel decapsulation")) goto fail; - if (!ASSERT_OK(send_and_test_data(cfg, true), + if (!ASSERT_OK(send_and_test_data(cfg), "connect with encap prog and kern decap")) goto fail; } @@ -411,7 +406,7 @@ static void run_test(struct subtest_cfg *cfg) /* Replace kernel decapsulation with BPF decapsulation, test must pass */ if (!ASSERT_OK(configure_ebpf_decapsulation(cfg), "configure ebpf decapsulation")) goto fail; - ASSERT_OK(send_and_test_data(cfg, true), "connect with encap and decap progs"); + ASSERT_OK(send_and_test_data(cfg), "connect with encap and decap progs"); fail: close_netns(nstoken); @@ -443,7 +438,6 @@ static int setup(void) SYS(fail_close_ns_client, "ip link add %s type veth peer name %s", "veth1 mtu 1500 netns " CLIENT_NS " address " MAC_ADDR_VETH1, "veth2 mtu 1500 netns " SERVER_NS " address " MAC_ADDR_VETH2); - SYS(fail_close_ns_client, "ethtool -K veth1 tso off"); SYS(fail_close_ns_client, "ip link set veth1 up"); nstoken_server = open_netns(SERVER_NS); if (!ASSERT_OK_PTR(nstoken_server, "open server ns")) diff --git a/tools/testing/selftests/bpf/prog_tests/test_veristat.c b/tools/testing/selftests/bpf/prog_tests/test_veristat.c index 9aff08ac55c0..11f3de2b66ad 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_veristat.c +++ b/tools/testing/selftests/bpf/prog_tests/test_veristat.c @@ -37,6 +37,14 @@ static struct fixture *init_fixture(void) return fix; } +static void read_output(struct fixture *fix) +{ + ssize_t len = pread(fix->fd, fix->output, fix->sz - 1, 0); + + fix->output[len < 0 ? 0 : len] = 0; + ASSERT_GE(len, 0, "pread"); +} + static void teardown_fixture(struct fixture *fix) { free(fix->output); @@ -74,7 +82,7 @@ static void test_set_global_vars_succeeds(void) " -G \"struct11 [ 7 ] [ 5 ] .struct2[0][1].u.mat[3][0] = 175\" " \ " -vl2 > %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("=0xf000000000000001 ", "var_s64 = 0xf000000000000001"); __CHECK_STR("=0xfedcba9876543210 ", "var_u64 = 0xfedcba9876543210"); __CHECK_STR("=0x80000000 ", "var_s32 = -0x80000000"); @@ -116,7 +124,7 @@ static void test_set_global_vars_from_file_succeeds(void) syncfs(fd); SYS(out, "%s set_global_vars.bpf.o -G \"@%s\" -vl2 > %s", fix->veristat, input_file, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("=0x8000 ", "var_s16 = -32768"); __CHECK_STR("=0xecec ", "var_u16 = 60652"); @@ -134,7 +142,7 @@ static void test_set_global_vars_out_of_range(void) "%s set_global_vars.bpf.o -G \"var_s32 = 2147483648\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("is out of range [-2147483648; 2147483647]", "out of range"); out: @@ -149,7 +157,7 @@ static void test_unsupported_ptr_array_type(void) "%s set_global_vars.bpf.o -G \"ptr_arr[0] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Can't set ptr_arr[0]. Only ints and enums are supported", "ptr_arr"); out: @@ -164,7 +172,7 @@ static void test_array_out_of_bounds(void) "%s set_global_vars.bpf.o -G \"arr[99] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Array index 99 is out of bounds", "arr[99]"); out: @@ -179,7 +187,7 @@ static void test_array_index_not_found(void) "%s set_global_vars.bpf.o -G \"arr[EG2] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Can't resolve enum value EG2", "arr[EG2]"); out: @@ -230,6 +238,97 @@ out: teardown_fixture(fix); } +/* + * Name filter tests below run veristat on veristat_foo.bpf.o and + * veristat_bar.bpf.o, both defining programs 'foo', 'bar' and 'buz'. + * Every entry describes a single (filters, file, prog) combination and + * tells whether that program is expected in the veristat output: + * 'true' if it is, 'false' if it is not and -1 if veristat is expected + * to reject the filter. + */ +#define FILTER_OBJS "veristat_foo.bpf.o veristat_bar.bpf.o" + +static const struct name_filter_case { + const char *filters; + const char *file; + const char *prog; + int included; +} name_filter_cases[] = { + /* no filters, every program is processed */ + { "", "foo", "foo", true }, + { "", "foo", "bar", true }, + { "", "foo", "buz", true }, + { "", "bar", "foo", true }, + { "", "bar", "bar", true }, + { "", "bar", "buz", true }, + /* deny filters */ + { "-f '!*foo*'", "foo", "bar", false }, + { "-f '!*foo*'", "bar", "foo", false }, + { "-f '!*foo*'", "bar", "bar", true }, + { "-f '!*foo*/bar'", "foo", "bar", false }, + { "-f '!*foo*/bar'", "foo", "buz", true }, + { "-f '!*foo*/bar'", "bar", "bar", true }, + { "-f '!*foo*/'", "foo", "bar", false }, + { "-f '!*foo*/'", "bar", "bar", true }, + { "-f '!/bar'", "foo", "bar", false }, + { "-f '!/bar'", "foo", "foo", true }, + { "-f '!/'", "foo", "bar", -1 }, + { "-f '!'", "foo", "bar", -1 }, + /* allow filters */ + { "-f '*foo*'", "foo", "bar", true }, + { "-f '*foo*'", "bar", "foo", true }, + { "-f '*foo*'", "bar", "bar", false }, + { "-f '*foo*/bar'", "foo", "bar", true }, + { "-f '*foo*/bar'", "foo", "buz", false }, + { "-f '*foo*/bar'", "bar", "bar", false }, + { "-f '*foo*/'", "foo", "bar", true }, + { "-f '*foo*/'", "bar", "bar", false }, + { "-f '/bar'", "foo", "bar", true }, + { "-f '/bar'", "foo", "foo", false }, + { "-f '/'", "foo", "bar", -1 }, + { "-f ''", "foo", "bar", -1 }, + /* allow and deny filters combined */ + { "-f '*foo*/' -f '!/bar'", "foo", "foo", true }, + { "-f '*foo*/' -f '!/bar'", "foo", "bar", false }, + { "-f '*foo*/' -f '!/bar'", "bar", "foo", false }, +}; + +static void test_name_filters(void) +{ + struct fixture *fix = init_fixture(); + const struct name_filter_case *t; + char cmd[512], row[64], name[128]; + int i, err; + + for (i = 0; i < ARRAY_SIZE(name_filter_cases); i++) { + t = &name_filter_cases[i]; + /* stderr is merged with stdout in order to catch error messages */ + snprintf(cmd, sizeof(cmd), "%s " FILTER_OBJS " -q -o csv -e file,prog %s > %s 2>&1", + fix->veristat, t->filters, fix->tmpfile); + err = system(cmd); + read_output(fix); + + snprintf(row, sizeof(row), "veristat_%s.bpf.o,%s", t->file, t->prog); + snprintf(name, sizeof(name), "veristat %s: %s", t->filters, row); + switch (t->included) { + case true: + ASSERT_OK(err, name); + ASSERT_HAS_SUBSTR(fix->output, row, name); + break; + case false: + ASSERT_OK(err, name); + ASSERT_FALSE(!!strstr(fix->output, row), name); + break; + case -1: + ASSERT_NEQ(err, 0, name); + ASSERT_HAS_SUBSTR(fix->output, "Invalid filter", name); + break; + } + } + + teardown_fixture(fix); +} + void test_veristat(void) { if (test__start_subtest("set_global_vars_succeeds")) @@ -256,6 +355,8 @@ void test_veristat(void) if (test__start_subtest("test_no_array_index_for_array")) test_no_array_index_for_array(); + if (test__start_subtest("name_filters")) + test_name_filters(); } #undef __CHECK_STR diff --git a/tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c b/tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c index 3e98a1665936..1675b32753a8 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c +++ b/tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c @@ -456,7 +456,11 @@ static void xdp_veth_egress(u32 flags) .remote_flags = flags, } }; - const char magic_mac[6] = { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF}; + const unsigned char egress_macs[VETH_PAIRS_COUNT][ETH_ALEN] = { + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x01 }, + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x02 }, + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x03 }, + }; struct xdp_redirect_multi_kern *xdp_redirect_multi_kern; struct bpf_object *bpf_objs[VETH_EGRESS_SKEL_NB]; struct xdp_redirect_map *xdp_redirect_map; @@ -512,7 +516,13 @@ static void xdp_veth_egress(u32 flags) &net_config, prog_cfg, i)) goto destroy_xdp_redirect_map; - err = bpf_map_update_elem(mac_map, &ifindex, magic_mac, 0); + { + __be64 mac = 0; + + memcpy(&mac, egress_macs[i], ETH_ALEN); + err = bpf_map_update_elem(mac_map, &ifindex, &mac, 0); + } + if (!ASSERT_OK(err, "bpf_map_update_elem")) goto destroy_xdp_redirect_map; @@ -531,15 +541,162 @@ static void xdp_veth_egress(u32 flags) for (i = 0; i < 2; i++) { u32 key = i; + __be64 expected = 0; u64 res; err = bpf_map_lookup_elem(res_map, &key, &res); if (!ASSERT_OK(err, "get MAC res")) goto destroy_xdp_redirect_map; - ASSERT_STRNEQ((const char *)&res, magic_mac, ETH_ALEN, "compare mac"); + /* store_mac_1/2 run on the second/third remote veths. */ + memcpy(&expected, egress_macs[i + 1], ETH_ALEN); + ASSERT_EQ(res, expected, "compare mac"); + } + +destroy_xdp_redirect_map: + close_netns(nstoken); + xdp_redirect_map__destroy(xdp_redirect_map); +destroy_xdp_redirect_multi_kern: + xdp_redirect_multi_kern__destroy(xdp_redirect_multi_kern); +destroy_xdp_dummy: + xdp_dummy__destroy(xdp_dummy); + + cleanup_network(&net_config); +} + +static void xdp_veth_egress_last_dst(u32 flags) +{ + struct prog_configuration prog_cfg[VETH_PAIRS_COUNT] = { + { + .local_name = "xdp_redirect_map_all_prog", + .remote_name = "xdp_dummy_prog", + .local_flags = flags, + .remote_flags = flags, + }, + { + .local_name = "xdp_redirect_map_all_prog", + .remote_name = "store_mac_1", + .local_flags = flags, + .remote_flags = flags, + }, + { + .local_name = "xdp_redirect_map_all_prog", + .remote_name = "xdp_dummy_prog", + .local_flags = flags, + .remote_flags = flags, + } + }; + const unsigned char egress_macs[VETH_PAIRS_COUNT][ETH_ALEN] = { + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x01 }, + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x02 }, + { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0x03 }, + }; + struct xdp_redirect_multi_kern *xdp_redirect_multi_kern; + struct bpf_object *bpf_objs[VETH_EGRESS_SKEL_NB]; + struct xdp_redirect_map *xdp_redirect_map; + struct net_configuration net_config = {}; + int mac_map, egress_map, res_map; + struct nstoken *nstoken = NULL; + struct xdp_dummy *xdp_dummy; + __be64 sentinel_mac = 0; + __be64 last_mac = 0; + __be64 res; + u32 key; + int err; + int i; + + xdp_dummy = xdp_dummy__open_and_load(); + if (!ASSERT_OK_PTR(xdp_dummy, "xdp_dummy__open_and_load")) + return; + + xdp_redirect_multi_kern = xdp_redirect_multi_kern__open_and_load(); + if (!ASSERT_OK_PTR(xdp_redirect_multi_kern, "xdp_redirect_multi_kern__open_and_load")) + goto destroy_xdp_dummy; + + xdp_redirect_map = xdp_redirect_map__open_and_load(); + if (!ASSERT_OK_PTR(xdp_redirect_map, "xdp_redirect_map__open_and_load")) + goto destroy_xdp_redirect_multi_kern; + + if (!ASSERT_OK(create_network(&net_config), "create network")) + goto destroy_xdp_redirect_map; + + mac_map = bpf_map__fd(xdp_redirect_multi_kern->maps.mac_map); + if (!ASSERT_OK_FD(mac_map, "open mac_map")) + goto destroy_xdp_redirect_map; + + egress_map = bpf_map__fd(xdp_redirect_multi_kern->maps.map_egress); + if (!ASSERT_OK_FD(egress_map, "open map_egress")) + goto destroy_xdp_redirect_map; + + bpf_objs[0] = xdp_dummy->obj; + bpf_objs[1] = xdp_redirect_multi_kern->obj; + bpf_objs[2] = xdp_redirect_map->obj; + + nstoken = open_netns(net_config.ns0_name); + if (!ASSERT_OK_PTR(nstoken, "open NS0")) + goto destroy_xdp_redirect_map; + + for (i = 0; i < VETH_PAIRS_COUNT; i++) { + struct bpf_devmap_val devmap_val = {}; + int ifindex = if_nametoindex(net_config.veth_cfg[i].local_veth); + u32 key = i; + + SYS(destroy_xdp_redirect_map, + "ip -n %s neigh add %s lladdr 00:00:00:00:00:01 dev %s", + net_config.veth_cfg[i].namespace, IP_NEIGH, + net_config.veth_cfg[i].remote_veth); + + if (attach_programs_to_veth_pair(bpf_objs, VETH_EGRESS_SKEL_NB, + &net_config, prog_cfg, i)) + goto destroy_xdp_redirect_map; + + { + __be64 mac = 0; + + memcpy(&mac, egress_macs[i], ETH_ALEN); + err = bpf_map_update_elem(mac_map, &ifindex, &mac, 0); + } + + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto destroy_xdp_redirect_map; + + devmap_val.ifindex = ifindex; + devmap_val.bpf_prog.fd = -1; + + if (i == VETH_PAIRS_COUNT - 1) + devmap_val.bpf_prog.fd = + bpf_program__fd(xdp_redirect_multi_kern->progs.xdp_devmap_prog); + + err = bpf_map_update_elem(egress_map, &key, &devmap_val, 0); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto destroy_xdp_redirect_map; } + res_map = bpf_map__fd(xdp_redirect_map->maps.rx_mac); + if (!ASSERT_OK_FD(res_map, "open rx_map")) + goto destroy_xdp_redirect_map; + + memcpy(&sentinel_mac, egress_macs[VETH_PAIRS_COUNT - 1], ETH_ALEN); + memcpy(&last_mac, egress_macs[VETH_PAIRS_COUNT - 1], ETH_ALEN); + + key = 0; + err = bpf_map_update_elem(res_map, &key, &sentinel_mac, 0); + if (!ASSERT_OK(err, "init rx mac")) + goto destroy_xdp_redirect_map; + + SYS_NOFAIL("ip netns exec %s ping %s -i 0.1 -c 4 -W1 > /dev/null ", + net_config.veth_cfg[0].namespace, IP_NEIGH); + + err = bpf_map_lookup_elem(res_map, &key, &res); + if (!ASSERT_OK(err, "get MAC res")) + goto destroy_xdp_redirect_map; + + if (!ASSERT_NEQ(res, sentinel_mac, "rx_mac overwritten by store_mac_1")) + goto destroy_xdp_redirect_map; + + if (!ASSERT_NEQ(res, last_mac, "earlier dst not rewritten by last dst")) + goto destroy_xdp_redirect_map; + destroy_xdp_redirect_map: close_netns(nstoken); xdp_redirect_map__destroy(xdp_redirect_map); @@ -596,4 +753,7 @@ void test_xdp_veth_egress(void) if (test__start_subtest("SKB_MODE/egress")) xdp_veth_egress(XDP_FLAGS_SKB_MODE); + + if (test__start_subtest("SKB_MODE/egress_last_dst")) + xdp_veth_egress_last_dst(XDP_FLAGS_SKB_MODE); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_xsk.c b/tools/testing/selftests/bpf/prog_tests/test_xsk.c index 7950c504ed28..4549358cc8c2 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_xsk.c +++ b/tools/testing/selftests/bpf/prog_tests/test_xsk.c @@ -7,7 +7,6 @@ #include <linux/netdev.h> #include <poll.h> #include <pthread.h> -#include <signal.h> #include <string.h> #include <sys/mman.h> #include <sys/socket.h> @@ -65,11 +64,6 @@ static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) eth_hdr->h_proto = htons(ETH_P_LOOPBACK); } -static bool is_umem_valid(struct ifobject *ifobj) -{ - return !!ifobj->umem->umem; -} - static u32 mode_to_xdp_flags(enum test_mode mode) { return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; @@ -213,6 +207,7 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, for (i = 0; i < MAX_INTERFACES; i++) { struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; + struct xsk_umem_info *umem_real; ifobj->xsk = &ifobj->xsk_arr[0]; ifobj->use_poll = false; @@ -229,24 +224,30 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, ifobj->tx_on = false; } - memset(ifobj->umem, 0, sizeof(*ifobj->umem)); - ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; - ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; - + umem_real = ifobj->xsk_arr[0].umem_real; + memset(umem_real, 0, sizeof(*umem_real)); for (j = 0; j < MAX_SOCKETS; j++) { - memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); - ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; - ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE; + struct xsk_socket_info *xsk = &ifobj->xsk_arr[j]; + + memset(xsk, 0, sizeof(*xsk)); + xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; + if (j == 0) + xsk->umem_real = umem_real; + xsk->umem = umem_real; + xsk->batch_size = DEFAULT_BATCH_SIZE; if (i == 0) - ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default; + xsk->pkt_stream = test->tx_pkt_stream_default; else - ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default; + xsk->pkt_stream = test->rx_pkt_stream_default; - memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN); - memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN); - ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0); - ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1); + memcpy(xsk->src_mac, g_mac, ETH_ALEN); + memcpy(xsk->dst_mac, g_mac, ETH_ALEN); + xsk->src_mac[5] += ((j * 2) + 0); + xsk->dst_mac[5] += ((j * 2) + 1); } + + ifobj->xsk->umem->num_frames = DEFAULT_UMEM_BUFFERS; + ifobj->xsk->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; } if (ifobj_tx->hw_ring_size_supp) @@ -303,6 +304,18 @@ static void test_spec_reset(struct test_spec *test) __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); } +static void test_spec_set_unaligned(struct test_spec *test) +{ + test->ifobj_tx->xsk->umem->unaligned_mode = true; + test->ifobj_rx->xsk->umem->unaligned_mode = true; +} + +static void test_spec_set_frame_size(struct test_spec *test, u32 size) +{ + test->ifobj_tx->xsk->umem->frame_size = size; + test->ifobj_rx->xsk->umem->frame_size = size; +} + static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, struct bpf_map *xskmap_tx) @@ -414,14 +427,14 @@ static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pk } /* Search for the end of the packet in verbatim mode */ - if (!pkt_continues(pkt->options) || !pkt->valid) + if (!pkt_continues(pkt->options)) return nb_frags; next_frag = pkt_stream->current_pkt_nb; pkt++; while (next_frag++ < pkt_stream->nb_pkts) { nb_frags++; - if (!pkt_continues(pkt->options) || !pkt->valid) + if (!pkt_continues(pkt->options)) break; pkt++; } @@ -652,11 +665,11 @@ static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, s if (!frame->valid || !pkt_continues(frame->options)) payload++; } else { - if (frame->valid) + if (frame->valid) { len += frame->len; - if (frame->valid && pkt_continues(frame->options)) - continue; - + if (pkt_continues(frame->options)) + continue; + } pkt->pkt_nb = pkt_nb; pkt->len = len; pkt->valid = frame->valid; @@ -810,11 +823,11 @@ static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 exp { u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; void *data = xsk_umem__get_data(umem->buffer, addr); + u64 umem_sz = umem_size(umem); addr -= umem->base_addr; - if (addr >= umem->num_frames * umem->frame_size || - addr + len > umem->num_frames * umem->frame_size) { + if (addr >= umem_sz || addr + len > umem_sz) { ksft_print_msg("Frag invalid addr: %llx len: %u\n", (unsigned long long)addr, len); return false; @@ -991,7 +1004,7 @@ static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) return TEST_FAILURE; if (!ret) { - if (!is_umem_valid(test->ifobj_tx)) + if (test->poll_tmout) return TEST_PASS; ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); @@ -1130,7 +1143,7 @@ static int receive_pkts(struct test_spec *test) break; res = __receive_pkts(test, xsk); - if (!(res == TEST_PASS || res == TEST_CONTINUE)) + if (res != TEST_CONTINUE) return res; ret = gettimeofday(&tv_now, NULL); @@ -1147,11 +1160,12 @@ static int receive_pkts(struct test_spec *test) return TEST_PASS; } -static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) +static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, + bool test_timeout) { u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; + struct xsk_umem_info *umem = ifobject->xsk_arr[0].umem_real; struct pkt_stream *pkt_stream = xsk->pkt_stream; - struct xsk_umem_info *umem = ifobject->umem; bool use_poll = ifobject->use_poll; struct pollfd fds = { }; int ret; @@ -1159,7 +1173,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); /* pkts_in_flight might be negative if many invalid packets are sent */ if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) / - buffer_len)) { + buffer_len) && !test_timeout) { ret = kick_tx(xsk); if (ret) return TEST_FAILURE; @@ -1172,7 +1186,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) { if (use_poll) { ret = poll(&fds, 1, POLL_TMOUT); - if (timeout) { + if (test_timeout) { if (ret < 0) { ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, errno); @@ -1210,7 +1224,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b while (nb_frags_left--) { struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); - tx_desc->addr = pkt_get_addr(pkt, ifobject->umem); + tx_desc->addr = pkt_get_addr(pkt, umem); if (pkt_stream->verbatim) { tx_desc->len = pkt->len; tx_desc->options = pkt->options; @@ -1236,10 +1250,9 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b } } - if (pkt && pkt->valid) { + if (pkt && pkt->valid) valid_pkts++; - valid_frags += nb_frags; - } + valid_frags += nb_frags; } pthread_mutex_lock(&pacing_mutex); @@ -1252,7 +1265,7 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b if (use_poll) { ret = poll(&fds, 1, POLL_TMOUT); if (ret <= 0) { - if (ret == 0 && timeout) + if (ret == 0 && test_timeout) return TEST_PASS; ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); @@ -1260,14 +1273,14 @@ static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, b } } - if (!timeout) { + if (!test_timeout) { if (complete_pkts(xsk, i)) return TEST_FAILURE; usleep(10); - return TEST_PASS; } + /* Loop completion is driven by send_pkts() stream progress checks. */ return TEST_CONTINUE; } @@ -1303,7 +1316,6 @@ bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) static int send_pkts(struct test_spec *test, struct ifobject *ifobject) { - bool timeout = !is_umem_valid(test->ifobj_rx); DECLARE_BITMAP(bitmap, test->nb_sockets); u32 i, ret; @@ -1318,19 +1330,18 @@ static int send_pkts(struct test_spec *test, struct ifobject *ifobject) __set_bit(i, bitmap); continue; } - ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout); - if (ret == TEST_CONTINUE && !test->fail) - continue; - - if ((ret || test->fail) && !timeout) - return TEST_FAILURE; - - if (ret == TEST_PASS && timeout) + ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], test->poll_tmout); + if (ret != TEST_CONTINUE) return ret; - ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); - if (ret) + if (test->fail) return TEST_FAILURE; + + if (!test->poll_tmout) { + ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); + if (ret) + return TEST_FAILURE; + } } } @@ -1488,14 +1499,25 @@ static int xsk_configure(struct test_spec *test, struct ifobject *ifobject, static int thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) { - int ret = xsk_configure(test, ifobject, test->ifobj_rx->umem, true); + struct xsk_umem_info *umem_rx, *umem_tx; + int ret; + + if (!test->ifobj_rx || !test->ifobj_rx->xsk_arr[0].umem->umem) { + ksft_print_msg("Error: RX UMEM is not initialized before shared-UMEM TX setup\n"); + return -EINVAL; + } + umem_rx = test->ifobj_rx->xsk_arr[0].umem; + umem_tx = ifobject->xsk_arr[0].umem_real; + memcpy(umem_tx, umem_rx, sizeof(*umem_tx)); + umem_tx->base_addr = 0; + umem_tx->next_buffer = 0; + + ret = xsk_configure(test, ifobject, umem_rx, true); if (ret) return ret; ifobject->xsk = &ifobject->xsk_arr[0]; ifobject->xskmap = test->ifobj_rx->xskmap; - memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); - ifobject->umem->base_addr = 0; return 0; } @@ -1548,31 +1570,37 @@ static int xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream static int thread_common_ops(struct test_spec *test, struct ifobject *ifobject) { + struct xsk_umem_info *umem = ifobject->xsk->umem; LIBBPF_OPTS(bpf_xdp_query_opts, opts); int mmap_flags; - u64 umem_sz; + u64 umem_sz, mmap_sz; void *bufs; int ret; u32 i; - umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; + umem_sz = umem_size(umem); mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; - if (ifobject->umem->unaligned_mode) + if (umem->unaligned_mode) mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; if (ifobject->shared_umem) umem_sz *= 2; - bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); + mmap_sz = umem->unaligned_mode ? + ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE : umem_sz; + + bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); if (bufs == MAP_FAILED) return -errno; - ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz); + umem->mmap_size = mmap_sz; + + ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz); if (ret) return ret; - ret = xsk_configure(test, ifobject, ifobject->umem, false); + ret = xsk_configure(test, ifobject, umem, false); if (ret) return ret; @@ -1581,14 +1609,13 @@ static int thread_common_ops(struct test_spec *test, struct ifobject *ifobject) if (!ifobject->rx_on) return 0; - ret = xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, + ret = xsk_populate_fill_ring(umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring); if (ret) return ret; for (i = 0; i < test->nb_sockets; i++) { - ifobject->xsk = &ifobject->xsk_arr[i]; - ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i); + ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk_arr[i].xsk, i); if (ret) return ret; } @@ -1642,7 +1669,8 @@ void *worker_testapp_validate_rx(void *arg) strerror(-err)); } - pthread_barrier_wait(&barr); + if (test->use_barrier) + pthread_barrier_wait(&barr); /* We leave only now in case of error to avoid getting stuck in the barrier */ if (err) { @@ -1675,19 +1703,10 @@ void *worker_testapp_validate_rx(void *arg) static void testapp_clean_xsk_umem(struct ifobject *ifobj) { - u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; - - if (ifobj->shared_umem) - umem_sz *= 2; - - umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; - xsk_umem__delete(ifobj->umem->umem); - munmap(ifobj->umem->buffer, umem_sz); -} + struct xsk_umem_info *umem = ifobj->xsk->umem; -static void handler(int signum) -{ - pthread_exit(NULL); + xsk_umem__delete(umem->umem); + munmap(umem->buffer, umem->mmap_size); } static bool xdp_prog_changed_rx(struct test_spec *test) @@ -1794,9 +1813,18 @@ static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *i return TEST_FAILURE; } - if (ifobj2) { + err = xsk_attach_xdp_progs(test, ifobj1, ifobj2); + if (err) { + ksft_print_msg("Error: failed to attach XDP programs: %d (%s)\n", + err, strerror(-err)); + return TEST_FAILURE; + } + test->use_barrier = !!ifobj2; + + if (test->use_barrier) { if (pthread_barrier_init(&barr, NULL, 2)) return TEST_FAILURE; + pkt_stream_reset(ifobj2->xsk->pkt_stream); } @@ -1804,29 +1832,27 @@ static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *i pkt_stream_reset(ifobj1->xsk->pkt_stream); pkts_in_flight = 0; - signal(SIGUSR1, handler); /*Spawn RX thread */ pthread_create(&t0, NULL, ifobj1->func_ptr, test); - if (ifobj2) { + if (test->use_barrier) { pthread_barrier_wait(&barr); if (pthread_barrier_destroy(&barr)) { - pthread_kill(t0, SIGUSR1); + test->use_barrier = false; + pthread_join(t0, NULL); clean_sockets(test, ifobj1); clean_umem(test, ifobj1, NULL); return TEST_FAILURE; } + } + if (ifobj2) { /*Spawn TX thread */ pthread_create(&t1, NULL, ifobj2->func_ptr, test); - pthread_join(t1, NULL); } - if (!ifobj2) - pthread_kill(t0, SIGUSR1); - else - pthread_join(t0, NULL); + pthread_join(t0, NULL); if (test->total_steps == test->current_step || test->fail) { clean_sockets(test, ifobj1); @@ -1845,8 +1871,8 @@ static int testapp_validate_traffic(struct test_spec *test) struct ifobject *ifobj_rx = test->ifobj_rx; struct ifobject *ifobj_tx = test->ifobj_tx; - if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || - (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { + if ((ifobj_rx->xsk->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || + (ifobj_tx->xsk->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { ksft_print_msg("No huge pages present.\n"); return TEST_SKIP; } @@ -1863,8 +1889,6 @@ static int testapp_validate_traffic(struct test_spec *test) } } - if (xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx)) - return TEST_FAILURE; return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); } @@ -1953,12 +1977,13 @@ int testapp_xdp_prog_cleanup(struct test_spec *test) int testapp_headroom(struct test_spec *test) { - test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; + test->ifobj_rx->xsk->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; return testapp_validate_traffic(test); } int testapp_stats_rx_dropped(struct test_spec *test) { + struct xsk_umem_info *umem = test->ifobj_rx->xsk->umem; u32 umem_tr = test->ifobj_tx->umem_tailroom; if (test->mode == TEST_MODE_ZC) { @@ -1968,7 +1993,7 @@ int testapp_stats_rx_dropped(struct test_spec *test) if (pkt_stream_replace_half(test, (MIN_PKT_SIZE * 3) + umem_tr, 0)) return TEST_FAILURE; - test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - + umem->frame_headroom = umem->frame_size - XDP_PACKET_HEADROOM - (MIN_PKT_SIZE * 2) - umem_tr; if (pkt_stream_receive_half(test)) return TEST_FAILURE; @@ -2025,8 +2050,7 @@ int testapp_stats_fill_empty(struct test_spec *test) int testapp_send_receive_unaligned(struct test_spec *test) { - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; + test_spec_set_unaligned(test); /* Let half of the packets straddle a 4K buffer boundary */ if (pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2)) return TEST_FAILURE; @@ -2037,8 +2061,7 @@ int testapp_send_receive_unaligned(struct test_spec *test) int testapp_send_receive_unaligned_mb(struct test_spec *test) { test->mtu = MAX_ETH_JUMBO_SIZE; - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; + test_spec_set_unaligned(test); if (pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE)) return TEST_FAILURE; return testapp_validate_traffic(test); @@ -2064,8 +2087,8 @@ int testapp_send_receive_mb(struct test_spec *test) int testapp_invalid_desc_mb(struct test_spec *test) { - struct xsk_umem_info *umem = test->ifobj_tx->umem; - u64 umem_size = umem->num_frames * umem->frame_size; + struct xsk_umem_info *umem = test->ifobj_tx->xsk->umem; + u64 umem_sz = umem_size(umem); struct pkt pkts[] = { /* Valid packet for synch to start with */ {0, MIN_PKT_SIZE, 0, true, 0}, @@ -2075,13 +2098,16 @@ int testapp_invalid_desc_mb(struct test_spec *test) {0, 0, 0, false, 0}, /* Invalid address in the second frame */ {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, - {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {umem_sz * 2, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, MIN_PKT_SIZE, 0, false, 0}, /* Invalid len in the middle */ {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, + {0, MIN_PKT_SIZE, 0, false, 0}, /* Invalid options in the middle */ {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, + {0, MIN_PKT_SIZE, 0, false, 0}, /* Transmit 2 frags, receive 3 */ {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, @@ -2093,8 +2119,8 @@ int testapp_invalid_desc_mb(struct test_spec *test) if (umem->unaligned_mode) { /* Crossing a chunk boundary allowed */ - pkts[12].valid = true; - pkts[13].valid = true; + pkts[15].valid = true; + pkts[16].valid = true; } test->mtu = MAX_ETH_JUMBO_SIZE; @@ -2105,8 +2131,8 @@ int testapp_invalid_desc_mb(struct test_spec *test) int testapp_invalid_desc(struct test_spec *test) { - struct xsk_umem_info *umem = test->ifobj_tx->umem; - u64 umem_size = umem->num_frames * umem->frame_size; + struct xsk_umem_info *umem = test->ifobj_tx->xsk->umem; + u64 umem_sz = umem_size(umem); struct pkt pkts[] = { /* Zero packet address allowed */ {0, MIN_PKT_SIZE, 0, true}, @@ -2117,11 +2143,11 @@ int testapp_invalid_desc(struct test_spec *test) /* Packet too large */ {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, /* Up to end of umem allowed */ - {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, + {umem_sz - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, /* After umem ends */ - {umem_size, MIN_PKT_SIZE, 0, false}, + {umem_sz, MIN_PKT_SIZE, 0, false}, /* Straddle the end of umem */ - {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, + {umem_sz - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, /* Straddle a 4K boundary */ {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, /* Straddle a 2K boundary */ @@ -2139,9 +2165,9 @@ int testapp_invalid_desc(struct test_spec *test) } if (test->ifobj_tx->shared_umem) { - pkts[4].offset += umem_size; - pkts[5].offset += umem_size; - pkts[6].offset += umem_size; + pkts[4].offset += umem_sz; + pkts[5].offset += umem_sz; + pkts[6].offset += umem_sz; } if (pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts))) @@ -2202,16 +2228,33 @@ int testapp_xdp_shared_umem(struct test_spec *test) int testapp_poll_txq_tmout(struct test_spec *test) { + bool shared_umem = test->ifobj_tx->shared_umem; + int ret; + + test->poll_tmout = true; + /* + * POLL_TXQ_FULL exercises TX timeout setup in isolation. + * Keep TX out of shared-UMEM mode here so TX setup does not require + * RX UMEM to be initialized first. + */ + test->ifobj_tx->shared_umem = false; test->ifobj_tx->use_poll = true; /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ - test->ifobj_tx->umem->frame_size = 2048; - if (pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048)) + test->ifobj_tx->xsk->umem->frame_size = 2048; + if (pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048)) { + test->ifobj_tx->shared_umem = shared_umem; return TEST_FAILURE; - return testapp_validate_traffic_single_thread(test, test->ifobj_tx); + } + + ret = testapp_validate_traffic_single_thread(test, test->ifobj_tx); + test->ifobj_tx->shared_umem = shared_umem; + + return ret; } int testapp_poll_rxq_tmout(struct test_spec *test) { + test->poll_tmout = true; test->ifobj_rx->use_poll = true; return testapp_validate_traffic_single_thread(test, test->ifobj_rx); } @@ -2229,7 +2272,7 @@ int testapp_too_many_frags(struct test_spec *test) max_frags += 1; } - pkts = calloc(2 * max_frags + 2, sizeof(struct pkt)); + pkts = calloc(2 * max_frags + 3, sizeof(struct pkt)); if (!pkts) return TEST_FAILURE; @@ -2247,24 +2290,30 @@ int testapp_too_many_frags(struct test_spec *test) } pkts[max_frags].options = 0; - /* An invalid packet with the max amount of frags but signals packet - * continues on the last frag - */ - for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { + /* An invalid packet with the max + 1 amount of frags */ + for (i = max_frags + 1; i < 2 * max_frags + 2; i++) { pkts[i].len = MIN_PKT_SIZE; pkts[i].options = XDP_PKT_CONTD; - pkts[i].valid = false; + pkts[i].valid = true; } + pkts[2 * max_frags + 1].options = 0; /* Valid packet for synch */ - pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; - pkts[2 * max_frags + 1].valid = true; + pkts[2 * max_frags + 2].len = MIN_PKT_SIZE; + pkts[2 * max_frags + 2].valid = true; - if (pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2)) { + if (pkt_stream_generate_custom(test, pkts, 2 * max_frags + 3)) { free(pkts); return TEST_FAILURE; } + /* The generated Tx stream must keep the too-big packet valid so that + * __send_pkts() accounts its descriptors in outstanding_tx. The Rx + * stream, however, must not expect this packet on the wire. + */ + test->ifobj_rx->xsk->pkt_stream->pkts[2].valid = false; + test->ifobj_rx->xsk->pkt_stream->nb_valid_entries--; + ret = testapp_validate_traffic(test); free(pkts); return ret; @@ -2337,8 +2386,7 @@ int testapp_send_receive(struct test_spec *test) int testapp_send_receive_2k_frame(struct test_spec *test) { - test->ifobj_tx->umem->frame_size = 2048; - test->ifobj_rx->umem->frame_size = 2048; + test_spec_set_frame_size(test, 2048); if (pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE)) return TEST_FAILURE; return testapp_validate_traffic(test); @@ -2363,34 +2411,30 @@ int testapp_aligned_inv_desc(struct test_spec *test) int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) { - test->ifobj_tx->umem->frame_size = 2048; - test->ifobj_rx->umem->frame_size = 2048; + test_spec_set_frame_size(test, 2048); return testapp_invalid_desc(test); } int testapp_unaligned_inv_desc(struct test_spec *test) { - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; + test_spec_set_unaligned(test); return testapp_invalid_desc(test); } int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) { - u64 page_size, umem_size; + u64 page_size, umem_sz; /* Odd frame size so the UMEM doesn't end near a page boundary. */ - test->ifobj_tx->umem->frame_size = 4001; - test->ifobj_rx->umem->frame_size = 4001; - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; + test_spec_set_frame_size(test, 4001); + test_spec_set_unaligned(test); /* This test exists to test descriptors that staddle the end of * the UMEM but not a page. */ page_size = sysconf(_SC_PAGESIZE); - umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; - assert(umem_size % page_size > MIN_PKT_SIZE); - assert(umem_size % page_size < page_size - MIN_PKT_SIZE); + umem_sz = umem_size(test->ifobj_tx->xsk->umem); + assert(umem_sz % page_size > MIN_PKT_SIZE); + assert(umem_sz % page_size < page_size - MIN_PKT_SIZE); return testapp_invalid_desc(test); } @@ -2402,8 +2446,7 @@ int testapp_aligned_inv_desc_mb(struct test_spec *test) int testapp_unaligned_inv_desc_mb(struct test_spec *test) { - test->ifobj_tx->umem->unaligned_mode = true; - test->ifobj_rx->umem->unaligned_mode = true; + test_spec_set_unaligned(test); return testapp_invalid_desc_mb(test); } @@ -2447,9 +2490,9 @@ int testapp_hw_sw_max_ring_size(struct test_spec *test) test->total_steps = 2; test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending; test->ifobj_tx->ring.rx_pending = test->ifobj_tx->ring.rx_max_pending; - test->ifobj_rx->umem->num_frames = max_descs; - test->ifobj_rx->umem->fill_size = max_descs; - test->ifobj_rx->umem->comp_size = max_descs; + test->ifobj_rx->xsk->umem->num_frames = max_descs; + test->ifobj_rx->xsk->umem->fill_size = max_descs; + test->ifobj_rx->xsk->umem->comp_size = max_descs; test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; @@ -2590,8 +2633,8 @@ struct ifobject *ifobject_create(void) if (!ifobj->xsk_arr) goto out_xsk_arr; - ifobj->umem = calloc(1, sizeof(*ifobj->umem)); - if (!ifobj->umem) + ifobj->xsk_arr[0].umem_real = calloc(1, sizeof(struct xsk_umem_info)); + if (!ifobj->xsk_arr[0].umem_real) goto out_umem; return ifobj; @@ -2605,7 +2648,9 @@ out_xsk_arr: void ifobject_delete(struct ifobject *ifobj) { - free(ifobj->umem); + if (ifobj->xsk_arr) + free(ifobj->xsk_arr[0].umem_real); + free(ifobj->xsk_arr); free(ifobj); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_xsk.h b/tools/testing/selftests/bpf/prog_tests/test_xsk.h index 1ab8aee4ce56..03753ddc5dcd 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_xsk.h +++ b/tools/testing/selftests/bpf/prog_tests/test_xsk.h @@ -83,6 +83,7 @@ typedef int (*test_func_t)(struct test_spec *test); struct xsk_socket_info { struct xsk_ring_cons rx; struct xsk_ring_prod tx; + struct xsk_umem_info *umem_real; struct xsk_umem_info *umem; struct xsk_socket *xsk; struct pkt_stream *pkt_stream; @@ -102,6 +103,7 @@ struct xsk_umem_info { struct xsk_ring_cons cq; struct xsk_umem *umem; u64 next_buffer; + u64 mmap_size; u32 num_frames; u32 frame_headroom; void *buffer; @@ -123,7 +125,6 @@ struct ifobject { char ifname[MAX_INTERFACE_NAME_CHARS]; struct xsk_socket_info *xsk; struct xsk_socket_info *xsk_arr; - struct xsk_umem_info *umem; thread_func_t func_ptr; validation_func_t validation_func; struct xsk_xdp_progs *xdp_progs; @@ -206,6 +207,8 @@ struct test_spec { bool set_ring; bool adjust_tail; bool adjust_tail_support; + bool poll_tmout; + bool use_barrier; enum test_mode mode; char name[MAX_TEST_NAME_SIZE]; }; diff --git a/tools/testing/selftests/bpf/prog_tests/timer_mim.c b/tools/testing/selftests/bpf/prog_tests/timer_mim.c index c930c7d7105b..fa7bb769ca31 100644 --- a/tools/testing/selftests/bpf/prog_tests/timer_mim.c +++ b/tools/testing/selftests/bpf/prog_tests/timer_mim.c @@ -59,10 +59,32 @@ void serial_test_timer_mim(void) int err; old_print_fn = libbpf_set_print(NULL); - timer_reject_skel = timer_mim_reject__open_and_load(); - libbpf_set_print(old_print_fn); - if (!ASSERT_ERR_PTR(timer_reject_skel, "timer_reject_skel_load")) + timer_reject_skel = timer_mim_reject__open(); + if (!ASSERT_OK_PTR(timer_reject_skel, "timer_reject_skel_open")) + goto cleanup; + bpf_program__set_autoload(timer_reject_skel->progs.test1, true); + err = timer_mim_reject__load(timer_reject_skel); + ASSERT_ERR(err, "timer_reject_skel_load"); + timer_mim_reject__destroy(timer_reject_skel); + + timer_reject_skel = timer_mim_reject__open(); + if (!ASSERT_OK_PTR(timer_reject_skel, "callback_reject_skel_open")) goto cleanup; + bpf_program__set_autoload(timer_reject_skel->progs.callback_map_uid_mismatch, true); + err = timer_mim_reject__load(timer_reject_skel); + ASSERT_ERR(err, "callback_reject_skel_load"); + timer_mim_reject__destroy(timer_reject_skel); + + timer_reject_skel = timer_mim_reject__open(); + if (!ASSERT_OK_PTR(timer_reject_skel, "callback_accept_skel_open")) + goto cleanup; + bpf_program__set_autoload(timer_reject_skel->progs.callback_map_uid_match, true); + err = timer_mim_reject__load(timer_reject_skel); + if (!ASSERT_OK(err, "callback_accept_skel_load")) + goto cleanup; + timer_mim_reject__destroy(timer_reject_skel); + timer_reject_skel = NULL; + libbpf_set_print(old_print_fn); timer_skel = timer_mim__open_and_load(); if (!timer_skel && errno == EOPNOTSUPP) { @@ -75,6 +97,7 @@ void serial_test_timer_mim(void) err = timer_mim(timer_skel); ASSERT_OK(err, "timer_mim"); cleanup: + libbpf_set_print(old_print_fn); timer_mim__destroy(timer_skel); timer_mim_reject__destroy(timer_reject_skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/token.c b/tools/testing/selftests/bpf/prog_tests/token.c index b81dde283052..f2f5d36ae00a 100644 --- a/tools/testing/selftests/bpf/prog_tests/token.c +++ b/tools/testing/selftests/bpf/prog_tests/token.c @@ -1,9 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ #define _GNU_SOURCE -#include <test_progs.h> #include <bpf/btf.h> -#include "cap_helpers.h" #include <fcntl.h> #include <sched.h> #include <signal.h> @@ -15,9 +13,17 @@ #include <sys/stat.h> #include <sys/syscall.h> #include <sys/un.h> + +#include "bpf_util.h" +#include "cap_helpers.h" +#include "sysctl_helpers.h" +#include "test_progs.h" +#include "trace_helpers.h" + #include "priv_map.skel.h" #include "priv_prog.skel.h" #include "dummy_st_ops_success.skel.h" +#include "token_kallsyms.skel.h" #include "token_lsm.skel.h" #include "priv_freplace_prog.skel.h" @@ -1045,6 +1051,58 @@ err_out: return -EINVAL; } +static bool kallsyms_has_bpf_func(struct ksyms *ksyms, const char *func_name) +{ + char name[256]; + int i; + + for (i = 0; i < ksyms->sym_cnt; i++) { + if (sscanf(ksyms->syms[i].name, "bpf_prog_%*[^_]_%255s", name) == 1 && + strcmp(name, func_name) == 0) + return true; + } + return false; +} + +static int userns_obj_priv_prog_kallsyms(int mnt_fd, struct token_lsm *lsm_skel) +{ + const char *func_names[] = { "xdp_main", "token_ksym_subprog" }; + LIBBPF_OPTS(bpf_object_open_opts, opts); + struct token_kallsyms *skel; + struct ksyms *ksyms = NULL; + char buf[256]; + int i, err; + + snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd); + opts.bpf_token_path = buf; + skel = token_kallsyms__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "token_kallsyms__open_opts")) + return -EINVAL; + + err = token_kallsyms__load(skel); + if (!ASSERT_OK(err, "token_kallsyms__load")) + goto cleanup; + + ksyms = load_kallsyms_local(); + if (!ASSERT_OK_PTR(ksyms, "load_kallsyms_local")) { + err = -EINVAL; + goto cleanup; + } + + for (i = 0; i < ARRAY_SIZE(func_names); i++) { + if (!ASSERT_TRUE(kallsyms_has_bpf_func(ksyms, func_names[i]), + func_names[i])) { + err = -EINVAL; + break; + } + } + +cleanup: + free_kallsyms_local(ksyms); + token_kallsyms__destroy(skel); + return err; +} + #define bit(n) (1ULL << (n)) static int userns_bpf_token_info(int mnt_fd, struct token_lsm *lsm_skel) @@ -1082,7 +1140,7 @@ cleanup: return err; } -void test_token(void) +void serial_test_token(void) { if (test__start_subtest("map_token")) { struct bpffs_opts opts = { @@ -1194,4 +1252,26 @@ void test_token(void) subtest_userns(&opts, userns_bpf_token_info); } + if (test__start_subtest("obj_priv_prog_kallsyms")) { + char perf_paranoid_orig[32] = {}; + char kptr_restrict_orig[32] = {}; + struct bpffs_opts opts = { + .cmds = bit(BPF_BTF_LOAD) | bit(BPF_PROG_LOAD), + .progs = bit(BPF_PROG_TYPE_XDP), + .attachs = ~0ULL, + }; + + if (sysctl_set_or_fail("/proc/sys/kernel/perf_event_paranoid", perf_paranoid_orig, "0")) + goto cleanup; + if (sysctl_set_or_fail("/proc/sys/kernel/kptr_restrict", kptr_restrict_orig, "0")) + goto cleanup; + + subtest_userns(&opts, userns_obj_priv_prog_kallsyms); + +cleanup: + if (perf_paranoid_orig[0]) + sysctl_set_or_fail("/proc/sys/kernel/perf_event_paranoid", NULL, perf_paranoid_orig); + if (kptr_restrict_orig[0]) + sysctl_set_or_fail("/proc/sys/kernel/kptr_restrict", NULL, kptr_restrict_orig); + } } diff --git a/tools/testing/selftests/bpf/prog_tests/tp_btf_ids.c b/tools/testing/selftests/bpf/prog_tests/tp_btf_ids.c new file mode 100644 index 000000000000..c0e7e11e71b8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tp_btf_ids.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpf/btf.h> + +#define TRACEFS "/sys/kernel/tracing" +#define DEBUGFS_TRACING "/sys/kernel/debug/tracing" +#define EVENT_SUBPATH "events/bpf_testmod/bpf_testmod_test_read/btf_ids" + +struct btf_ids_info { + __u32 obj_id; + __u32 raw_id; + __u32 tp_id; +}; + +static const char *btf_ids_path(char *buf, size_t sz) +{ + if (access(TRACEFS "/trace", F_OK) == 0) + snprintf(buf, sz, "%s/%s", TRACEFS, EVENT_SUBPATH); + else + snprintf(buf, sz, "%s/%s", DEBUGFS_TRACING, EVENT_SUBPATH); + return buf; +} + +static int read_btf_ids(struct btf_ids_info *info) +{ + char path[256], buf[256]; + int fd, n; + + fd = open(btf_ids_path(path, sizeof(path)), O_RDONLY); + if (fd < 0) + return -errno; + + n = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (n <= 0) + return -EIO; + buf[n] = '\0'; + + if (sscanf(buf, + "btf_obj_id: %u\nraw_btf_id: %u\ntp_btf_id: %u\n", + &info->obj_id, &info->raw_id, &info->tp_id) != 3) + return -EINVAL; + return 0; +} + +static const char *param_name(struct btf *btf, const struct btf_param *p) +{ + return btf__name_by_offset(btf, p->name_off); +} + +static const char *member_name(struct btf *btf, const struct btf_member *m) +{ + return btf__name_by_offset(btf, m->name_off); +} + +void test_tp_btf_ids(void) +{ + const struct btf_type *proto_t, *rec_t; + const struct btf_param *params; + const struct btf_member *members; + struct btf_ids_info info; + struct btf *vmlinux_btf, *btf; + const char *name; + int err; + + if (!env.has_testmod) { + test__skip(); + return; + } + + err = read_btf_ids(&info); + if (!ASSERT_OK(err, "read btf_ids")) + return; + + ASSERT_GT(info.obj_id, 0, "obj_id non-zero"); + ASSERT_GT(info.raw_id, 0, "raw_id non-zero"); + ASSERT_GT(info.tp_id, 0, "tp_id non-zero"); + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(vmlinux_btf, "load vmlinux BTF")) + return; + + /* Module BTF is split BTF; load with vmlinux as base. */ + btf = btf__load_from_kernel_by_id_split(info.obj_id, vmlinux_btf); + if (!ASSERT_OK_PTR(btf, "load module BTF")) { + btf__free(vmlinux_btf); + return; + } + + /* + * raw_btf_id should be the FUNC_PROTO of __bpf_trace_<call>: + * void *__data, struct task_struct *task, + * struct bpf_testmod_test_read_ctx *ctx + */ + proto_t = btf__type_by_id(btf, info.raw_id); + if (!ASSERT_OK_PTR(proto_t, "raw type_by_id")) + goto out; + if (!ASSERT_TRUE(btf_is_func_proto(proto_t), "raw is FUNC_PROTO")) + goto out; + if (!ASSERT_EQ(btf_vlen(proto_t), 3, "func_proto arg count")) + goto out; + + params = btf_params(proto_t); + ASSERT_STREQ(param_name(btf, ¶ms[0]), "__data", "arg0 name"); + ASSERT_STREQ(param_name(btf, ¶ms[1]), "task", "arg1 name"); + ASSERT_STREQ(param_name(btf, ¶ms[2]), "ctx", "arg2 name"); + + /* + * tp_btf_id should be STRUCT trace_event_raw_<call> with the + * fields declared by TP_STRUCT__entry plus the common header. + */ + rec_t = btf__type_by_id(btf, info.tp_id); + if (!ASSERT_OK_PTR(rec_t, "tp type_by_id")) + goto out; + if (!ASSERT_TRUE(btf_is_struct(rec_t), "tp is STRUCT")) + goto out; + name = btf__name_by_offset(btf, rec_t->name_off); + ASSERT_STREQ(name, "trace_event_raw_bpf_testmod_test_read", + "tp struct name"); + if (!ASSERT_GE(btf_vlen(rec_t), 5, "tp struct field count")) + goto out; + + members = btf_members(rec_t); + ASSERT_STREQ(member_name(btf, &members[0]), "ent", "field0 name"); + ASSERT_STREQ(member_name(btf, &members[1]), "pid", "field1 name"); + ASSERT_STREQ(member_name(btf, &members[2]), "comm", "field2 name"); + ASSERT_STREQ(member_name(btf, &members[3]), "off", "field3 name"); + ASSERT_STREQ(member_name(btf, &members[4]), "len", "field4 name"); +out: + btf__free(btf); + btf__free(vmlinux_btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/trace_printk.c b/tools/testing/selftests/bpf/prog_tests/trace_printk.c index e56e88596d64..a5a8104c1ddd 100644 --- a/tools/testing/selftests/bpf/prog_tests/trace_printk.c +++ b/tools/testing/selftests/bpf/prog_tests/trace_printk.c @@ -6,18 +6,21 @@ #include "trace_printk.lskel.h" #define SEARCHMSG "testing,testing" +#define SEARCHMSG_UTF8 "ä¸æ–‡,测试" static void trace_pipe_cb(const char *str, void *data) { if (strstr(str, SEARCHMSG) != NULL) - (*(int *)data)++; + ((int *)data)[0]++; + if (strstr(str, SEARCHMSG_UTF8)) + ((int *)data)[1]++; } void serial_test_trace_printk(void) { struct trace_printk_lskel__bss *bss; struct trace_printk_lskel *skel; - int err = 0, found = 0; + int err = 0, found[2] = {}; skel = trace_printk_lskel__open(); if (!ASSERT_OK_PTR(skel, "trace_printk__open")) @@ -46,11 +49,24 @@ void serial_test_trace_printk(void) if (!ASSERT_GT(bss->trace_printk_ret, 0, "bss->trace_printk_ret")) goto cleanup; - /* verify our search string is in the trace buffer */ - ASSERT_OK(read_trace_pipe_iter(trace_pipe_cb, &found, 1000), - "read_trace_pipe_iter"); + if (!ASSERT_GT(bss->trace_printk_utf8_ran, 0, "bss->trace_printk_utf8_ran")) + goto cleanup; + + if (!ASSERT_GT(bss->trace_printk_utf8_ret, 0, "bss->trace_printk_utf8_ret")) + goto cleanup; + + if (!ASSERT_LT(bss->trace_printk_invalid_spec_ret, 0, + "bss->trace_printk_invalid_spec_ret")) + goto cleanup; + + /* verify our search strings are in the trace buffer */ + ASSERT_OK(read_trace_pipe_iter(trace_pipe_cb, found, 1000), + "read_trace_pipe_iter"); + + if (!ASSERT_EQ(found[0], bss->trace_printk_ran, "found")) + goto cleanup; - if (!ASSERT_EQ(found, bss->trace_printk_ran, "found")) + if (!ASSERT_EQ(found[1], bss->trace_printk_utf8_ran, "found_utf8")) goto cleanup; cleanup: diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_failure.c b/tools/testing/selftests/bpf/prog_tests/tracing_failure.c index f9f9e1cb87bf..eb585918f0d4 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_failure.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_failure.c @@ -76,6 +76,24 @@ static void test_fexit_noreturns(void) "Attaching fexit/fsession/fmod_ret to __noreturn function 'do_exit' is rejected."); } +static void test_fexit_int128_ret(void) +{ + /* + * __int128 is returned in a register pair on x86_64 and arm64, so + * bpf_testmod_test_int128_ret() is BTF-encoded and attachable and the + * verifier can reject its >8 byte return value. Other architectures + * return a __int128 differently (e.g. s390x returns larger values by + * reference, which makes pahole skip BTF encoding of the function), so + * only exercise this on x86_64 and arm64. + */ +#if defined(__x86_64__) || defined(__aarch64__) + test_tracing_fail_prog("fexit_int128_ret", + "with a >8 byte return value is not supported for this attach type"); +#else + test__skip(); +#endif +} + void test_tracing_failure(void) { if (test__start_subtest("bpf_spin_lock")) @@ -86,4 +104,6 @@ void test_tracing_failure(void) test_tracing_deny(); if (test__start_subtest("fexit_noreturns")) test_fexit_noreturns(); + if (test__start_subtest("fexit_int128_ret")) + test_fexit_int128_ret(); } diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c new file mode 100644 index 000000000000..0aa9532a05cf --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c @@ -0,0 +1,1029 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <bpf/btf.h> +#include <search.h> +#include "bpf/libbpf_internal.h" +#include "tracing_multi.skel.h" +#include "tracing_multi_module.skel.h" +#include "tracing_multi_intersect.skel.h" +#include "tracing_multi_session.skel.h" +#include "tracing_multi_fail.skel.h" +#include "tracing_multi_verifier.skel.h" +#include "tracing_multi_bench.skel.h" +#include "tracing_multi_rollback.skel.h" +#include "trace_helpers.h" + +static __u64 bpf_fentry_test_cookies[] = { + 8, /* bpf_fentry_test1 */ + 9, /* bpf_fentry_test2 */ + 7, /* bpf_fentry_test3 */ + 5, /* bpf_fentry_test4 */ + 4, /* bpf_fentry_test5 */ + 2, /* bpf_fentry_test6 */ + 3, /* bpf_fentry_test7 */ + 1, /* bpf_fentry_test8 */ + 10, /* bpf_fentry_test9 */ + 6, /* bpf_fentry_test10 */ +}; + +static const char * const bpf_fentry_test[] = { + "bpf_fentry_test1", + "bpf_fentry_test2", + "bpf_fentry_test3", + "bpf_fentry_test4", + "bpf_fentry_test5", + "bpf_fentry_test6", + "bpf_fentry_test7", + "bpf_fentry_test8", + "bpf_fentry_test9", + "bpf_fentry_test10", +}; + +static const char * const bpf_testmod_fentry_test[] = { + "bpf_testmod_fentry_test1", + "bpf_testmod_fentry_test2", + "bpf_testmod_fentry_test3", + "bpf_testmod_fentry_test7", + "bpf_testmod_fentry_test11", +}; + +#define FUNCS_CNT (ARRAY_SIZE(bpf_fentry_test)) + +static int get_random_funcs(const char **funcs) +{ + int i, cnt = 0; + + for (i = 0; i < FUNCS_CNT; i++) { + if (rand() % 2) + funcs[cnt++] = bpf_fentry_test[i]; + } + /* we always need at least one.. */ + if (!cnt) + funcs[cnt++] = bpf_fentry_test[rand() % FUNCS_CNT]; + return cnt; +} + +static int compare(const void *ppa, const void *ppb) +{ + const char *pa = *(const char **) ppa; + const char *pb = *(const char **) ppb; + + return strcmp(pa, pb); +} + +static void tdestroy_free_nop(void *ptr) +{ +} + +static __u32 *get_ids(const char * const funcs[], int funcs_cnt, const char *mod) +{ + struct btf *btf, *vmlinux_btf = NULL; + __u32 nr, type_id, cnt = 0; + void *root = NULL; + __u32 *ids = NULL; + int i, err = 0; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return NULL; + + if (mod) { + vmlinux_btf = btf; + btf = btf__load_module_btf(mod, vmlinux_btf); + if (!ASSERT_OK_PTR(btf, "btf__load_module_btf")) { + btf__free(vmlinux_btf); + return NULL; + } + } + + ids = calloc(funcs_cnt, sizeof(ids[0])); + if (!ids) + goto out; + + /* + * We sort function names by name and search them + * below for each function. + */ + for (i = 0; i < funcs_cnt; i++) { + if (!tsearch(&funcs[i], &root, compare)) { + ASSERT_FAIL("tsearch failed"); + err = -1; + goto error; + } + } + + nr = btf__type_cnt(btf); + for (type_id = 1; type_id < nr && cnt < funcs_cnt; type_id++) { + const struct btf_type *type; + const char *str, ***val; + unsigned int idx; + + type = btf__type_by_id(btf, type_id); + if (!type) { + err = -1; + break; + } + + if (BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) + continue; + + str = btf__name_by_offset(btf, type->name_off); + if (!str) { + err = -1; + break; + } + + val = tfind(&str, &root, compare); + if (!val) + continue; + + /* + * We keep pointer for each function name so we can get the original + * array index and have the resulting ids array matching the original + * function array. + * + * Doing it this way allow us to easily test the cookies support, + * because each cookie is attached to particular function/id. + */ + idx = *val - funcs; + ids[idx] = type_id; + cnt++; + } + +error: + if (err) { + free(ids); + ids = NULL; + } + +out: + tdestroy(root, tdestroy_free_nop); + btf__free(vmlinux_btf); + btf__free(btf); + return ids; +} + +static void tracing_multi_test_run(struct tracing_multi *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, prog_fd; + + prog_fd = bpf_program__fd(skel->progs.test_fentry); + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "test_run"); + + /* extra +1 count for sleepable programs */ + ASSERT_EQ(skel->bss->test_result_fentry, FUNCS_CNT + 1, "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, FUNCS_CNT + 1, "test_result_fexit"); +} + +static void test_skel_api(void) +{ + struct tracing_multi *skel; + int err; + + skel = tracing_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi__open_and_load")) + return; + + skel->bss->pid = getpid(); + + err = tracing_multi__attach(skel); + if (!ASSERT_OK(err, "tracing_multi__attach")) + goto cleanup; + + tracing_multi_test_run(skel); + +cleanup: + tracing_multi__destroy(skel); +} + +static void test_link_api_pattern(void) +{ + struct tracing_multi *skel; + + skel = tracing_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi__open_and_load")) + return; + + skel->bss->pid = getpid(); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_fentry_test*", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + "bpf_fentry_test*", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fentry_s = bpf_program__attach_tracing_multi(skel->progs.test_fentry_s, + "bpf_fentry_test1", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fentry_s, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit_s = bpf_program__attach_tracing_multi(skel->progs.test_fexit_s, + "bpf_fentry_test1", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fexit_s, "bpf_program__attach_tracing_multi")) + goto cleanup; + + tracing_multi_test_run(skel); + +cleanup: + tracing_multi__destroy(skel); +} + +static void test_link_api_ids(bool test_cookies) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi *skel; + size_t cnt = FUNCS_CNT; + __u32 *ids; + + skel = tracing_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi__open_and_load")) + return; + + skel->bss->pid = getpid(); + skel->bss->test_cookies = test_cookies; + + ids = get_ids(bpf_fentry_test, cnt, NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = cnt; + + if (test_cookies) + opts.cookies = bpf_fentry_test_cookies; + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + /* Only bpf_fentry_test1 is allowed for sleepable programs. */ + opts.cnt = 1; + skel->links.test_fentry_s = bpf_program__attach_tracing_multi(skel->progs.test_fentry_s, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fentry_s, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit_s = bpf_program__attach_tracing_multi(skel->progs.test_fexit_s, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fexit_s, "bpf_program__attach_tracing_multi")) + goto cleanup; + + tracing_multi_test_run(skel); + +cleanup: + tracing_multi__destroy(skel); + free(ids); +} + +static void test_module_skel_api(void) +{ + struct tracing_multi_module *skel = NULL; + int err; + + skel = tracing_multi_module__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi__open_and_load")) + return; + + skel->bss->pid = getpid(); + + err = tracing_multi_module__attach(skel); + if (!ASSERT_OK(err, "tracing_multi__attach")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + ASSERT_EQ(skel->bss->test_result_fentry, 5, "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, 5, "test_result_fexit"); + +cleanup: + tracing_multi_module__destroy(skel); +} + +static void test_module_link_api_pattern(void) +{ + struct tracing_multi_module *skel = NULL; + + skel = tracing_multi_module__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_module__open_and_load")) + return; + + skel->bss->pid = getpid(); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_testmod:bpf_testmod_fentry_test*", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + "bpf_testmod:bpf_testmod_fentry_test*", NULL); + if (!ASSERT_OK_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + ASSERT_EQ(skel->bss->test_result_fentry, 5, "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, 5, "test_result_fexit"); + +cleanup: + tracing_multi_module__destroy(skel); +} + +static void test_module_link_api_ids(void) +{ + size_t cnt = ARRAY_SIZE(bpf_testmod_fentry_test); + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_module *skel = NULL; + __u32 *ids; + + skel = tracing_multi_module__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_module__open_and_load")) + return; + + skel->bss->pid = getpid(); + + ids = get_ids(bpf_testmod_fentry_test, cnt, "bpf_testmod"); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = cnt; + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + NULL, &opts); + if (!ASSERT_OK_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + ASSERT_EQ(skel->bss->test_result_fentry, 5, "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, 5, "test_result_fexit"); + +cleanup: + tracing_multi_module__destroy(skel); + free(ids); +} + +static bool is_set(__u32 mask, __u32 bit) +{ + return (1 << bit) & mask; +} + +static void __test_intersect(__u32 mask, const struct bpf_program *progs[4], __u64 *test_results[4]) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_link *links[4] = { NULL }; + const char *funcs[FUNCS_CNT]; + __u64 expected[4]; + __u32 *ids, i; + int err, cnt; + + /* + * We have 4 programs in progs and the mask bits pick which + * of them gets attached to randomly chosen functions. + */ + for (i = 0; i < 4; i++) { + if (!is_set(mask, i)) + continue; + + cnt = get_random_funcs(funcs); + ids = get_ids(funcs, cnt, NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = cnt; + links[i] = bpf_program__attach_tracing_multi(progs[i], NULL, &opts); + free(ids); + + if (!ASSERT_OK_PTR(links[i], "bpf_program__attach_tracing_multi")) + goto cleanup; + + expected[i] = *test_results[i] + cnt; + } + + err = bpf_prog_test_run_opts(bpf_program__fd(progs[0]), &topts); + ASSERT_OK(err, "test_run"); + + for (i = 0; i < 4; i++) { + if (!is_set(mask, i)) + continue; + ASSERT_EQ(*test_results[i], expected[i], "test_results"); + } + +cleanup: + for (i = 0; i < 4; i++) + bpf_link__destroy(links[i]); +} + +static void test_intersect(void) +{ + struct tracing_multi_intersect *skel; + const struct bpf_program *progs[4]; + __u64 *test_results[4]; + __u32 i; + + skel = tracing_multi_intersect__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_intersect__open_and_load")) + return; + + skel->bss->pid = getpid(); + + progs[0] = skel->progs.fentry_1; + progs[1] = skel->progs.fexit_1; + progs[2] = skel->progs.fentry_2; + progs[3] = skel->progs.fexit_2; + + test_results[0] = &skel->bss->test_result_fentry_1; + test_results[1] = &skel->bss->test_result_fexit_1; + test_results[2] = &skel->bss->test_result_fentry_2; + test_results[3] = &skel->bss->test_result_fexit_2; + + for (i = 1; i < 16; i++) + __test_intersect(i, progs, test_results); + + tracing_multi_intersect__destroy(skel); +} + +static void test_fentry_after_multi(void) +{ + static const char * const funcs[] = { + "bpf_fentry_test1", + }; + struct bpf_link *fentry_link = NULL, *multi_link = NULL; + struct tracing_multi_intersect *skel = NULL; + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); + __u32 *ids = NULL; + int err; + + skel = tracing_multi_intersect__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_intersect__open_and_load")) + return; + + skel->bss->pid = getpid(); + + ids = get_ids(funcs, ARRAY_SIZE(funcs), NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = ARRAY_SIZE(funcs); + multi_link = bpf_program__attach_tracing_multi(skel->progs.fentry_1, NULL, &opts); + if (!ASSERT_OK_PTR(multi_link, "attach_multi")) + goto cleanup; + + fentry_link = bpf_program__attach(skel->progs.fentry); + if (!ASSERT_OK_PTR(fentry_link, "attach_fentry")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 1, "multi_fentry"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry"); + + err = bpf_link__destroy(fentry_link); + fentry_link = NULL; + if (!ASSERT_OK(err, "destroy_fentry")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run_multi")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 2, "multi_fentry_only"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry_detached"); + + err = bpf_link__destroy(multi_link); + multi_link = NULL; + if (!ASSERT_OK(err, "destroy_multi")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run_detached")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 2, "multi_fentry_detached"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry_still_detached"); + +cleanup: + bpf_link__destroy(fentry_link); + bpf_link__destroy(multi_link); + free(ids); + tracing_multi_intersect__destroy(skel); +} + +static void test_session(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct tracing_multi_session *skel; + int err, prog_fd; + + skel = tracing_multi_session__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_session__open_and_load")) + return; + + skel->bss->pid = getpid(); + + err = tracing_multi_session__attach(skel); + if (!ASSERT_OK(err, "tracing_multi_session__attach")) + goto cleanup; + + /* execute kernel session */ + prog_fd = bpf_program__fd(skel->progs.test_session_1); + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "test_run"); + + /* 10 for test_session_1, 1 for test_fsession_s */ + ASSERT_EQ(skel->bss->test_result_fentry, 11, "test_result_fentry"); + /* extra count (+1 for each fexit execution) for test_result_fexit cookie check/inc */ + ASSERT_EQ(skel->bss->test_result_fexit, 22, "test_result_fexit"); + + skel->bss->test_result_fentry = 0; + skel->bss->test_result_fexit = 0; + + /* execute bpf_testmo.ko session */ + ASSERT_OK(trigger_module_test_read(1), "trigger_read"); + + /* 5 for test_session_2 */ + ASSERT_EQ(skel->bss->test_result_fentry, 5, "test_result_fentry"); + /* extra count (+1 for each fexit execution) for test_result_fexit cookie */ + ASSERT_EQ(skel->bss->test_result_fexit, 10, "test_result_fexit"); + + +cleanup: + tracing_multi_session__destroy(skel); +} + +static void test_attach_api_fails(void) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + static const char * const func[] = { + "bpf_fentry_test2", + }; + struct tracing_multi_fail *skel = NULL; + __u32 ids[2] = {}, *ids2 = NULL; + __u64 cookies[2]; + + skel = tracing_multi_fail__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_fail__open_and_load")) + return; + + /* fail#1 (libbpf) pattern and opts NULL */ + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, NULL); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -EINVAL, "fail_1")) + goto cleanup; + + /* fail#2 (libbpf) pattern and ids */ + LIBBPF_OPTS_RESET(opts, + .ids = ids, + .cnt = 2, + ); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_fentry_test*", &opts); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -EINVAL, "fail_2")) + goto cleanup; + + /* fail#3 (libbpf) pattern and cookies */ + LIBBPF_OPTS_RESET(opts, + .ids = NULL, + .cnt = 2, + .cookies = cookies, + ); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_fentry_test*", &opts); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -EINVAL, "fail_3")) + goto cleanup; + + /* fail#4 (libbpf) bogus pattern */ + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + "bpf_not_really_a_function*", NULL); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -EINVAL, "fail_4")) + goto cleanup; + + /* fail#5 (kernel) abnormal cnt */ + LIBBPF_OPTS_RESET(opts, + .ids = ids, + .cnt = INT_MAX, + ); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -E2BIG, "fail_5")) + goto cleanup; + + /* fail#6 (kernel) attach sleepable program to not-allowed function */ + ids2 = get_ids(func, 1, NULL); + if (!ASSERT_OK_PTR(ids2, "get_ids")) + goto cleanup; + + LIBBPF_OPTS_RESET(opts, + .ids = ids2, + .cnt = 1, + ); + + skel->links.test_fentry_s = bpf_program__attach_tracing_multi(skel->progs.test_fentry_s, + NULL, &opts); + if (!ASSERT_EQ(libbpf_get_error(skel->links.test_fentry_s), -EINVAL, "fail_6")) + goto cleanup; + + /* fail#7 (kernel) attach with duplicate id */ + ids[0] = ids2[0]; + ids[1] = ids2[0]; + + LIBBPF_OPTS_RESET(opts, + .ids = ids, + .cnt = 2, + ); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + ASSERT_EQ(libbpf_get_error(skel->links.test_fentry), -EINVAL, "fail_7"); + +cleanup: + tracing_multi_fail__destroy(skel); + free(ids2); +} + +void serial_test_tracing_multi_bench_attach(void) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_bench *skel = NULL; + long attach_start_ns, attach_end_ns; + long detach_start_ns, detach_end_ns; + double attach_delta, detach_delta; + struct bpf_link *link = NULL; + size_t i, cap = 0, cnt = 0; + struct ksyms *ksyms = NULL; + void *root = NULL; + void *dups = NULL; + __u32 *ids = NULL; + __u32 nr, type_id; + struct btf *btf; + int err; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return; + + skel = tracing_multi_bench__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_bench__open_and_load")) + goto cleanup; + + if (!ASSERT_OK(bpf_get_ksyms(&ksyms, true), "get_syms")) + goto cleanup; + + /* Get all ftrace 'safe' symbols.. */ + for (i = 0; i < ksyms->filtered_cnt; i++) { + if (!tsearch(&ksyms->filtered_syms[i], &root, compare)) { + ASSERT_FAIL("tsearch failed"); + goto cleanup; + } + } + + /* + * Collect names that are not unique in kallsyms. The kernel resolves a + * tracing-multi BTF id to an address with kallsyms_lookup_name(), which + * returns the first symbol of that name. For a duplicate name that may + * be a different (non-ftrace-able) instance than the ftrace-able one in + * available_filter_functions, so attaching to it by BTF id fails with + * -ENOENT (e.g. t_start/t_next/t_stop). ksyms->syms is sorted by name, + * so equal names are adjacent. + */ + for (i = 1; i < ksyms->sym_cnt; i++) { + if (strcmp(ksyms->syms[i].name, ksyms->syms[i - 1].name)) + continue; + if (!tsearch(&ksyms->syms[i].name, &dups, compare)) { + ASSERT_FAIL("tsearch failed"); + goto cleanup; + } + } + + /* ..and filter them through BTF and btf_type_is_traceable_func. */ + nr = btf__type_cnt(btf); + for (type_id = 1; type_id < nr; type_id++) { + const struct btf_type *type; + const char *str; + + type = btf__type_by_id(btf, type_id); + if (!type) + break; + + if (BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) + continue; + + str = btf__name_by_offset(btf, type->name_off); + if (!str) + break; + + if (!tfind(&str, &root, compare)) + continue; + + /* Skip names that are not unique in kallsyms, see above. */ + if (tfind(&str, &dups, compare)) + continue; + + if (!btf_type_is_traceable_func(btf, type)) + continue; + + err = libbpf_ensure_mem((void **) &ids, &cap, sizeof(*ids), cnt + 1); + if (err) + goto cleanup; + + ids[cnt++] = type_id; + } + + opts.ids = ids; + opts.cnt = cnt; + + attach_start_ns = get_time_ns(); + link = bpf_program__attach_tracing_multi(skel->progs.bench, NULL, &opts); + attach_end_ns = get_time_ns(); + + if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi")) + goto cleanup; + + detach_start_ns = get_time_ns(); + bpf_link__destroy(link); + detach_end_ns = get_time_ns(); + + attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0; + detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0; + + printf("%s: found %lu functions\n", __func__, cnt); + printf("%s: attached in %7.3lfs\n", __func__, attach_delta); + printf("%s: detached in %7.3lfs\n", __func__, detach_delta); + +cleanup: + tracing_multi_bench__destroy(skel); + tdestroy(root, tdestroy_free_nop); + tdestroy(dups, tdestroy_free_nop); + free_kallsyms_local(ksyms); + free(ids); + btf__free(btf); +} + +static void tracing_multi_rollback_run(struct tracing_multi_rollback *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, prog_fd; + + prog_fd = bpf_program__fd(skel->progs.test_fentry); + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "test_run"); + + /* make sure the rollback code did not leave any program attached */ + ASSERT_EQ(skel->bss->test_result_fentry, 0, "test_result_fentry"); + ASSERT_EQ(skel->bss->test_result_fexit, 0, "test_result_fexit"); +} + +static void test_rollback_put(void) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_rollback *skel = NULL; + size_t cnt = FUNCS_CNT; + __u32 *ids = NULL; + int err; + + skel = tracing_multi_rollback__open(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_rollback__open")) + return; + + bpf_program__set_autoload(skel->progs.test_fentry, true); + bpf_program__set_autoload(skel->progs.test_fexit, true); + + err = tracing_multi_rollback__load(skel); + if (!ASSERT_OK(err, "tracing_multi_rollback__load")) + goto cleanup; + + ids = get_ids(bpf_fentry_test, cnt, NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + /* + * Mangle last id to trigger rollback, which needs to do put + * on get-ed trampolines. + */ + ids[9] = 0; + + opts.ids = ids; + opts.cnt = cnt; + + skel->bss->pid = getpid(); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + if (!ASSERT_ERR_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + NULL, &opts); + if (!ASSERT_ERR_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + /* We don't really attach any program, but let's make sure. */ + tracing_multi_rollback_run(skel); + +cleanup: + tracing_multi_rollback__destroy(skel); + free(ids); +} + +static void fillers_cleanup(struct tracing_multi_rollback **skels, int cnt) +{ + int i; + + for (i = 0; i < cnt; i++) + tracing_multi_rollback__destroy(skels[i]); + + free(skels); +} + +static struct tracing_multi_rollback *extra_load_and_link(void) +{ + struct tracing_multi_rollback *skel; + int err; + + skel = tracing_multi_rollback__open(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_rollback__open")) + goto cleanup; + + bpf_program__set_autoload(skel->progs.extra, true); + + err = tracing_multi_rollback__load(skel); + if (!ASSERT_OK(err, "tracing_multi_rollback__load")) + goto cleanup; + + skel->links.extra = bpf_program__attach_trace(skel->progs.extra); + if (!ASSERT_OK_PTR(skel->links.extra, "bpf_program__attach_trace")) + goto cleanup; + + return skel; + +cleanup: + tracing_multi_rollback__destroy(skel); + return NULL; +} + +static struct tracing_multi_rollback **fillers_load_and_link(int max) +{ + struct tracing_multi_rollback **skels, *skel; + int i, err; + + skels = calloc(max + 1, sizeof(*skels)); + if (!ASSERT_OK_PTR(skels, "calloc")) + return NULL; + + for (i = 0; i < max; i++) { + skel = skels[i] = tracing_multi_rollback__open(); + if (!ASSERT_OK_PTR(skels[i], "tracing_multi_rollback__open")) + goto cleanup; + + bpf_program__set_autoload(skel->progs.filler, true); + + err = tracing_multi_rollback__load(skel); + if (!ASSERT_OK(err, "tracing_multi_rollback__load")) + goto cleanup; + + skel->links.filler = bpf_program__attach_trace(skel->progs.filler); + if (!ASSERT_OK_PTR(skels[i]->links.filler, "bpf_program__attach_trace")) + goto cleanup; + } + + return skels; + +cleanup: + fillers_cleanup(skels, i + 1); + return NULL; +} + +static void test_rollback_unlink(void) +{ + struct tracing_multi_rollback *skel = NULL, *extra; + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tracing_multi_rollback **fillers; + size_t cnt = FUNCS_CNT; + __u32 *ids = NULL; + int err, max; + + max = get_bpf_max_tramp_links(); + if (!ASSERT_GE(max, 1, "bpf_max_tramp_links")) + return; + + /* Attach maximum allowed programs to bpf_fentry_test10 */ + fillers = fillers_load_and_link(max); + if (!ASSERT_OK_PTR(fillers, "fillers_load_and_link")) + return; + + extra = extra_load_and_link(); + if (!ASSERT_OK_PTR(extra, "extra_load_and_link")) + goto cleanup; + + skel = tracing_multi_rollback__open(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_rollback__open")) + goto cleanup; + + bpf_program__set_autoload(skel->progs.test_fentry, true); + bpf_program__set_autoload(skel->progs.test_fexit, true); + + /* + * Attach tracing_multi link on bpf_fentry_test1-10, which will + * fail on bpf_fentry_test10 function, because it already has + * maximum allowed programs attached. + * + * The rollback needs to unlink already link-ed trampolines and + * put all of them. + */ + err = tracing_multi_rollback__load(skel); + if (!ASSERT_OK(err, "tracing_multi_rollback__load")) + goto cleanup; + + ids = get_ids(bpf_fentry_test, cnt, NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = cnt; + + skel->bss->pid = getpid(); + + skel->links.test_fentry = bpf_program__attach_tracing_multi(skel->progs.test_fentry, + NULL, &opts); + if (!ASSERT_ERR_PTR(skel->links.test_fentry, "bpf_program__attach_tracing_multi")) + goto cleanup; + + skel->links.test_fexit = bpf_program__attach_tracing_multi(skel->progs.test_fexit, + NULL, &opts); + if (!ASSERT_ERR_PTR(skel->links.test_fexit, "bpf_program__attach_tracing_multi")) + goto cleanup; + + tracing_multi_rollback_run(skel); + +cleanup: + fillers_cleanup(fillers, max); + tracing_multi_rollback__destroy(extra); + tracing_multi_rollback__destroy(skel); + free(ids); +} + +void serial_test_tracing_multi_attach_rollback(void) +{ + if (test__start_subtest("put")) + test_rollback_put(); + if (test__start_subtest("unlink")) + test_rollback_unlink(); +} + +void test_tracing_multi_test(void) +{ +#ifndef __x86_64__ + test__skip(); + return; +#endif + + if (test__start_subtest("skel_api")) + test_skel_api(); + if (test__start_subtest("link_api_pattern")) + test_link_api_pattern(); + if (test__start_subtest("link_api_ids")) + test_link_api_ids(false); + if (test__start_subtest("module_skel_api")) + test_module_skel_api(); + if (test__start_subtest("module_link_api_pattern")) + test_module_link_api_pattern(); + if (test__start_subtest("module_link_api_ids")) + test_module_link_api_ids(); + if (test__start_subtest("intersect")) + test_intersect(); + if (test__start_subtest("cookies")) + test_link_api_ids(true); + if (test__start_subtest("session")) + test_session(); + if (test__start_subtest("attach_api_fails")) + test_attach_api_fails(); + RUN_TESTS(tracing_multi_verifier); + if (test__start_subtest("fentry_after_multi")) + test_fentry_after_multi(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 6f8c0bfb0415..15b95d0235b5 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -4,6 +4,7 @@ #include <test_progs.h> #include "tracing_struct.skel.h" #include "tracing_struct_many_args.skel.h" +#include "tracing_struct_int128.skel.h" static void test_struct_args(void) { @@ -112,6 +113,39 @@ destroy_skel: tracing_struct_many_args__destroy(skel); } +static void test_int128_args(void) +{ + /* + * __int128 arguments are passed in a register pair on x86_64 and + * arm64, which the trampoline packs into two context slots. Other + * architectures pass a __int128 differently (e.g. s390x passes larger + * arguments by reference), so only exercise this on x86_64 and arm64. + */ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_struct_int128 *skel; + int err; + + skel = tracing_struct_int128__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_struct_int128__open_and_load")) + return; + + err = tracing_struct_int128__attach(skel); + if (!ASSERT_OK(err, "tracing_struct_int128__attach")) + goto destroy_skel; + + ASSERT_OK(trigger_module_test_read(256), "trigger_read"); + + ASSERT_EQ(skel->bss->t_b, 2, "t:b"); + ASSERT_EQ(skel->bss->t_c, 3, "t:c"); + ASSERT_EQ(skel->bss->t_ret, 6, "t ret"); + +destroy_skel: + tracing_struct_int128__destroy(skel); +#else + test__skip(); +#endif +} + static void test_union_args(void) { struct tracing_struct *skel; @@ -145,6 +179,8 @@ void test_tracing_struct(void) test_struct_args(); if (test__start_subtest("struct_many_args")) test_struct_many_args(); + if (test__start_subtest("int128_args")) + test_int128_args(); if (test__start_subtest("union_args")) test_union_args(); } diff --git a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c index 6cd7349d4a2b..7321850db75f 100644 --- a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c +++ b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c @@ -30,16 +30,14 @@ static struct bpf_program *load_prog(char *file, char *name, struct inst *inst) return prog; } -/* TODO: use different target function to run in concurrent mode */ -void serial_test_trampoline_count(void) +void test_trampoline_count(void) { char *file = "test_trampoline_count.bpf.o"; char *const progs[] = { "fentry_test", "fmod_ret_test", "fexit_test" }; - int bpf_max_tramp_links, err, i, prog_fd; + int bpf_max_tramp_links, i; struct bpf_program *prog; struct bpf_link *link; struct inst *inst; - LIBBPF_OPTS(bpf_test_run_opts, opts); bpf_max_tramp_links = get_bpf_max_tramp_links(); if (!ASSERT_GE(bpf_max_tramp_links, 1, "bpf_max_tramp_links")) @@ -80,16 +78,7 @@ void serial_test_trampoline_count(void) goto cleanup; /* and finally execute the probe */ - prog_fd = bpf_program__fd(prog); - if (!ASSERT_GE(prog_fd, 0, "bpf_program__fd")) - goto cleanup; - - err = bpf_prog_test_run_opts(prog_fd, &opts); - if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) - goto cleanup; - - ASSERT_EQ(opts.retval & 0xffff, 33, "bpf_modify_return_test.result"); - ASSERT_EQ(opts.retval >> 16, 2, "bpf_modify_return_test.side_effect"); + ASSERT_OK(trigger_module_test_read(256), "trigger_module_test_read"); cleanup: for (; i >= 0; i--) { diff --git a/tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c b/tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c index 472f4f9fa95f..64404602b9ab 100644 --- a/tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c +++ b/tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c @@ -8,6 +8,7 @@ #include "cap_helpers.h" #include "bpf_util.h" +#include "sysctl_helpers.h" /* Using CAP_LAST_CAP is risky here, since it can get pulled in from * an old /usr/include/linux/capability.h and be < CAP_BPF; as a result @@ -36,26 +37,6 @@ static void process_perfbuf(void *ctx, int cpu, void *data, __u32 len) got_perfbuf_val = *(__u32 *)data; } -static int sysctl_set(const char *sysctl_path, char *old_val, const char *new_val) -{ - int ret = 0; - FILE *fp; - - fp = fopen(sysctl_path, "r+"); - if (!fp) - return -errno; - if (old_val && fscanf(fp, "%s", old_val) <= 0) { - ret = -ENOENT; - } else if (!old_val || strcmp(old_val, new_val) != 0) { - fseek(fp, 0, SEEK_SET); - if (fprintf(fp, "%s", new_val) < 0) - ret = -errno; - } - fclose(fp); - - return ret; -} - static void test_unpriv_bpf_disabled_positive(struct test_unpriv_bpf_disabled *skel, __u32 prog_id, int prog_fd, int perf_fd, char **map_paths, int *map_fds) diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c index 56cbea280fbd..f0baf5738b75 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c @@ -2,6 +2,7 @@ #include <unistd.h> #include <pthread.h> +#include <fcntl.h> #include <test_progs.h> #include "uprobe_multi.skel.h" #include "uprobe_multi_bench.skel.h" @@ -536,7 +537,37 @@ static void test_attach_api_fails(void) link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); if (!ASSERT_ERR(link_fd, "link_fd")) goto cleanup; - ASSERT_EQ(link_fd, -EINVAL, "pid_is_wrong"); + if (!ASSERT_EQ(link_fd, -EINVAL, "pid_is_wrong")) + goto cleanup; + + /* wrong path_fd */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = NULL, + .uprobe_multi.path_fd = -1, + .uprobe_multi.flags = BPF_F_UPROBE_MULTI_PATH_FD, + .uprobe_multi.offsets = (unsigned long *)&offset, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + if (!ASSERT_EQ(link_fd, -EBADF, "path_fd_is_wrong")) + goto cleanup; + + /* path and path_fd both set with BPF_F_UPROBE_MULTI_PATH_FD flag */ + LIBBPF_OPTS_RESET(opts, + .uprobe_multi.path = path, + .uprobe_multi.path_fd = 1, + .uprobe_multi.flags = BPF_F_UPROBE_MULTI_PATH_FD, + .uprobe_multi.offsets = (unsigned long *)&offset, + .uprobe_multi.cnt = 1, + ); + + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_ERR(link_fd, "link_fd")) + goto cleanup; + ASSERT_EQ(link_fd, -EINVAL, "path_and_path_fd_together"); cleanup: if (link_fd >= 0) @@ -757,6 +788,65 @@ static void test_link_api(void) __test_link_api(&child); } +static void test_link_api_path_fd(void) +{ + LIBBPF_OPTS(bpf_link_create_opts, opts); + const char *resolve_path = "/proc/self/exe"; + int prog_fd, link_fd = -1, path_fd = -1; + struct uprobe_multi *skel = NULL; + unsigned long *offsets = NULL; + const char *syms[3] = { + "uprobe_multi_func_1", + "uprobe_multi_func_2", + "uprobe_multi_func_3", + }; + int err; + + err = elf_resolve_syms_offsets(resolve_path, ARRAY_SIZE(syms), syms, + &offsets, STT_FUNC); + if (!ASSERT_OK(err, "elf_resolve_syms_offsets")) + return; + + path_fd = open(resolve_path, O_RDONLY); + if (!ASSERT_GE(path_fd, 0, "path_fd")) + goto cleanup; + + opts.uprobe_multi.path_fd = path_fd; + opts.uprobe_multi.offsets = offsets; + opts.uprobe_multi.cnt = ARRAY_SIZE(syms); + opts.uprobe_multi.flags = BPF_F_UPROBE_MULTI_PATH_FD; + + skel = uprobe_multi__open_and_load(); + if (!ASSERT_OK_PTR(skel, "uprobe_multi__open_and_load")) + goto cleanup; + + prog_fd = bpf_program__fd(skel->progs.uprobe); + link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &opts); + if (!ASSERT_GE(link_fd, 0, "bpf_link_create")) + goto cleanup; + + skel->bss->uprobe_multi_func_1_addr = (__u64)uprobe_multi_func_1; + skel->bss->uprobe_multi_func_2_addr = (__u64)uprobe_multi_func_2; + skel->bss->uprobe_multi_func_3_addr = (__u64)uprobe_multi_func_3; + skel->bss->pid = getpid(); + + uprobe_multi_func_1(); + uprobe_multi_func_2(); + uprobe_multi_func_3(); + + ASSERT_EQ(skel->bss->uprobe_multi_func_1_result, 1, "uprobe_multi_func_1_result"); + ASSERT_EQ(skel->bss->uprobe_multi_func_2_result, 1, "uprobe_multi_func_2_result"); + ASSERT_EQ(skel->bss->uprobe_multi_func_3_result, 1, "uprobe_multi_func_3_result"); + +cleanup: + if (link_fd >= 0) + close(link_fd); + if (path_fd >= 0) + close(path_fd); + uprobe_multi__destroy(skel); + free(offsets); +} + static struct bpf_program * get_program(struct uprobe_multi_consumers *skel, int prog) { @@ -1354,6 +1444,8 @@ void test_uprobe_multi_test(void) test_attach_api_syms(); if (test__start_subtest("link_api")) test_link_api(); + if (test__start_subtest("link_api_path_fd")) + test_link_api_path_fd(); if (test__start_subtest("bench_uprobe")) test_bench_attach_uprobe(); if (test__start_subtest("bench_usdt")) diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c index 955a37751b52..e193206fc5d2 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c @@ -4,6 +4,8 @@ #ifdef __x86_64__ +#define _GNU_SOURCE +#include <sched.h> #include <unistd.h> #include <asm/ptrace.h> #include <linux/compiler.h> @@ -13,11 +15,12 @@ #include <sys/syscall.h> #include <sys/prctl.h> #include <asm/prctl.h> +#include <stdnoreturn.h> #include "uprobe_syscall.skel.h" #include "uprobe_syscall_executed.skel.h" #include "bpf/libbpf_internal.h" -#define USDT_NOP .byte 0x0f, 0x1f, 0x44, 0x00, 0x00 +#define USDT_NOP .byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 #include "usdt.h" #pragma GCC diagnostic ignored "-Wattributes" @@ -26,7 +29,7 @@ __attribute__((aligned(16))) __nocf_check __weak __naked unsigned long uprobe_regs_trigger(void) { asm volatile ( - ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00\n" /* nop5 */ + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ "movq $0xdeadbeef, %rax\n" "ret\n" ); @@ -345,9 +348,9 @@ cleanup: __attribute__((aligned(16))) __nocf_check __weak __naked void uprobe_test(void) { - asm volatile (" \n" - ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00 \n" - "ret \n" + asm volatile ( + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ + "ret\n" ); } @@ -357,6 +360,50 @@ __nocf_check __weak void usdt_test(void) USDT(optimized_uprobe, usdt); } +/* + * Assembly-level red zone clobbering test. Stores known values in the + * red zone (below RSP), executes a nop10 (uprobe site), and checks that + * the values survived. Returns 0 if intact, 1 if clobbered. + * + * The nop5 optimization used CALL (which pushes a return address to + * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization + * should escape that by moving stackpointer below the redzone before + * doing the CALL. + * + * Align the code at 64 bytes, to make sure nop10 is not on page boundary. + */ +__attribute__((aligned(64))) +__nocf_check __weak __naked unsigned long uprobe_red_zone_test(void) +{ + asm volatile ( + "movabs $0x1111111111111111, %%rax\n" + "movq %%rax, -8(%%rsp)\n" + "movabs $0x2222222222222222, %%rax\n" + "movq %%rax, -16(%%rsp)\n" + "movabs $0x3333333333333333, %%rax\n" + "movq %%rax, -24(%%rsp)\n" + + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10: uprobe site */ + + "movabs $0x1111111111111111, %%rax\n" + "cmpq %%rax, -8(%%rsp)\n" + "jne 1f\n" + "movabs $0x2222222222222222, %%rax\n" + "cmpq %%rax, -16(%%rsp)\n" + "jne 1f\n" + "movabs $0x3333333333333333, %%rax\n" + "cmpq %%rax, -24(%%rsp)\n" + "jne 1f\n" + + "xorl %%eax, %%eax\n" + "retq\n" + "1:\n" + "movl $1, %%eax\n" + "retq\n" + ::: "rax", "memory" + ); +} + static int find_uprobes_trampoline(void *tramp_addr) { void *start, *end; @@ -388,14 +435,15 @@ static int find_uprobes_trampoline(void *tramp_addr) return ret; } -static unsigned char nop5[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 }; +static unsigned char nop10[10] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static unsigned char lea_rsp[5] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; -static void *find_nop5(void *fn) +static void *find_nop10(void *fn) { int i; - for (i = 0; i < 10; i++) { - if (!memcmp(nop5, fn + i, 5)) + for (i = 0; i < 128; i++) { + if (!memcmp(nop10, fn + i, 10)) return fn + i; } return NULL; @@ -403,6 +451,16 @@ static void *find_nop5(void *fn) typedef void (__attribute__((nocf_check)) *trigger_t)(void); +static void check_attach_notrigger(struct uprobe_syscall_executed *skel, + void *addr, int executed) +{ + unsigned char *op = addr; + + /* Make sure bpf program was not executed. */ + ASSERT_EQ(skel->bss->executed, executed, "executed"); + ASSERT_EQ(*op, 0xcc, "int3"); +} + static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigger, void *addr, int executed) { @@ -420,7 +478,8 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge ASSERT_EQ(skel->bss->executed, executed, "executed"); /* .. and check the trampoline is as expected. */ - call = (struct __arch_relative_insn *) addr; + ASSERT_OK(memcmp(addr, lea_rsp, 5), "lea_rsp"); + call = (struct __arch_relative_insn *)(addr + 5); tramp = (void *) (call + 1) + call->raddr; ASSERT_EQ(call->op, 0xe8, "call"); ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); @@ -428,21 +487,26 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge return tramp; } -static void check_detach(void *addr, void *tramp) +static bool check_detach(void *addr, void *tramp) { + static const unsigned char nop10_prefix[] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84 }; + bool ok = true; + /* [uprobes_trampoline] stays after detach */ - ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); - ASSERT_OK(memcmp(addr, nop5, 5), "nop5"); + ok &= ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); + ok &= ASSERT_OK(memcmp(addr, nop10_prefix, 5), "nop10_prefix"); + return ok; } -static void check(struct uprobe_syscall_executed *skel, struct bpf_link *link, - trigger_t trigger, void *addr, int executed) +static void *check(struct uprobe_syscall_executed *skel, struct bpf_link *link, + trigger_t trigger, void *addr, int executed) { void *tramp; tramp = check_attach(skel, trigger, addr, executed); bpf_link__destroy(link); check_detach(addr, tramp); + return tramp; } static void test_uprobe_legacy(void) @@ -453,6 +517,7 @@ static void test_uprobe_legacy(void) ); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -470,7 +535,30 @@ static void test_uprobe_legacy(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 2); + tramp = check(skel, link, uprobe_test, uprobe_test, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 2); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 4); /* uretprobe */ skel->bss->executed = 0; @@ -492,6 +580,7 @@ static void test_uprobe_multi(void) LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -512,7 +601,30 @@ static void test_uprobe_multi(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 2); + tramp = check(skel, link, uprobe_test, uprobe_test, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 2); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 4); /* uretprobe.multi */ skel->bss->executed = 0; @@ -536,6 +648,7 @@ static void test_uprobe_session(void) ); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -555,7 +668,30 @@ static void test_uprobe_session(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 4); + tramp = check(skel, link, uprobe_test, uprobe_test, 4); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 4); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 4, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 8); cleanup: uprobe_syscall_executed__destroy(skel); @@ -565,11 +701,11 @@ static void test_uprobe_usdt(void) { struct uprobe_syscall_executed *skel; struct bpf_link *link; - void *addr; + void *addr, *tramp; errno = 0; - addr = find_nop5(usdt_test); - if (!ASSERT_OK_PTR(addr, "find_nop5")) + addr = find_nop10(usdt_test); + if (!ASSERT_OK_PTR(addr, "find_nop10")) return; skel = uprobe_syscall_executed__open_and_load(); @@ -584,7 +720,32 @@ static void test_uprobe_usdt(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) goto cleanup; - check(skel, link, usdt_test, addr, 2); + tramp = check(skel, link, usdt_test, addr, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_usdt(skel->progs.test_usdt, + -1 /* all PIDs */, "/proc/self/exe", + "optimized_uprobe", "usdt", NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) + goto cleanup; + + check_attach_notrigger(skel, addr, 2); + + bpf_link__destroy(link); + if (!check_detach(addr, tramp)) + goto cleanup; + + usdt_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_usdt(skel->progs.test_usdt, + -1 /* all PIDs */, "/proc/self/exe", + "optimized_uprobe", "usdt", NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) + goto cleanup; + + check(skel, link, usdt_test, addr, 4); cleanup: uprobe_syscall_executed__destroy(skel); @@ -757,12 +918,124 @@ cleanup: #define __NR_uprobe 336 #endif +static void test_uprobe_red_zone(void) +{ + struct uprobe_syscall_executed *skel; + struct bpf_link *link; + void *nop10_addr; + size_t offset; + int i; + + nop10_addr = find_nop10(uprobe_red_zone_test); + if (!ASSERT_NEQ(nop10_addr, NULL, "find_nop10")) + return; + + skel = uprobe_syscall_executed__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + offset = get_uprobe_offset(nop10_addr); + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "attach_uprobe")) + goto cleanup; + + for (i = 0; i < 10; i++) + ASSERT_EQ(uprobe_red_zone_test(), 0, "red_zone_intact"); + + bpf_link__destroy(link); + +cleanup: + uprobe_syscall_executed__destroy(skel); +} + static void test_uprobe_error(void) { long err = syscall(__NR_uprobe); ASSERT_EQ(err, -1, "error"); - ASSERT_EQ(errno, ENXIO, "errno"); + ASSERT_EQ(errno, EPROTO, "errno"); +} + +__attribute__((aligned(16))) +__nocf_check __weak __naked void uprobe_fork_test(void) +{ + asm volatile ( + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ + "ret\n" + ); +} + +static noreturn int child_func(void *arg) +{ + struct uprobe_syscall_executed *skel = arg; + + /* Make sure the child's probe is still there and optimized.. */ + if (memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp))) + _exit(1); + + skel->bss->pid = getpid(); + + /* .. and it executes properly. */ + uprobe_fork_test(); + + if (skel->bss->executed != 3) + _exit(2); + + _exit(0); +} + +static void test_uprobe_fork_optimized(bool clone_vm) +{ + struct uprobe_syscall_executed *skel = NULL; + unsigned long offset; + int pid, status, err; + char stack[65535]; + + offset = get_uprobe_offset(&uprobe_fork_test); + if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) + return; + + skel = uprobe_syscall_executed__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto cleanup; + + skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + -1, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(skel->links.test_uprobe, "attach_uprobe")) + goto cleanup; + + skel->bss->pid = getpid(); + + /* Trigger optimization of uprobe in uprobe_fork_test. */ + uprobe_fork_test(); + uprobe_fork_test(); + + /* Make sure it got optimied. */ + if (!ASSERT_OK(memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp)), "optimized")) + goto cleanup; + + if (clone_vm) { + pid = clone(child_func, stack + sizeof(stack), CLONE_VM|SIGCHLD, skel); + if (!ASSERT_GT(pid, 0, "clone")) + goto cleanup; + } else { + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) + goto cleanup; + if (pid == 0) + child_func(skel); + } + + /* Wait for the child and verify it exited properly with 0. */ + err = waitpid(pid, &status, 0); + if (ASSERT_EQ(err, pid, "waitpid")) { + ASSERT_EQ(WIFEXITED(status), 1, "child_exited"); + ASSERT_EQ(WEXITSTATUS(status), 0, "child_exit_code"); + } + +cleanup: + uprobe_syscall_executed__destroy(skel); } static void __test_uprobe_syscall(void) @@ -783,6 +1056,12 @@ static void __test_uprobe_syscall(void) test_uprobe_usdt(); if (test__start_subtest("uprobe_race")) test_uprobe_race(); + if (test__start_subtest("uprobe_red_zone")) + test_uprobe_red_zone(); + if (test__start_subtest("uprobe_optimized_fork")) + test_uprobe_fork_optimized(false); + if (test__start_subtest("uprobe_optimized_clone_vm")) + test_uprobe_fork_optimized(true); if (test__start_subtest("uprobe_error")) test_uprobe_error(); if (test__start_subtest("uprobe_regs_equal")) diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c index f4be5269fa90..8004c9568ffa 100644 --- a/tools/testing/selftests/bpf/prog_tests/usdt.c +++ b/tools/testing/selftests/bpf/prog_tests/usdt.c @@ -247,6 +247,148 @@ cleanup: #undef TRIGGER } +#ifdef __x86_64__ +extern void usdt_1(void); +extern void usdt_2(void); +extern void usdt_red_zone_trigger(void); + +static unsigned char nop1[1] = { 0x90 }; +static unsigned char nop1_nop10_combo[11] = { 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; + +static void *find_instr(void *fn, unsigned char *instr, size_t cnt) +{ + int i; + + for (i = 0; i < 10; i++) { + if (!memcmp(instr, fn + i, cnt)) + return fn + i; + } + return NULL; +} + +static void subtest_optimized_attach(void) +{ + struct test_usdt *skel; + __u8 *addr_1, *addr_2; + + /* usdt_1 USDT probe has single nop instruction */ + addr_1 = find_instr(usdt_1, nop1_nop10_combo, 11); + if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop10_combo")) + return; + + addr_1 = find_instr(usdt_1, nop1, 1); + if (!ASSERT_OK_PTR(addr_1, "usdt_1_find_nop1")) + return; + + /* usdt_2 USDT probe has nop,nop10 instructions combo */ + addr_2 = find_instr(usdt_2, nop1_nop10_combo, 11); + if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop10_combo")) + return; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_usdt__open_and_load")) + return; + + skel->bss->expected_ip = (unsigned long) addr_1; + + /* + * Attach program on top of usdt_1 which is single nop probe, + * so the probe won't get optimized. + */ + skel->links.usdt_executed = bpf_program__attach_usdt(skel->progs.usdt_executed, + 0 /*self*/, "/proc/self/exe", + "optimized_attach", "usdt_1", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt_executed, "bpf_program__attach_usdt")) + goto cleanup; + + usdt_1(); + usdt_1(); + + /* int3 is on addr_1 address */ + ASSERT_EQ(*addr_1, 0xcc, "int3"); + ASSERT_EQ(skel->bss->executed, 2, "executed"); + + bpf_link__destroy(skel->links.usdt_executed); + + /* we expect the nop10 ip */ + skel->bss->expected_ip = (unsigned long) addr_2 + 1; + + /* + * Attach program on top of usdt_2 which is probe defined on top + * of nop1,nop10 combo, so the probe gets optimized on top of nop10. + */ + skel->links.usdt_executed = bpf_program__attach_usdt(skel->progs.usdt_executed, + 0 /*self*/, "/proc/self/exe", + "optimized_attach", "usdt_2", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt_executed, "bpf_program__attach_usdt")) + goto cleanup; + + usdt_2(); + usdt_2(); + + /* nop stays on addr_2 address */ + ASSERT_EQ(*addr_2, 0x90, "nop"); + + /* + * lea -0x80(%rsp), %rsp + * call ... + */ + static unsigned char expected[] = { 0x48, 0x8d, 0x64, 0x24, 0x80, 0xe8 }; + + ASSERT_MEMEQ(addr_2 + 1, expected, sizeof(expected), "lea_and_call"); + ASSERT_EQ(skel->bss->executed, 4, "executed"); + +cleanup: + test_usdt__destroy(skel); +} + +/* + * Test that USDT arguments survive nop10 optimization in a function where + * the compiler places operands in the red zone. + * + * Signal handlers are prone to having the compiler place USDT argument + * operands in the red zone (below rsp). + * + * The nop5 optimization used CALL (which pushes a return address to + * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization + * should escape that by moving stackpointer below the redzone before + * doing the CALL. + */ +static void subtest_optimized_red_zone(void) +{ + struct test_usdt *skel; + int i; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + skel->bss->expected_arg[0] = 0xDEADBEEF; + skel->bss->expected_arg[1] = 0xCAFEBABE; + skel->bss->expected_arg[2] = 0xFEEDFACE; + skel->bss->expected_pid = getpid(); + + skel->links.usdt_check_arg = bpf_program__attach_usdt( + skel->progs.usdt_check_arg, 0, "/proc/self/exe", + "optimized_attach", "usdt_red_zone", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt_check_arg, "attach_usdt_red_zone")) + goto cleanup; + + for (i = 0; i < 10; i++) + usdt_red_zone_trigger(); + + ASSERT_EQ(skel->bss->arg_total, 10, "arg_total"); + ASSERT_EQ(skel->bss->arg_bad, 0, "arg_bad"); + ASSERT_EQ(skel->bss->arg_last[0], 0xDEADBEEF, "arg_last_1"); + ASSERT_EQ(skel->bss->arg_last[1], 0xCAFEBABE, "arg_last_2"); + ASSERT_EQ(skel->bss->arg_last[2], 0xFEEDFACE, "arg_last_3"); + +cleanup: + test_usdt__destroy(skel); +} + +#endif + unsigned short test_usdt_100_semaphore SEC(".probes"); unsigned short test_usdt_300_semaphore SEC(".probes"); unsigned short test_usdt_400_semaphore SEC(".probes"); @@ -516,6 +658,10 @@ void test_usdt(void) #ifdef __x86_64__ if (test__start_subtest("basic_optimized")) subtest_basic_usdt(true); + if (test__start_subtest("optimized_attach")) + subtest_optimized_attach(); + if (test__start_subtest("optimized_red_zone")) + subtest_optimized_red_zone(); #endif if (test__start_subtest("multispec")) subtest_multispec_usdt(); diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 302286a80154..8b439e194bcc 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -2,7 +2,10 @@ #include <test_progs.h> +#include "arena_kfunc.skel.h" +#include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" +#include "verifier_align.skel.h" #include "verifier_and.skel.h" #include "verifier_arena.skel.h" #include "verifier_arena_large.skel.h" @@ -20,7 +23,9 @@ #include "verifier_bpf_trap.skel.h" #include "verifier_bswap.skel.h" #include "verifier_btf_ctx_access.skel.h" +#include "verifier_btf_flex_array.skel.h" #include "verifier_btf_unreliable_prog.skel.h" +#include "verifier_call_large_imm.skel.h" #include "verifier_cfg.skel.h" #include "verifier_cgroup_inv_retcode.skel.h" #include "verifier_cgroup_skb.skel.h" @@ -36,6 +41,7 @@ #include "verifier_div0.skel.h" #include "verifier_div_mod_bounds.skel.h" #include "verifier_div_overflow.skel.h" +#include "verifier_flow_keys.skel.h" #include "verifier_global_subprogs.skel.h" #include "verifier_global_ptr_args.skel.h" #include "verifier_gotol.skel.h" @@ -48,21 +54,25 @@ #include "verifier_iterating_callbacks.skel.h" #include "verifier_jeq_infer_not_null.skel.h" #include "verifier_jit_convergence.skel.h" +#include "verifier_kfunc_perfmon.skel.h" #include "verifier_ld_ind.skel.h" #include "verifier_ldsx.skel.h" #include "verifier_leak_ptr.skel.h" #include "verifier_linked_scalars.skel.h" #include "verifier_live_stack.skel.h" +#include "verifier_liveness_exp.skel.h" #include "verifier_load_acquire.skel.h" #include "verifier_loops1.skel.h" #include "verifier_lwt.skel.h" #include "verifier_map_in_map.skel.h" +#include "verifier_map_lookup_refine.skel.h" #include "verifier_map_ptr.skel.h" #include "verifier_map_ptr_mixing.skel.h" #include "verifier_map_ret_val.skel.h" #include "verifier_masking.skel.h" #include "verifier_may_goto_1.skel.h" #include "verifier_may_goto_2.skel.h" +#include "verifier_mem_size_reg.skel.h" #include "verifier_meta_access.skel.h" #include "verifier_movsx.skel.h" #include "verifier_mtu.skel.h" @@ -71,9 +81,11 @@ #include "verifier_netfilter_retcode.skel.h" #include "verifier_bpf_fastcall.skel.h" #include "verifier_or_jmp32_k.skel.h" +#include "verifier_percpu_addr.skel.h" #include "verifier_precision.skel.h" #include "verifier_prevent_map_lookup.skel.h" #include "verifier_private_stack.skel.h" +#include "verifier_ptr_to_buf.skel.h" #include "verifier_raw_stack.skel.h" #include "verifier_raw_tp_writable.skel.h" #include "verifier_reg_equal.skel.h" @@ -89,9 +101,13 @@ #include "verifier_sockmap_mutate.skel.h" #include "verifier_spill_fill.skel.h" #include "verifier_spin_lock.skel.h" +#include "verifier_stack_arg.skel.h" +#include "verifier_stack_arg_order.skel.h" #include "verifier_stack_ptr.skel.h" #include "verifier_store_release.skel.h" +#include "verifier_subprog_insn_stats.skel.h" #include "verifier_subprog_precision.skel.h" +#include "verifier_subprog_topo.skel.h" #include "verifier_subreg.skel.h" #include "verifier_tailcall.skel.h" #include "verifier_tailcall_jit.skel.h" @@ -111,9 +127,12 @@ #include "verifier_xdp.skel.h" #include "verifier_xdp_direct_packet_access.skel.h" #include "verifier_bits_iter.skel.h" +#include "verifier_set_retval.skel.h" #include "verifier_lsm.skel.h" #include "verifier_jit_inline.skel.h" #include "irq.skel.h" +#include "verifier_ctx_ptr_param.skel.h" +#include "verifier_zext.skel.h" #define MAX_ENTRIES 11 @@ -149,6 +168,11 @@ static void run_tests_aux(const char *skel_name, #define RUN(skel) run_tests_aux(#skel, skel##__elf_bytes, NULL) +void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } + +void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } + +void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } void test_verifier_arena_large(void) { RUN(verifier_arena_large); } @@ -164,14 +188,16 @@ void test_verifier_bpf_get_stack(void) { RUN(verifier_bpf_get_stack); } void test_verifier_bpf_trap(void) { RUN(verifier_bpf_trap); } void test_verifier_bswap(void) { RUN(verifier_bswap); } void test_verifier_btf_ctx_access(void) { RUN(verifier_btf_ctx_access); } +void test_verifier_btf_flex_array(void) { RUN(verifier_btf_flex_array); } void test_verifier_btf_unreliable_prog(void) { RUN(verifier_btf_unreliable_prog); } +void test_verifier_call_large_imm(void) { RUN(verifier_call_large_imm); } void test_verifier_cfg(void) { RUN(verifier_cfg); } void test_verifier_cgroup_inv_retcode(void) { RUN(verifier_cgroup_inv_retcode); } void test_verifier_cgroup_skb(void) { RUN(verifier_cgroup_skb); } void test_verifier_cgroup_storage(void) { RUN(verifier_cgroup_storage); } void test_verifier_const(void) { RUN(verifier_const); } void test_verifier_const_or(void) { RUN(verifier_const_or); } -void test_verifier_ctx(void) { RUN(verifier_ctx); } +void test_verifier_ctx(void) { RUN_TESTS(verifier_ctx); } void test_verifier_ctx_sk_msg(void) { RUN(verifier_ctx_sk_msg); } void test_verifier_d_path(void) { RUN(verifier_d_path); } void test_verifier_default_trusted_ptr(void) { RUN_TESTS(verifier_default_trusted_ptr); } @@ -180,6 +206,7 @@ void test_verifier_direct_stack_access_wraparound(void) { RUN(verifier_direct_st void test_verifier_div0(void) { RUN(verifier_div0); } void test_verifier_div_mod_bounds(void) { RUN(verifier_div_mod_bounds); } void test_verifier_div_overflow(void) { RUN(verifier_div_overflow); } +void test_verifier_flow_keys(void) { RUN(verifier_flow_keys); } void test_verifier_global_subprogs(void) { RUN(verifier_global_subprogs); } void test_verifier_global_ptr_args(void) { RUN(verifier_global_ptr_args); } void test_verifier_gotol(void) { RUN(verifier_gotol); } @@ -192,21 +219,25 @@ void test_verifier_int_ptr(void) { RUN(verifier_int_ptr); } void test_verifier_iterating_callbacks(void) { RUN(verifier_iterating_callbacks); } void test_verifier_jeq_infer_not_null(void) { RUN(verifier_jeq_infer_not_null); } void test_verifier_jit_convergence(void) { RUN(verifier_jit_convergence); } +void test_verifier_kfunc_perfmon(void) { RUN(verifier_kfunc_perfmon); } void test_verifier_load_acquire(void) { RUN(verifier_load_acquire); } void test_verifier_ld_ind(void) { RUN(verifier_ld_ind); } void test_verifier_ldsx(void) { RUN(verifier_ldsx); } void test_verifier_leak_ptr(void) { RUN(verifier_leak_ptr); } void test_verifier_linked_scalars(void) { RUN(verifier_linked_scalars); } void test_verifier_live_stack(void) { RUN(verifier_live_stack); } +void test_verifier_liveness_exp(void) { RUN(verifier_liveness_exp); } void test_verifier_loops1(void) { RUN(verifier_loops1); } void test_verifier_lwt(void) { RUN(verifier_lwt); } void test_verifier_map_in_map(void) { RUN(verifier_map_in_map); } +void test_verifier_map_lookup_refine(void) { RUN(verifier_map_lookup_refine); } void test_verifier_map_ptr(void) { RUN(verifier_map_ptr); } void test_verifier_map_ptr_mixing(void) { RUN(verifier_map_ptr_mixing); } void test_verifier_map_ret_val(void) { RUN(verifier_map_ret_val); } void test_verifier_masking(void) { RUN(verifier_masking); } void test_verifier_may_goto_1(void) { RUN(verifier_may_goto_1); } void test_verifier_may_goto_2(void) { RUN(verifier_may_goto_2); } +void test_verifier_mem_size_reg(void) { RUN(verifier_mem_size_reg); } void test_verifier_meta_access(void) { RUN(verifier_meta_access); } void test_verifier_movsx(void) { RUN(verifier_movsx); } void test_verifier_mul(void) { RUN(verifier_mul); } @@ -214,9 +245,11 @@ void test_verifier_netfilter_ctx(void) { RUN(verifier_netfilter_ctx); } void test_verifier_netfilter_retcode(void) { RUN(verifier_netfilter_retcode); } void test_verifier_bpf_fastcall(void) { RUN(verifier_bpf_fastcall); } void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); } +void test_verifier_percpu_addr(void) { RUN(verifier_percpu_addr); } void test_verifier_precision(void) { RUN(verifier_precision); } void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); } void test_verifier_private_stack(void) { RUN(verifier_private_stack); } +void test_verifier_ptr_to_buf(void) { RUN(verifier_ptr_to_buf); } void test_verifier_raw_stack(void) { RUN(verifier_raw_stack); } void test_verifier_raw_tp_writable(void) { RUN(verifier_raw_tp_writable); } void test_verifier_reg_equal(void) { RUN(verifier_reg_equal); } @@ -232,9 +265,13 @@ void test_verifier_sock_addr(void) { RUN(verifier_sock_addr); } void test_verifier_sockmap_mutate(void) { RUN(verifier_sockmap_mutate); } void test_verifier_spill_fill(void) { RUN(verifier_spill_fill); } void test_verifier_spin_lock(void) { RUN(verifier_spin_lock); } +void test_verifier_stack_arg(void) { RUN(verifier_stack_arg); } +void test_verifier_stack_arg_order(void) { RUN(verifier_stack_arg_order); } void test_verifier_stack_ptr(void) { RUN(verifier_stack_ptr); } void test_verifier_store_release(void) { RUN(verifier_store_release); } +void test_verifier_subprog_insn_stats(void) { RUN(verifier_subprog_insn_stats); } void test_verifier_subprog_precision(void) { RUN(verifier_subprog_precision); } +void test_verifier_subprog_topo(void) { RUN(verifier_subprog_topo); } void test_verifier_subreg(void) { RUN(verifier_subreg); } void test_verifier_tailcall(void) { RUN(verifier_tailcall); } void test_verifier_tailcall_jit(void) { RUN(verifier_tailcall_jit); } @@ -253,10 +290,13 @@ void test_verifier_xadd(void) { RUN(verifier_xadd); } void test_verifier_xdp(void) { RUN(verifier_xdp); } void test_verifier_xdp_direct_packet_access(void) { RUN(verifier_xdp_direct_packet_access); } void test_verifier_bits_iter(void) { RUN(verifier_bits_iter); } +void test_verifier_set_retval(void) { RUN(verifier_set_retval); } void test_verifier_lsm(void) { RUN(verifier_lsm); } void test_irq(void) { RUN(irq); } void test_verifier_mtu(void) { RUN(verifier_mtu); } void test_verifier_jit_inline(void) { RUN(verifier_jit_inline); } +void test_verifier_ctx_ptr_param(void) { RUN(verifier_ctx_ptr_param); } +void test_verifier_zext(void) { RUN_TESTS(verifier_zext); } static int init_test_val_map(struct bpf_object *obj, char *map_name) { diff --git a/tools/testing/selftests/bpf/prog_tests/verifier_log.c b/tools/testing/selftests/bpf/prog_tests/verifier_log.c index aaa2854974c0..4542bb586d72 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier_log.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier_log.c @@ -25,10 +25,10 @@ static bool check_prog_load(int prog_fd, bool expect_err, const char *tag) static struct { /* strategically placed before others to avoid accidental modification by kernel */ - char filler[1024]; - char buf[1024]; + char filler[16384]; + char buf[16384]; /* strategically placed after buf[] to catch more accidental corruptions */ - char reference[1024]; + char reference[16384]; } logs; static const struct bpf_insn *insns; static size_t insn_cnt; @@ -317,6 +317,7 @@ static void verif_btf_log_subtest(bool bad_btf) res = load_btf(&opts, true); ASSERT_EQ(res, -ENOSPC, "half_log_fd"); ASSERT_EQ(strlen(logs.buf), 24, "log_fixed_25"); + strscpy(op_name, "log_fixed", sizeof(op_name)); ASSERT_STRNEQ(logs.buf, logs.reference, 24, op_name); /* validate rolling verifier log logic: try all variations of log buf diff --git a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c index 4d69d9d55e17..f327feb8e38c 100644 --- a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c +++ b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c @@ -18,6 +18,7 @@ #include <linux/keyctl.h> #include <sys/xattr.h> #include <linux/fsverity.h> +#include <linux/module_signature.h> #include <test_progs.h> #include "test_verify_pkcs7_sig.skel.h" @@ -33,29 +34,6 @@ #define SHA256_DIGEST_SIZE 32 #endif -/* In stripped ARM and x86-64 modules, ~ is surprisingly rare. */ -#define MODULE_SIG_STRING "~Module signature appended~\n" - -/* - * Module signature information block. - * - * The constituents of the signature section are, in order: - * - * - Signer's name - * - Key identifier - * - Signature data - * - Information block - */ -struct module_signature { - __u8 algo; /* Public-key crypto algorithm [0] */ - __u8 hash; /* Digest algorithm [0] */ - __u8 id_type; /* Key identifier type [PKEY_ID_PKCS7] */ - __u8 signer_len; /* Length of signer's name [0] */ - __u8 key_id_len; /* Length of key identifier [0] */ - __u8 __pad[3]; - __be32 sig_len; /* Length of signature data */ -}; - struct data { __u8 data[MAX_DATA_SIZE]; __u32 data_len; @@ -215,7 +193,7 @@ static int populate_data_item_mod(struct data *data_item) return 0; modlen = st.st_size; - marker_len = sizeof(MODULE_SIG_STRING) - 1; + marker_len = sizeof(MODULE_SIGNATURE_MARKER) - 1; fd = open(mod_path, O_RDONLY); if (fd == -1) @@ -228,7 +206,7 @@ static int populate_data_item_mod(struct data *data_item) if (mod == MAP_FAILED) return -errno; - if (strncmp(mod + modlen - marker_len, MODULE_SIG_STRING, marker_len)) { + if (strncmp(mod + modlen - marker_len, MODULE_SIGNATURE_MARKER, marker_len)) { ret = -EINVAL; goto out; } diff --git a/tools/testing/selftests/bpf/prog_tests/vmlinux.c b/tools/testing/selftests/bpf/prog_tests/vmlinux.c index 6fb2217d940b..b5fdd593910d 100644 --- a/tools/testing/selftests/bpf/prog_tests/vmlinux.c +++ b/tools/testing/selftests/bpf/prog_tests/vmlinux.c @@ -14,21 +14,61 @@ static void nsleep() (void)syscall(__NR_nanosleep, &ts, NULL); } +static const char *hrtimer_func = "hrtimer_start_range_ns"; + +static int setup_hrtimer_progs(struct test_vmlinux *skel) +{ + int err; + + if (libbpf_find_vmlinux_btf_id("hrtimer_start_range_ns_user", BPF_TRACE_FENTRY) > 0) + hrtimer_func = "hrtimer_start_range_ns_user"; + + err = bpf_program__set_attach_target(skel->progs.handle__fentry, 0, hrtimer_func); + if (err) + return err; + + /* + * Bare SEC("kprobe") has no target function, so attach it manually + * later after selecting the hrtimer function to probe. + */ + bpf_program__set_autoattach(skel->progs.handle__kprobe, false); + + return 0; +} + void test_vmlinux(void) { int err; struct test_vmlinux* skel; struct test_vmlinux__bss *bss; + struct bpf_link *kprobe_link = NULL; - skel = test_vmlinux__open_and_load(); - if (!ASSERT_OK_PTR(skel, "test_vmlinux__open_and_load")) + skel = test_vmlinux__open(); + if (!ASSERT_OK_PTR(skel, "test_vmlinux__open")) return; + + err = setup_hrtimer_progs(skel); + if (!ASSERT_OK(err, "setup_hrtimer_progs")) + goto cleanup; + + err = test_vmlinux__load(skel); + if (!ASSERT_OK(err, "test_vmlinux__load")) + goto cleanup; + bss = skel->bss; err = test_vmlinux__attach(skel); if (!ASSERT_OK(err, "test_vmlinux__attach")) goto cleanup; + /* manually attach kprobe with the selected function */ + if (hrtimer_func) { + kprobe_link = bpf_program__attach_kprobe(skel->progs.handle__kprobe, + false /* retprobe */, hrtimer_func); + if (!ASSERT_OK_PTR(kprobe_link, "bpf_program__attach_kprobe")) + goto cleanup; + } + /* trigger everything */ nsleep(); @@ -39,5 +79,6 @@ void test_vmlinux(void) ASSERT_TRUE(bss->fentry_called, "fentry"); cleanup: + bpf_link__destroy(kprobe_link); test_vmlinux__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/wakeup_source.c b/tools/testing/selftests/bpf/prog_tests/wakeup_source.c new file mode 100644 index 000000000000..ebfdc03271b9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/wakeup_source.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2026 Google LLC */ + +#include <test_progs.h> +#include <bpf/btf.h> +#include <fcntl.h> +#include "test_wakeup_source.skel.h" +#include "wakeup_source_fail.skel.h" +#include "progs/wakeup_source.h" + +static int lock_ws(const char *name) +{ + int fd; + ssize_t bytes; + + fd = open("/sys/power/wake_lock", O_WRONLY); + if (!ASSERT_OK_FD(fd, "open /sys/power/wake_lock")) + return -1; + + bytes = write(fd, name, strlen(name)); + close(fd); + if (!ASSERT_EQ(bytes, strlen(name), "write to wake_lock")) + return -1; + + return 0; +} + +static void unlock_ws(const char *name) +{ + int fd; + + fd = open("/sys/power/wake_unlock", O_WRONLY); + if (fd < 0) + return; + + write(fd, name, strlen(name)); + close(fd); +} + +struct rb_ctx { + const char *name; + bool found; + long long active_time_ns; + long long total_time_ns; +}; + +static int process_sample(void *ctx, void *data, size_t len) +{ + struct rb_ctx *rb_ctx = ctx; + struct wakeup_event_t *e = data; + + if (strcmp(e->name, rb_ctx->name) == 0) { + rb_ctx->found = true; + rb_ctx->active_time_ns = e->active_time_ns; + rb_ctx->total_time_ns = e->total_time_ns; + } + return 0; +} + +void test_wakeup_source(void) +{ + struct btf *btf; + int id; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf_vmlinux")) + return; + + id = btf__find_by_name_kind(btf, "bpf_wakeup_sources_get_head", BTF_KIND_FUNC); + btf__free(btf); + + if (id < 0) { + printf("%s:SKIP:bpf_wakeup_sources_get_head kfunc not found in BTF\n", __func__); + test__skip(); + return; + } + + if (test__start_subtest("iterate_and_verify_times")) { + struct test_wakeup_source *skel; + struct ring_buffer *rb = NULL; + struct rb_ctx rb_ctx = { + .name = "bpf_selftest_ws_times", + .found = false, + }; + int err; + + skel = test_wakeup_source__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + rb = ring_buffer__new(bpf_map__fd(skel->maps.rb), process_sample, &rb_ctx, NULL); + if (!ASSERT_OK_PTR(rb, "ring_buffer__new")) + goto destroy; + + /* Create a temporary wakeup source */ + if (!ASSERT_OK(lock_ws(rb_ctx.name), "lock_ws")) + goto unlock; + + err = bpf_prog_test_run_opts(bpf_program__fd( + skel->progs.iterate_wakeupsources), NULL); + ASSERT_OK(err, "bpf_prog_test_run"); + + ring_buffer__consume(rb); + + ASSERT_TRUE(rb_ctx.found, "found_test_ws_in_rb"); + ASSERT_GT(rb_ctx.active_time_ns, 0, "active_time_gt_0"); + ASSERT_GT(rb_ctx.total_time_ns, 0, "total_time_gt_0"); + +unlock: + unlock_ws(rb_ctx.name); +destroy: + if (rb) + ring_buffer__free(rb); + test_wakeup_source__destroy(skel); + } + + RUN_TESTS(wakeup_source_fail); +} diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_bonding.c b/tools/testing/selftests/bpf/prog_tests/xdp_bonding.c index e8ea26464349..c42488e445c2 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_bonding.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_bonding.c @@ -191,13 +191,18 @@ fail: return -1; } -static void bonding_cleanup(struct skeletons *skeletons) +static void link_cleanup(struct skeletons *skeletons) { - restore_root_netns(); while (skeletons->nlinks) { skeletons->nlinks--; bpf_link__destroy(skeletons->links[skeletons->nlinks]); } +} + +static void bonding_cleanup(struct skeletons *skeletons) +{ + restore_root_netns(); + link_cleanup(skeletons); ASSERT_OK(system("ip link delete bond1"), "delete bond1"); ASSERT_OK(system("ip link delete veth1_1"), "delete veth1_1"); ASSERT_OK(system("ip link delete veth1_2"), "delete veth1_2"); @@ -493,6 +498,90 @@ out: system("ip link del bond_nest2"); } +/* + * Test that XDP redirect via xdp_master_redirect() does not crash when + * the bond master device is not up. When bond is in round-robin mode but + * never opened, rr_tx_counter is NULL. + */ +static void test_xdp_bonding_redirect_no_up(struct skeletons *skeletons) +{ + struct nstoken *nstoken = NULL; + int xdp_pass_fd; + int veth1_ifindex; + int err; + char pkt[ETH_HLEN + 1]; + struct xdp_md ctx_in = {}; + + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = &pkt, + .data_size_in = sizeof(pkt), + .ctx_in = &ctx_in, + .ctx_size_in = sizeof(ctx_in), + .flags = BPF_F_TEST_XDP_LIVE_FRAMES, + .repeat = 1, + .batch_size = 1, + ); + + /* We can't use bonding_setup() because bond will be active */ + SYS(out, "ip netns add ns_rr_no_up"); + nstoken = open_netns("ns_rr_no_up"); + if (!ASSERT_OK_PTR(nstoken, "open ns_rr_no_up")) + goto out; + + /* bond0: active-backup, UP with slave veth0. + * Attaching native XDP to bond0 enables bpf_master_redirect_enabled_key + * globally. + */ + SYS(out, "ip link add bond0 type bond mode active-backup"); + SYS(out, "ip link add veth0 type veth peer name veth0p"); + SYS(out, "ip link set veth0 master bond0"); + SYS(out, "ip link set bond0 up"); + SYS(out, "ip link set veth0p up"); + + /* bond1: round-robin, never UP -> rr_tx_counter stays NULL */ + SYS(out, "ip link add bond1 type bond mode balance-rr"); + SYS(out, "ip link add veth1 type veth peer name veth1p"); + SYS(out, "ip link set veth1 master bond1"); + + veth1_ifindex = if_nametoindex("veth1"); + if (!ASSERT_GT(veth1_ifindex, 0, "veth1_ifindex")) + goto out; + + /* Attach native XDP to bond0 -> enables global redirect key */ + if (xdp_attach(skeletons, skeletons->xdp_tx->progs.xdp_tx, "bond0")) + goto out; + + /* Attach generic XDP (XDP_TX) to veth1. + * When packets arrive at veth1 via netif_receive_skb, do_xdp_generic() + * runs this program. XDP_TX + bond slave triggers xdp_master_redirect(). + */ + err = bpf_xdp_attach(veth1_ifindex, + bpf_program__fd(skeletons->xdp_tx->progs.xdp_tx), + XDP_FLAGS_SKB_MODE, NULL); + if (!ASSERT_OK(err, "attach generic XDP to veth1")) + goto out; + + /* Run BPF_PROG_TEST_RUN with XDP_PASS live frames on veth1. + * XDP_PASS frames become SKBs with skb->dev = veth1, entering + * netif_receive_skb -> do_xdp_generic -> xdp_master_redirect. + * Without the fix, bond_rr_gen_slave_id() dereferences NULL + * rr_tx_counter and crashes. + */ + xdp_pass_fd = bpf_program__fd(skeletons->xdp_dummy->progs.xdp_dummy_prog); + + memset(pkt, 0, sizeof(pkt)); + ctx_in.data_end = sizeof(pkt); + ctx_in.ingress_ifindex = veth1_ifindex; + + err = bpf_prog_test_run_opts(xdp_pass_fd, &opts); + ASSERT_OK(err, "xdp_pass test_run should not crash"); + +out: + link_cleanup(skeletons); + close_netns(nstoken); + SYS_NOFAIL("ip netns del ns_rr_no_up"); +} + static void test_xdp_bonding_features(struct skeletons *skeletons) { LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); @@ -738,6 +827,9 @@ void serial_test_xdp_bonding(void) if (test__start_subtest("xdp_bonding_redirect_multi")) test_xdp_bonding_redirect_multi(&skeletons); + if (test__start_subtest("xdp_bonding_redirect_no_up")) + test_xdp_bonding_redirect_no_up(&skeletons); + out: xdp_dummy__destroy(skeletons.xdp_dummy); xdp_tx__destroy(skeletons.xdp_tx); diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c b/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c index 26159e0499c7..448807676176 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_context_test_run.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> #include <network_helpers.h> +#include <linux/ipv6.h> +#include <arpa/inet.h> #include "test_xdp_context_test_run.skel.h" #include "test_xdp_meta.skel.h" @@ -8,9 +10,12 @@ #define TX_NAME "veth1" #define TX_NETNS "xdp_context_tx" #define RX_NETNS "xdp_context_rx" +#define RX_MAC "02:00:00:00:00:01" +#define TX_MAC "02:00:00:00:00:02" #define TAP_NAME "tap0" #define DUMMY_NAME "dum0" #define TAP_NETNS "xdp_context_tuntap" +#define LWT_NETNS "xdp_context_lwt" #define TEST_PAYLOAD_LEN 32 static const __u8 test_payload[TEST_PAYLOAD_LEN] = { @@ -187,6 +192,42 @@ static int write_test_packet(int tap_fd) return 0; } +/* Inject Ethernet+IPv6+UDP frame into TAP */ +static int write_test_packet_udp(int tap_fd) +{ + __u8 pkt[sizeof(struct ethhdr) + sizeof(struct ipv6hdr) + + sizeof(struct udphdr) + TEST_PAYLOAD_LEN] = {}; + struct ethhdr *eth = (void *)pkt; + struct ipv6hdr *ip6 = (void *)(eth + 1); + struct udphdr *udp = (void *)(ip6 + 1); + __u8 *payload = (void *)(udp + 1); + const __u8 tap_mac[ETH_ALEN] = { 0x02, 0, 0, 0, 0, 0x01 }; + int n; + + memcpy(eth->h_dest, tap_mac, ETH_ALEN); + eth->h_proto = htons(ETH_P_IPV6); + + ip6->version = 6; + ip6->hop_limit = 64; + ip6->nexthdr = IPPROTO_UDP; + ip6->payload_len = htons(sizeof(*udp) + TEST_PAYLOAD_LEN); + inet_pton(AF_INET6, "fd00::2", &ip6->saddr); + inet_pton(AF_INET6, "fd00:1::1", &ip6->daddr); + + udp->source = htons(42); + udp->dest = htons(42); + udp->len = htons(sizeof(*udp) + TEST_PAYLOAD_LEN); + /* UDP checksum is not validated on the forwarding path. */ + + memcpy(payload, test_payload, TEST_PAYLOAD_LEN); + + n = write(tap_fd, pkt, sizeof(pkt)); + if (!ASSERT_EQ(n, sizeof(pkt), "write frame")) + return -1; + + return 0; +} + static void dump_err_stream(const struct bpf_program *prog) { char buf[512]; @@ -518,3 +559,137 @@ void test_xdp_context_tuntap(void) test_xdp_meta__destroy(skel); } + +/* + * Test topology: + * + * tap0 fd00::1 + * RX: injected IPv6 UDP frame, XDP ingress sets metadata + * fwd: encap route prepends outer header(s) + * TX: TC egress validates metadata + * + * A routable IPv6 UDP frame is written into the tap fd, so it enters the RX + * path where XDP stores metadata. Routing then forwards it back out the same + * tap through an encapsulating route that prepends outer header(s). The TC + * egress program checks that the pushed header did not silently corrupt + * metadata. + */ +#define LWT_PIN_PATH "/sys/fs/bpf/xdp_context_lwt_xmit" + +enum lwt_encap_type { + LWT_ENCAP_BPF, + LWT_ENCAP_MPLS, + LWT_ENCAP_SEG6, + LWT_ENCAP_IOAM6, +}; + +static void test_lwt_encap(struct test_xdp_meta *skel, + enum lwt_encap_type type) +{ + LIBBPF_OPTS(bpf_tc_hook, tc_hook, .attach_point = BPF_TC_EGRESS); + LIBBPF_OPTS(bpf_tc_opts, tc_opts, .handle = 1, .priority = 1); + struct bpf_program *lwt_prog = NULL; + struct netns_obj *ns = NULL; + const char *encap; + bool pinned = false; + int tap_ifindex; + int tap_fd = -1; + int ret; + + skel->bss->test_pass = false; + + switch (type) { + case LWT_ENCAP_BPF: + encap = "encap bpf xmit pinned " LWT_PIN_PATH " via fd00::2"; + lwt_prog = skel->progs.dummy_lwt_xmit; + break; + case LWT_ENCAP_MPLS: + encap = "encap mpls 100 via inet6 fd00::2"; + break; + case LWT_ENCAP_SEG6: + encap = "encap seg6 mode encap segs fd00::2"; + break; + case LWT_ENCAP_IOAM6: + encap = "encap ioam6 mode encap tundst fd00::2 " + "trace prealloc type 0x800000 ns 0 size 4 via fd00::2"; + break; + default: + return; + } + + if (lwt_prog) { + unlink(LWT_PIN_PATH); + ret = bpf_program__pin(lwt_prog, LWT_PIN_PATH); + if (!ASSERT_OK(ret, "pin lwt prog")) + return; + pinned = true; + } + + ns = netns_new(LWT_NETNS, true); + if (!ASSERT_OK_PTR(ns, "netns_new")) + goto close; + + tap_fd = open_tuntap(TAP_NAME, true); + if (!ASSERT_GE(tap_fd, 0, "open_tuntap")) + goto close; + + SYS(close, "ip link set dev " TAP_NAME " address " RX_MAC); + SYS(close, "sysctl -wq net.ipv6.conf.all.forwarding=1"); + SYS(close, "ip addr add fd00::1/64 dev " TAP_NAME " nodad"); + SYS(close, "ip link set dev " TAP_NAME " up"); + SYS(close, "ip neigh add fd00::2 lladdr " TX_MAC " nud permanent dev " TAP_NAME); + SYS(close, "ip -6 route add fd00:1::/64 %s dev %s", encap, TAP_NAME); + + tap_ifindex = if_nametoindex(TAP_NAME); + if (!ASSERT_GE(tap_ifindex, 0, "if_nametoindex")) + goto close; + + ret = bpf_xdp_attach(tap_ifindex, bpf_program__fd(skel->progs.ing_xdp), + 0, NULL); + if (!ASSERT_GE(ret, 0, "bpf_xdp_attach")) + goto close; + + tc_hook.ifindex = tap_ifindex; + ret = bpf_tc_hook_create(&tc_hook); + if (!ASSERT_OK(ret, "bpf_tc_hook_create")) + goto close; + + tc_opts.prog_fd = bpf_program__fd(skel->progs.tc_is_meta_empty); + ret = bpf_tc_attach(&tc_hook, &tc_opts); + if (!ASSERT_OK(ret, "bpf_tc_attach")) + goto close; + + ret = write_test_packet_udp(tap_fd); + if (!ASSERT_OK(ret, "write_test_packet_udp")) + goto close; + + if (!ASSERT_TRUE(skel->bss->test_pass, "test_pass")) + dump_err_stream(skel->progs.tc_is_meta_empty); + +close: + if (tap_fd >= 0) + close(tap_fd); + netns_free(ns); + if (pinned) + unlink(LWT_PIN_PATH); +} + +void test_xdp_context_lwt_encap(void) +{ + struct test_xdp_meta *skel; + + skel = test_xdp_meta__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load skeleton")) + return; + + if (test__start_subtest("bpf_encap")) + test_lwt_encap(skel, LWT_ENCAP_BPF); + if (test__start_subtest("mpls_encap")) + test_lwt_encap(skel, LWT_ENCAP_MPLS); + if (test__start_subtest("seg6_encap")) + test_lwt_encap(skel, LWT_ENCAP_SEG6); + if (test__start_subtest("ioam6_encap")) + test_lwt_encap(skel, LWT_ENCAP_IOAM6); + + test_xdp_meta__destroy(skel); +} |