diff options
272 files changed, 18968 insertions, 3417 deletions
diff --git a/Documentation/bpf/bpf_devel_QA.rst b/Documentation/bpf/bpf_devel_QA.rst index 45bc5c5cd793..edf8107a7beb 100644 --- a/Documentation/bpf/bpf_devel_QA.rst +++ b/Documentation/bpf/bpf_devel_QA.rst @@ -479,7 +479,10 @@ for details. To maximize the number of tests passing, the .config of the kernel under test should match the config file fragment in -tools/testing/selftests/bpf as closely as possible. +tools/testing/selftests/bpf as closely as possible. If not possible, +however, you can set ``BPF_STRICT_BUILD=0`` when invoking ``make`` +to tolerate individual compilation failures and continue building +the remaining tests rather than treating each failure as fatal. Finally to ensure support for latest BPF Type Format features - discussed in Documentation/bpf/btf.rst - pahole version 1.16 diff --git a/Documentation/bpf/index.rst b/Documentation/bpf/index.rst index 0d5c6f659266..638a00d42bc2 100644 --- a/Documentation/bpf/index.rst +++ b/Documentation/bpf/index.rst @@ -28,6 +28,7 @@ that goes into great technical depth about the BPF Architecture. classic_vs_extended.rst bpf_iterators bpf_licensing + signing test_debug clang-notes linux-notes diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 4c814ff6061e..85f73e0bbd0f 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -250,6 +250,71 @@ Or:: ... } +2.3.7 __const_map and __map Annotations +--------------------------------------- + +These annotations are used for ``struct bpf_map *`` arguments and distinguish a +verifier-known map from an opaque one. + +``__const_map`` indicates a map must be known at the verification time, i.e. a +concrete map fd the BPF program references directly. + +An example is given below:: + + __bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__const_map, + unsigned int flags) + { + ... + } + +``__map`` indicates an opaque ``struct bpf_map *`` that may be resolved +at run time. The argument may take either a map fd or a ``PTR_TO_BTF_ID`` +``struct bpf_map`` pointer. + +An example is given below:: + + __bpf_kfunc void *bpf_arena_alloc_pages(void *p__map, ...) + { + ... + } + +2.3.8 __arena and __arena__nullable Annotations +----------------------------------------------- + +Both annotations indicate that the pointer argument points into the +calling program's arena. The JIT rebases the value at the call site so +the kfunc receives a directly dereferenceable kernel address, subject to +the access rules described in :ref:`BPF_kfunc_arena_access` (at most +``GUARD_SZ / 2``, 32 KiB, past the pointer in a single unchecked access). + +With ``__arena`` the rebase is unconditional and the argument is never +NULL: a value whose lower 32 bits are zero arrives as the arena base +address (arena offset 0). The kfunc must not check the argument for NULL. +With ``__arena__nullable`` such a value arrives as NULL instead and the +kfunc must check before dereferencing. + +An example is given below:: + + __bpf_kfunc int bpf_process_item(struct item *item__arena) + { + ... + } + +Calling such a kfunc requires the program to use an arena map and a JIT with +arena argument support (currently x86-64 and arm64); verification fails +otherwise. The program can pass any value without compromising the kernel. A +value that does not point into the arena is a program bug. + +The suffixes have the same meaning on the arguments of struct_ops stub +functions, with the conversion running in the opposite direction. The +kernel caller passes the kernel arena address and the trampoline converts +it while saving the arguments, so the callback receives an arena pointer +it can dereference directly. With ``__arena`` the kernel caller must not +pass NULL. With ``__arena__nullable`` a NULL kernel pointer arrives as NULL. +However, there is no obligation to prove to the verifier that such a pointer is +non-NULL before use, in-line with existing semantics of arena pointers used in +a program (or obtained from any other source). + .. _BPF_kfunc_nodef: 2.4 Using an existing kernel function @@ -273,22 +338,29 @@ flags on a set of kfuncs as follows:: BTF_KFUNCS_END(bpf_task_set) This set encodes the BTF ID of each kfunc listed above, and encodes the flags -along with it. Ofcourse, it is also allowed to specify no flags. +along with it. It is also allowed to specify no flags. kfunc definitions should also always be annotated with the ``__bpf_kfunc`` -macro. This prevents issues such as the compiler inlining the kfunc if it's a -static kernel function, or the function being elided in an LTO build as it's -not used in the rest of the kernel. Developers should not manually add -annotations to their kfunc to prevent these issues. If an annotation is -required to prevent such an issue with your kfunc, it is a bug and should be -added to the definition of the macro so that other kfuncs are similarly -protected. An example is given below:: +macro. This prevents issues such as the compiler inlining the kfunc, or the +function being elided in an LTO build as it's not used in the rest of the +kernel. Developers should not manually add annotations to their kfunc to prevent +these issues. If an annotation is required to prevent such an issue with your +kfunc, it is a bug and should be added to the definition of the macro so that +other kfuncs are similarly protected. An example is given below:: __bpf_kfunc struct task_struct *bpf_get_task_pid(s32 pid) { ... } +Note that kfuncs must not be declared ``static``. A kfunc can be called from a +BPF program ``*.c`` file outside the compilation unit that defines it, so its +externally visible name must remain available for BTF ID lookup. ``static`` +linkage allows the compiler to rename the function, which can break this +BTF-based kfunc resolution. Further note that sparse may warn that an otherwise +unreferenced kfunc should be static. Such warnings should be ignored for kfunc +definitions. + 2.5.1 KF_ACQUIRE flag --------------------- @@ -404,7 +476,7 @@ Example declaration: .. code-block:: c __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { ... } Example usage in BPF program: @@ -437,6 +509,13 @@ type. An example is shown below:: } late_initcall(init_subsystem); +At kernel build time the ``resolve_btfids`` tool finds all kfuncs declared with +``BTF_KFUNCS_START()`` and emits their BTF annotations into the kernel's BTF. +For each kfunc it emits a ``bpf_kfunc`` BTF decl tag, a ``bpf_fastcall`` decl +tag when the kfunc is flagged ``KF_FASTCALL``, and the ``address_space(1)`` type +attribute on the return value and/or arguments that use arena pointers (see +sections 2.3.8 and 2.8). + 2.7 Specifying no-cast aliases with ___init -------------------------------------------- @@ -480,6 +559,8 @@ In order to accommodate such requirements, the verifier will enforce strict PTR_TO_BTF_ID type matching if two types have the exact same name, with one being suffixed with ``___init``. +.. _BPF_kfunc_arena_access: + 2.8 Accessing arena memory through kfunc arguments -------------------------------------------------- diff --git a/Documentation/bpf/signing.rst b/Documentation/bpf/signing.rst new file mode 100644 index 000000000000..e73eaaebd8b1 --- /dev/null +++ b/Documentation/bpf/signing.rst @@ -0,0 +1,497 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============ +BPF signing +============ + +This document describes how BPF programs are cryptographically signed, how the +kernel verifies them at load time, and how Linux Security Modules (LSMs) - +including the BPF LSM - use the resulting verdict to enforce policy. It is +written for developers who want to produce signed BPF objects, understand what +the signature actually guarantees, or build a policy on top of it. + +Motivation +========== + +A signed BPF program lets the kernel establish that the bytecode being loaded +originates from a trusted producer and was not modified in transit. On its own +the kernel does not *require* signatures - an unsigned program loads exactly as +before - but it records a verdict (see `The verdict`_) that an LSM can gate on. +This is the building block for policies such as "only run BPF that was signed by +a key in the trusted keyring", as could in the future be enforced by an LSM +such as IPE. + +Signing is orthogonal to the existing permission model: it does not replace the +capability checks or the verifier. A signed load still requires the usual +privileges (``CAP_BPF`` and any program-type-specific capability, subject to +``kernel.unprivileged_bpf_disabled``), and the loader's instructions are still +checked by the verifier like any other program. A valid signature establishes +*origin and integrity*, not safety - it lets a policy trust where the bytecode +came from, it does not let a load skip any check it would otherwise face. + +The hard part is *what* gets signed. A naive scheme would sign a program's +instruction buffer at build time and verify that signature at +``BPF_PROG_LOAD``. That does not survive contact with real BPF objects, because +the bytes the kernel finally loads are not the bytes the developer built and +signed. Between the two, libbpf and the kernel rewrite the program: + +- **map file descriptors** are patched into ``ld_imm64`` instructions + (``BPF_PSEUDO_MAP_FD``), and a map's fd is assigned at load time, so it + differs on every run; +- **CO-RE relocations** rewrite field offsets, sizes and existence flags against + the *running* kernel's BTF, so the result differs from one kernel to the next; +- **kfunc and ksym references** are resolved to ids/addresses in the running + kernel; +- **global data** (``.rodata``/``.data``/``.bss``) is created and seeded as maps + at load. + +So a signature over the original instructions cannot match the relocated +instructions the verifier ends up checking, and the relocated form cannot be +produced ahead of time because it depends on the target kernel. There is no +fixed byte string that is both signable at build time and what the kernel +actually loads - which is why a program cannot simply be signed and loaded +directly. + +The trusted loader +================== + +The solution is to move that setup work *into* a small BPF program - the +**loader** - and sign the loader instead of the individual programs. libbpf's +``gen_loader`` machinery (``bpftool gen skeleton -L``, the "light skeleton") +emits a ``BPF_PROG_TYPE_SYSCALL`` program whose body performs the bpf() syscalls +that create maps, apply relocations, and load the real programs. The payload it +installs - the serialized programs, map descriptions, relocation data and +initial values - lives in a separate array map, the **metadata map** +(``__loader.map``). + +So the unit of trust is the loader, and the signing contract is:: + + Sig(I_loader || D_meta) + +where ``I_loader`` is the loader's instruction stream and ``D_meta`` is the +content of the metadata map. Verifying the loader's signature establishes that +both the loader *and* the payload it is about to install are authentic. The +loader is reproducible: ``gen_loader`` builds it from primitives so the same +object yields the same bytes on any build host. + +Why the loader is signable when the program is not +-------------------------------------------------- + +The loader sidesteps every rewrite listed above, because the bytes that are +signed are *relocation-invariant*: + +- The loader's own instructions are a fixed sequence of bpf() syscalls emitted + by ``gen_loader``; they carry no CO-RE relocations and resolve no ksyms, so + they are identical on every kernel. The metadata map is referenced by *index* + into ``fd_array`` (``BPF_PSEUDO_MAP_IDX_VALUE``), not by a baked-in file + descriptor, so even that reference does not change between build and load. + The loader instruction bytes the kernel verifies are exactly the bytes that + were signed. +- The metadata map is opaque, frozen data - the serialized target programs, + their relocation records, map descriptions and initial values. Its bytes are + identical at build time and at load time, so they are simply appended to the + instructions and covered by the same signature (there is no separate metadata + hash to compute or compare). + +All the host-specific rewriting - creating maps, patching their fds into the +target programs, applying CO-RE, resolving ksyms, seeding global data - still +happens, but it happens *inside the loader at runtime*, on the verified +metadata, **after** the kernel has verified the ``insns || metadata`` signature. +The kernel never has to verify the relocated target programs: it verifies the +loader and its inputs once, and trust transfers to whatever that now-trusted, +deterministic loader installs. The relocation step is moved from "before the +signature can be checked" to "after a trusted program runs" - which is exactly +what makes it signable. + +Because the metadata map is the loader's only untrusted input, two existing map +properties are reused to keep it trustworthy across the load: + +Exclusive maps + A map created with ``excl_prog_hash`` (see ``BPF_MAP_CREATE``) may only be + accessed by a program whose digest matches that hash. The verifier enforces + ``map->excl_prog_sha == prog->digest`` for every map a program uses, so the + metadata map is bound to exactly the signed loader and cannot be shared with + or mutated by another program. + +Frozen maps + The metadata map is frozen (``BPF_MAP_FREEZE``) before the loader is loaded. + Freezing blocks further userspace writes, so the bytes folded into the + signature cannot change before the loader runs. (Freezing does not make the + map read-only to the loader program itself, which still writes created file + descriptors back into the blob's scratch area.) + +Load-time verification +======================= + +Rather than have the loader check its own metadata from within BPF, the kernel +verifies it directly at ``BPF_PROG_LOAD``, with no new UAPI. The mechanism +reuses the existing ``fd_array``: + +#. Userspace creates the metadata map with ``excl_prog_hash`` set to the + loader's digest, populates it, and freezes it. +#. The loader is loaded with ``signature``/``signature_size``/``keyring_id`` + set, the metadata map referenced through ``fd_array``, and ``fd_array_cnt`` + set so the kernel knows the array's length. +#. Signature verification runs inside the verifier (``bpf_check()``), once it + has resolved the ``fd_array`` entries into the program's ``used_maps``. The + maps folded into the signature are therefore the very objects the program + binds - a single resolution of ``fd_array``, not a separate read, so the + verified bytes cannot be swapped for a different map after the check (no + time-of-check/time-of-use window). Each folded map must be exclusive (carry + ``excl_prog_sha``) and a plain array map (``BPF_MAP_TYPE_ARRAY``); only an + array map exposes its value buffer through ``map_direct_value_addr()`` as a + kernel address spanning ``value_size`` bytes. A map that is not exclusive, not + frozen, or not a plain array is rejected, with a verifier log message naming + the offending map. The kernel appends each map's frozen + contents to the instruction buffer and verifies the PKCS#7 signature over the + concatenation ``insns || metadata_0 || metadata_1 || ...`` in ``used_maps`` + order, before it rewrites the (signed) instructions. + +A signed program therefore takes one of exactly two shapes, both fully +supported: + +- **No bound maps** (``fd_array_cnt == 0``): there is nothing to append, so the + kernel verifies the signature over the instructions alone. A valid signature + yields ``BPF_SIG_VERIFIED`` and the program loads. This is the ordinary case + for a directly-loaded signed program with no separate payload; it is *not* + rejected for "missing" metadata, because it has none to cover. +- **Exclusive bound maps** (``fd_array_cnt > 0``): every entry is exclusive and + folded, so the signature covers ``insns || metadata``. + +There is no third shape: a non-exclusive map in a signed program's ``fd_array`` +is rejected rather than silently left out of the signature, so a signed loader +never binds a map its signature does not cover. + +The digest binding (``excl_prog_sha == prog->digest``) is enforced by the +verifier as usual; because that check runs while ``fd_array`` is resolved - +before the verifier would otherwise compute the tag - ``prog->digest`` is +computed up front in the verifier, over the unmodified (signature-covered) +instructions, for any signed load. + +Coverage is then enforced as the verifier resolves instructions, at the point +each object is bound rather than by a count taken afterwards. Once the signature +has been verified, binding any further map is refused: a map reached by a +directly-referenced fd, or a map swapped into an ``fd_array`` slot the loader +reads, is not among those already folded, so it is rejected the moment the +verifier tries to bind it. A BTF is refused outright for a signed program - a +ksym or a BTF fd in ``fd_array``, whether resolved up front or lazily for a +module kfunc, is rejected when it would be bound. Together with the fold rule +above this keeps the verdict binary: a signed program cannot use a map its +signature does not cover, and a different but equally digest-bound map cannot be +substituted at an ``fd_array`` slot. Non-exclusive maps are never folded, so a +signed program cannot use one at all. + +The verdict +=========== + +A program is either unsigned or fully verified - there is no intermediate +state. The outcome is recorded in ``prog->aux->sig.verdict``: + +.. code-block:: c + + enum bpf_sig_verdict { + BPF_SIG_UNSIGNED = 0, + BPF_SIG_VERIFIED, + }; + +``BPF_SIG_VERIFIED`` means the signature is valid and covers the instructions +*and* the frozen contents of every exclusive map the program uses: + +- For an ordinary, directly-loaded signed program the instructions are the whole + artifact and it uses no exclusive maps, so a valid instruction signature is + the complete verification. +- For a signed loader the metadata map is exclusive, so its contents are folded + in and the signature covers ``insns || metadata``. + +There is deliberately no "instructions verified but metadata not" verdict: a +signed loader that fails to cover its metadata is *rejected* (see above), not +recorded with a weaker verdict. ``BPF_SIG_VERIFIED`` therefore always means the +program and everything the signature is responsible for are authentic, which is +what a policy can rely on. + +Alongside the verdict the kernel records which keyring validated the signature; +see `Keyrings`_. + +Enforcement via LSMs +==================== + +Signing only *records* a verdict; an LSM turns it into policy. The verdict and +keyring fields live in ``struct bpf_prog_aux``, so a BPF LSM program can read +them directly (see Documentation/bpf/prog_lsm.rst for writing and attaching BPF +LSM programs); the same fields are equally available to in-tree LSMs. Two hooks +are useful at different points of the load: the dedicated +``security_bpf_prog_load()`` gates admission before the main verification work, +and the existing ``security_bpf_prog()`` observes a program that has fully +loaded. + +Admission: ``security_bpf_prog_load()`` +--------------------------------------- + +This hook gates admission **for every load**, from a single call site inside the +verifier (``bpf_check()``), before the main verification work. It runs after the +optional signature verification, so the verdict and keyring fields are final - the +hook can see whether, and how strongly, the program was signed, which keyring +validated it, the load ``attr``, the BPF token and whether the load came from the +kernel. For a signed load the verdict is ``BPF_SIG_VERIFIED`` here (the signature +has just been checked); for an unsigned load it is ``BPF_SIG_UNSIGNED``. + +This is the place for *coarse admission* that must also see unsigned and +not-yet-verified loads: require a signature at all, restrict the acceptable +keyring, restrict which token/credentials may load BPF, apply per-program-type +rules, or audit every load attempt that makes it past signature verification - +attempts failing the signature or the metadata binding abort before this hook +fires. It is the primary deny point. + +One subtlety: this hook runs *before* the verifier finishes its work, so +``BPF_SIG_VERIFIED`` *here* means only "validly signed" - not "loaded". Allowing +a load at this point lets it *proceed*; it does not guarantee the program will +load. A validly signed program can still be rejected afterwards on two +independent grounds: the verifier may reject it like any other program (unsafe +memory access, bad control flow, resource limits, ...), and the kernel separately +refuses - as the verifier resolves instructions and binds each object - any map +the signature does not cover or any BTF at all, regardless of what this hook +returned. Only after the program has fully loaded, at the next hook +(``security_bpf_prog()``), does ``BPF_SIG_VERIFIED`` carry its full meaning: +validly signed *and* fully verified. + +A more realistic admission policy than "is it signed at all": accept programs +signed by a system keyring, accept a user-keyring signature only if the +key/keyring it was verified against is on an explicit allowlist, and emit a +tamper-evident record of every decision so that even denied attempts are +auditable. (Illustrative - error checking elided.) + +.. code-block:: c + + /* Serials of user keys/keyrings we additionally trust. */ + struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, __s32); /* keyring_serial */ + __type(value, __u8); + __uint(max_entries, 64); + } trusted_user_keys SEC(".maps"); + + /* Audit stream consumed by a userspace logger. */ + struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1 << 16); + } audit SEC(".maps"); + + struct decision { __u32 prog_type, verdict, ktype; __s32 serial, ret; }; + + SEC("lsm/bpf_prog_load") + int BPF_PROG(admit, struct bpf_prog *prog, union bpf_attr *attr, + struct bpf_token *token, bool kernel) + { + __u32 verdict = prog->aux->sig.verdict; + __u32 ktype = prog->aux->sig.keyring_type; + __s32 serial = prog->aux->sig.keyring_serial; + struct decision *d; + int ret = 0; + + if (kernel) + return 0; /* trust in-kernel loads */ + + if (verdict != BPF_SIG_VERIFIED) + ret = -EPERM; /* must be validly signed */ + else if (ktype == BPF_SIG_KEYRING_USER && + !bpf_map_lookup_elem(&trusted_user_keys, &serial)) + ret = -EPERM; /* key/keyring not allowlisted */ + + d = bpf_ringbuf_reserve(&audit, sizeof(*d), 0); + if (d) { + d->prog_type = attr->prog_type; + d->verdict = verdict; + d->ktype = ktype; + d->serial = serial; + d->ret = ret; + bpf_ringbuf_submit(d, 0); /* record allow *and* deny */ + } + return ret; + } + +Observing a verified load: ``security_bpf_prog()`` +-------------------------------------------------- + +There is deliberately no separate "metadata attested" hook. The coverage check +above is enforced by the kernel unconditionally, so a signed loader that fails +to cover its metadata never loads and an LSM never has to re-establish that +fact. To *act on* a program that has successfully and fully loaded, use the +existing ``security_bpf_prog()`` hook (``lsm/bpf_prog``), which fires from +``bpf_prog_new_fd()`` - after the verifier, after the coverage check, and after +``bpf_prog_alloc_id()``. Relative to the admission hook this point is strictly +later and stronger: + +- the program has an id (``prog->aux->id``), so it can be recorded or correlated + with later events; +- ``verdict == BPF_SIG_VERIFIED`` *here* means **fully** verified - a program + that used a map the signature does not cover was already rejected, so it cannot + reach this point; +- it observes only programs that actually loaded; a failed load never mints an + fd, so it never reaches this hook. + +It takes only the ``prog`` and a non-zero return still aborts (the fd is not +handed out), so it can veto as well as observe. One wrinkle: it also fires on +other paths that mint a new program fd - notably ``bpf_prog_get_fd_by_id()`` - +not just on a fresh load. Because the program already has its id here, an LSM +can tell the two apart with a small hash map: the *first* time an id is seen is +the load; a later sighting of the same id is just another fd to a program that +already exists. + +To bound the map and let a reused id read as a fresh load, this can be paired +with ``security_bpf_prog_free()`` (``lsm/bpf_prog_free``), which deletes the +entry on teardown - keyed by the same ``prog`` pointer, since +``bpf_prog_free_id()`` has already cleared ``prog->aux->id`` to ``0`` by the time +that hook runs. (Illustrative - privileged LSM, error checking elided.) + +.. code-block:: c + + struct rec { __u32 id, ktype; __s32 serial; }; + + struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, __u64); /* struct bpf_prog * -- stable id */ + __type(value, struct rec); + __uint(max_entries, 4096); + } live SEC(".maps"); + + SEC("lsm/bpf_prog") /* fires after load and on every later fd */ + int BPF_PROG(observe, struct bpf_prog *prog) + { + __u64 key = (__u64)(unsigned long)prog; + struct rec r; + + if (prog->aux->sig.verdict != BPF_SIG_VERIFIED) + return 0; + if (bpf_map_lookup_elem(&live, &key)) + return 0; /* seen before: a later fd, not a load */ + + /* First sighting == this program just loaded; id is valid here. */ + r.id = prog->aux->id; + r.ktype = prog->aux->sig.keyring_type; + r.serial = prog->aux->sig.keyring_serial; + bpf_map_update_elem(&live, &key, &r, BPF_NOEXIST); + /* ... newly-loaded verified-program action, e.g. record r.id ... */ + return 0; + } + +Putting them together: to *require* verified BPF, deny at the admission hook +unless the verdict is ``BPF_SIG_VERIFIED`` (and, if desired, restrict the +keyring). The kernel then guarantees that any program which actually loads with +that verdict covered all of its exclusive maps, rejecting any that did not - so +a deny-by-default admission policy needs no second enforcement point. Use +``security_bpf_prog()`` to record or finally gate the verified programs once +they carry an id. The ``verdict``, ``keyring_type`` and ``keyring_serial`` fields +let a policy distinguish, for example, "verified and signed by a builtin key" +from "verified by a user key". A policy LSM such as IPE could consume the same +hooks to enforce system policy without writing any BPF, though none implements +this today. + +Keyrings +======== + +``keyring_id`` selects the trusted keyring the PKCS#7 signature is verified +against. The well-known ids ``0`` (builtin), ``VERIFY_USE_SECONDARY_KEYRING`` +and ``VERIFY_USE_PLATFORM_KEYRING`` select the corresponding system keyrings; +any other value is treated as the serial of a user/session key or keyring. +The keyring is looked up first, before the signature bytes are examined, so a +signature naming a non-existent keyring is rejected up front, and a failed +verification aborts the load - so a program that loads successfully with a +signature always has consistent keyring fields recorded. + +Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect: + +``keyring_type`` (``enum bpf_sig_keyring``) + Classified purely from ``keyring_id`` whenever the program is signed: + ``BPF_SIG_KEYRING_BUILTIN``, ``_SECONDARY``, ``_PLATFORM`` for the system + keyrings, or ``_USER`` for a user/session keyring. It is + ``BPF_SIG_KEYRING_NONE`` for an unsigned program. + +``keyring_serial`` (``s32``) + Set **only** on a successful verification, to the serial of the + **user/session key or keyring** that ``keyring_id`` resolved to - the + object the signature was verified against, not the individual asymmetric + key inside it that matched the signer. Passing + ``KEY_SPEC_SESSION_KEYRING``, for example, records the session keyring's + serial. The system keyrings are trusted as a whole and expose no serial + here, so the serial is ``0`` for builtin, secondary and platform + signatures, and ``0`` for unsigned programs. In other words, a non-zero + ``keyring_serial`` is exactly "verified against the user key/keyring with + this serial". + +.. list-table:: + :header-rows: 1 + + * - ``keyring_id`` + - ``keyring_type`` + - ``keyring_serial`` + * - (no signature) + - ``BPF_SIG_KEYRING_NONE`` + - ``0`` + * - ``0`` + - ``BPF_SIG_KEYRING_BUILTIN`` + - ``0`` + * - ``VERIFY_USE_SECONDARY_KEYRING`` + - ``BPF_SIG_KEYRING_SECONDARY`` + - ``0`` + * - ``VERIFY_USE_PLATFORM_KEYRING`` + - ``BPF_SIG_KEYRING_PLATFORM`` + - ``0`` + * - other (a user/session key serial) + - ``BPF_SIG_KEYRING_USER`` + - serial of the resolved key/keyring + +Producing a signed object +========================== + +``bpftool`` generates and signs a light skeleton in one step:: + + bpftool gen skeleton -L -S -k <private_key.pem> -i <certificate.x509> \ + obj.bpf.o > obj.lskel.h + +``-L`` selects the light-skeleton (``gen_loader``) backend and ``-S`` enables +signing; ``-k`` and ``-i`` supply the signing key and its X.509 certificate. +``bpftool`` signs ``insns || metadata`` - the exact bytes the kernel +reconstructs - and also computes ``excl_prog_hash`` as the digest of the loader +instructions so the metadata map can be bound to the loader. The signature and +hash are embedded in the generated header; the certificate is used only for +signing and is not included. Loading the skeleton performs the +create/populate/freeze/load sequence described above. + +At runtime the trusted public key must be present in the chosen keyring (for +example added to the session keyring, or built into the kernel's builtin trusted +keyring) for verification to succeed. + +UAPI reference +============== + +``BPF_PROG_LOAD`` (``union bpf_attr``): + +``signature``, ``signature_size`` + Pointer to and length of the PKCS#7 signature blob. + +``keyring_id`` + Trusted keyring selector (see `Keyrings`_). + +``fd_array``, ``fd_array_cnt`` + Array of map (and module BTF) file descriptors bound to the program. + ``fd_array_cnt`` must be set for the kernel to scan the array. When a + signature is present, a BTF entry is rejected outright, and every map must + be exclusive; its frozen contents are folded into the verified buffer, and + a non-exclusive entry is rejected. + +``BPF_MAP_CREATE`` (``union bpf_attr``): + +``excl_prog_hash``, ``excl_prog_hash_size`` + SHA-256 digest of the program permitted to access this (exclusive) map. This + binds the metadata map to the loader; it is not a hash of the map *content*. + The map content is not hashed separately at all - it is covered, as bytes, + by the program signature. + +Notes and limitations +====================== + +- The instructions plus folded metadata are verified as one ``bpf_dynptr``, + which bounds the combined size (currently ~16 MiB); very large objects can + exceed it. +- The metadata container is a single-element array map, accessed through + ``map_direct_value_addr``. diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst index 1ca8c5f73ad0..0aa232b117b5 100644 --- a/Documentation/process/changes.rst +++ b/Documentation/process/changes.rst @@ -147,11 +147,6 @@ Since Linux 5.2, if CONFIG_DEBUG_INFO_BTF is selected, the build system generates BTF (BPF Type Format) from DWARF in vmlinux, a bit later from kernel modules as well. This requires pahole v1.22 or later. -Since Linux 7.0, kfuncs annotated with KF_IMPLICIT_ARGS require pahole v1.26 -or later. Without it, such kfuncs will have incorrect BTF prototypes in -vmlinux, causing BPF programs to fail to load with a "func_proto incompatible -with vmlinux" error. Many sched_ext kfuncs are affected. - It is found in the 'dwarves' or 'pahole' distro packages or from https://fedorapeople.org/~acme/dwarves/. diff --git a/MAINTAINERS b/MAINTAINERS index 040df7bfb24f..f5475597568e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4905,6 +4905,7 @@ R: Song Liu <song@kernel.org> R: Yonghong Song <yonghong.song@linux.dev> R: Jiri Olsa <jolsa@kernel.org> R: Emil Tsalapatis <emil@etsalapatis.com> +R: Ihor Solodrai <ihor.solodrai@linux.dev> L: bpf@vger.kernel.org S: Supported W: https://bpf.io/ @@ -4965,6 +4966,7 @@ F: net/unix/unix_bpf.c BPF [LIBRARY] (libbpf) M: Andrii Nakryiko <andrii@kernel.org> M: Eduard Zingerman <eddyz87@gmail.com> +R: Ihor Solodrai <ihor.solodrai@linux.dev> L: bpf@vger.kernel.org S: Maintained F: tools/lib/bpf/ @@ -5017,7 +5019,7 @@ F: kernel/bpf/ringbuf.c BPF [SECURITY & LSM] (Security Audit and Enforcement using BPF) M: KP Singh <kpsingh@kernel.org> -M: Matt Bobrowski <mattbobrowski@google.com> +M: Matt Bobrowski <matt@bobrowski.net> L: bpf@vger.kernel.org S: Maintained F: Documentation/bpf/prog_lsm.rst @@ -5030,6 +5032,7 @@ F: security/bpf/ BPF [SELFTESTS] (Test Runners & Infrastructure) M: Andrii Nakryiko <andrii@kernel.org> M: Eduard Zingerman <eddyz87@gmail.com> +R: Ihor Solodrai <ihor.solodrai@linux.dev> L: bpf@vger.kernel.org S: Maintained F: tools/testing/selftests/bpf/ @@ -5044,6 +5047,7 @@ F: tools/bpf/bpftool/ BPF [TRACING] M: Song Liu <song@kernel.org> R: Jiri Olsa <jolsa@kernel.org> +R: Ihor Solodrai <ihor.solodrai@linux.dev> L: bpf@vger.kernel.org S: Maintained F: kernel/bpf/stackmap.c diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index cc0702fa64a7..1ce75a442638 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -205,6 +205,18 @@ enum aarch64_insn_adsb_type { AARCH64_INSN_ADSB_SUB_SETFLAGS }; +/* option field of add/sub (extended register) */ +enum aarch64_insn_extend_type { + AARCH64_INSN_EXTEND_UXTB, + AARCH64_INSN_EXTEND_UXTH, + AARCH64_INSN_EXTEND_UXTW, + AARCH64_INSN_EXTEND_UXTX, + AARCH64_INSN_EXTEND_SXTB, + AARCH64_INSN_EXTEND_SXTH, + AARCH64_INSN_EXTEND_SXTW, + AARCH64_INSN_EXTEND_SXTX, +}; + enum aarch64_insn_movewide_type { AARCH64_INSN_MOVEWIDE_ZERO, AARCH64_INSN_MOVEWIDE_KEEP, @@ -378,6 +390,10 @@ __AARCH64_INSN_FUNCS(add, 0x7F200000, 0x0B000000) __AARCH64_INSN_FUNCS(adds, 0x7F200000, 0x2B000000) __AARCH64_INSN_FUNCS(sub, 0x7F200000, 0x4B000000) __AARCH64_INSN_FUNCS(subs, 0x7F200000, 0x6B000000) +__AARCH64_INSN_FUNCS(add_ext, 0x7FE00000, 0x0B200000) +__AARCH64_INSN_FUNCS(adds_ext, 0x7FE00000, 0x2B200000) +__AARCH64_INSN_FUNCS(sub_ext, 0x7FE00000, 0x4B200000) +__AARCH64_INSN_FUNCS(subs_ext, 0x7FE00000, 0x6B200000) __AARCH64_INSN_FUNCS(madd, 0x7FE08000, 0x1B000000) __AARCH64_INSN_FUNCS(msub, 0x7FE08000, 0x1B008000) __AARCH64_INSN_FUNCS(udiv, 0x7FE0FC00, 0x1AC00800) @@ -637,6 +653,13 @@ u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst, int shift, enum aarch64_insn_variant variant, enum aarch64_insn_adsb_type type); +u32 aarch64_insn_gen_add_sub_extended_reg(enum aarch64_insn_register dst, + enum aarch64_insn_register src, + enum aarch64_insn_register reg, + enum aarch64_insn_extend_type extend, + int shift, + enum aarch64_insn_variant variant, + enum aarch64_insn_adsb_type type); u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst, enum aarch64_insn_register src, enum aarch64_insn_variant variant, diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c index 37ce75f7f1f0..e70ac0238515 100644 --- a/arch/arm64/lib/insn.c +++ b/arch/arm64/lib/insn.c @@ -986,6 +986,66 @@ u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst, return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift); } +/* + * Unlike the shifted-register form, register 31 is not XZR everywhere here: + * it encodes SP for @src, and for @dst too unless @type sets the flags. Only + * @reg keeps the XZR meaning. + */ +u32 aarch64_insn_gen_add_sub_extended_reg(enum aarch64_insn_register dst, + enum aarch64_insn_register src, + enum aarch64_insn_register reg, + enum aarch64_insn_extend_type extend, + int shift, + enum aarch64_insn_variant variant, + enum aarch64_insn_adsb_type type) +{ + u32 insn; + + switch (type) { + case AARCH64_INSN_ADSB_ADD: + insn = aarch64_insn_get_add_ext_value(); + break; + case AARCH64_INSN_ADSB_SUB: + insn = aarch64_insn_get_sub_ext_value(); + break; + case AARCH64_INSN_ADSB_ADD_SETFLAGS: + insn = aarch64_insn_get_adds_ext_value(); + break; + case AARCH64_INSN_ADSB_SUB_SETFLAGS: + insn = aarch64_insn_get_subs_ext_value(); + break; + default: + pr_err("%s: unknown add/sub encoding %d\n", __func__, type); + return AARCH64_BREAK_FAULT; + } + + switch (variant) { + case AARCH64_INSN_VARIANT_32BIT: + break; + case AARCH64_INSN_VARIANT_64BIT: + insn |= AARCH64_INSN_SF_BIT; + break; + default: + pr_err("%s: unknown variant encoding %d\n", __func__, variant); + return AARCH64_BREAK_FAULT; + } + + if (shift < 0 || shift > 4) { + pr_err("%s: invalid shift encoding %d\n", __func__, shift); + return AARCH64_BREAK_FAULT; + } + + insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); + + insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); + + insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); + + /* option in bits [15:13] and imm3 in [12:10] together fill IMM_6 */ + return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, + (extend << 3) | shift); +} + u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst, enum aarch64_insn_register src, enum aarch64_insn_variant variant, diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h index d13de4222cfb..b2fe6e6dcf44 100644 --- a/arch/arm64/net/bpf_jit.h +++ b/arch/arm64/net/bpf_jit.h @@ -243,6 +243,17 @@ /* Rn - Rm; set condition flags */ #define A64_CMP(sf, Rn, Rm) A64_SUBS(sf, A64_ZR, Rn, Rm) +/* Add/subtract (extended register) */ +#define A64_ADDSUB_EREG(sf, Rd, Rn, Rm, ext, shift, type) \ + aarch64_insn_gen_add_sub_extended_reg(Rd, Rn, Rm, \ + AARCH64_INSN_EXTEND_##ext, shift, A64_VARIANT(sf), \ + AARCH64_INSN_ADSB_##type) +/* Rd = Rn + (EXT(Rm) << shift) */ +#define A64_ADD_EXT(sf, Rd, Rn, Rm, ext, shift) \ + A64_ADDSUB_EREG(sf, Rd, Rn, Rm, ext, shift, ADD) +/* Rd = Rn + (u32)Rm */ +#define A64_ADD_UXTW(Rd, Rn, Rm) A64_ADD_EXT(1, Rd, Rn, Rm, UXTW, 0) + /* Data-processing (1 source) */ #define A64_DATA1(sf, Rd, Rn, type) aarch64_insn_gen_data1(Rd, Rn, \ A64_VARIANT(sf), AARCH64_INSN_DATA1_##type) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index b0075ece4a6e..c18e005a41db 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1082,23 +1082,27 @@ static void build_epilogue(struct jit_ctx *ctx, bool was_classic) * * Bit layout of `fixup` (32-bit): * - * +-----------+--------+-----------+-----------+----------+ - * | 31-27 | 26-22 | 21 | 20-16 | 15-0 | - * | | | | | | - * | FIXUP_REG | Unused | ARENA_ACC | ARENA_REG | OFFSET | - * +-----------+--------+-----------+-----------+----------+ + * +-----------+--------+-------------+-----------+-----------+----------+ + * | 31-27 | 26-23 | 22 | 21 | 20-16 | 15-0 | + * | | | | | | | + * | FIXUP_REG | Unused | ARENA_WRITE | ARENA_ACC | ARENA_REG | OFFSET | + * +-----------+--------+-------------+-----------+-----------+----------+ * * - OFFSET (16 bits): Offset used to compute address for Load/Store instruction. * - ARENA_REG (5 bits): Register that is used to calculate the address for load/store when * accessing the arena region. * - ARENA_ACCESS (1 bit): This bit is set when the faulting instruction accessed the arena region. + * - ARENA_WRITE (1 bit): This bit is set when the faulting instruction wrote to the arena region. + * It is independent of FIXUP_REG, since a read-modify-write both writes to + * memory and reads the old value into a register. * - FIXUP_REG (5 bits): Destination register for the load instruction (cleared on fault) or set to - * DONT_CLEAR if it is a store instruction. + * DONT_CLEAR if the instruction does not read into a register. */ #define BPF_FIXUP_OFFSET_MASK GENMASK(15, 0) #define BPF_FIXUP_ARENA_REG_MASK GENMASK(20, 16) #define BPF_ARENA_ACCESS BIT(21) +#define BPF_ARENA_WRITE BIT(22) #define BPF_FIXUP_REG_MASK GENMASK(31, 27) #define DONT_CLEAR 5 /* Unused ARM64 register from BPF's POV */ @@ -1109,7 +1113,7 @@ bool ex_handler_bpf(const struct exception_table_entry *ex, s16 off = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); int arena_reg = FIELD_GET(BPF_FIXUP_ARENA_REG_MASK, ex->fixup); bool is_arena = !!(ex->fixup & BPF_ARENA_ACCESS); - bool is_write = (dst_reg == DONT_CLEAR); + bool is_write = !!(ex->fixup & BPF_ARENA_WRITE); unsigned long addr; if (is_arena) { @@ -1132,7 +1136,7 @@ static int add_exception_handler(const struct bpf_insn *insn, { off_t ins_offset; s16 off = insn->off; - bool is_arena; + bool is_arena, is_write; int arena_reg; unsigned long pc; struct exception_table_entry *ex; @@ -1178,13 +1182,21 @@ static int add_exception_handler(const struct bpf_insn *insn, ex->insn = ins_offset; - if (BPF_CLASS(insn->code) != BPF_LDX) - dst_reg = DONT_CLEAR; + /* + * A load-acquire is of BPF_STX class, but reads from src_reg into + * dst_reg like a BPF_LDX does, hence it must not be treated as a store + * here. A read-modify-write carrying BPF_FETCH is reported as a write + * even though it does have a register to clear, see the callers. + */ + is_write = BPF_CLASS(insn->code) != BPF_LDX && + !bpf_atomic_is_load_acq(insn); ex->fixup = FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); if (is_arena) { ex->fixup |= BPF_ARENA_ACCESS; + if (is_write) + ex->fixup |= BPF_ARENA_WRITE; /* * insn->src_reg/dst_reg holds the address in the arena region with upper 32-bits * being zero because of a preceding addr_space_cast(r<n>, 0x0, 0x1) instruction. @@ -1193,7 +1205,7 @@ static int add_exception_handler(const struct bpf_insn *insn, * memory access. Pass the reg holding the unmodified 32-bit address to * ex_handler_bpf. */ - if (BPF_CLASS(insn->code) == BPF_LDX) + if (BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn)) arena_reg = bpf2a64[insn->src_reg]; else arena_reg = bpf2a64[insn->dst_reg]; @@ -1244,6 +1256,43 @@ static void emit_stack_arg_store_imm(s32 imm, s16 bpf_off, const u8 tmp, struct } } +/* + * Rebase the __arena args of a kfunc call to arena kernel addresses, + * xN = kern_vm_start + (u32)xN, with the arena base register holding + * kern_vm_start. A nullable arg preserves NULL by skipping the add, tested + * on the truncated value as arena NULL is offset 0. + */ +static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *insn) +{ + const u8 arena_vm_base = bpf2a64[ARENA_VM_START]; + const struct btf_func_model *fm; + int i; + + fm = bpf_jit_find_kfunc_model(ctx->prog, insn); + if (!fm) + return -EINVAL; + + for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { + const u8 reg = bpf2a64[BPF_REG_1 + i]; + u8 flags = fm->arg_flags[i]; + + if (!(flags & BTF_FMODEL_ARENA_ARG)) + continue; + if (WARN_ON_ONCE(!ctx->arena_vm_start)) + return -EINVAL; + + if (flags & BTF_FMODEL_NULLABLE_ARG) { + /* 32-bit mov clears the upper 32 bits */ + emit(A64_MOV(0, reg, reg), ctx); + /* skip the add so that NULL stays NULL */ + emit(A64_CBZ(0, reg, 2), ctx); + } + emit(A64_ADD_UXTW(reg, arena_vm_base, reg), ctx); + } + + return 0; +} + /* JITs an eBPF instruction. * Returns: * 0 - successfully JITed an 8-byte eBPF instruction. @@ -1284,12 +1333,25 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn case BPF_ALU | BPF_MOV | BPF_X: case BPF_ALU64 | BPF_MOV | BPF_X: if (insn_is_cast_user(insn)) { - emit(A64_MOV(0, tmp, src), ctx); // 32-bit mov clears the upper 32 bits - emit_a64_mov_i(0, dst, ctx->user_vm_start >> 32, ctx); - emit(A64_LSL(1, dst, dst, 32), ctx); - emit(A64_CBZ(1, tmp, 2), ctx); - emit(A64_ORR(1, tmp, dst, tmp), ctx); - emit(A64_MOV(1, dst, tmp), ctx); + u32 upper = ctx->user_vm_start >> 32; + u16 upper_low = upper & 0xffff; + u16 upper_high = upper >> 16; + int nr_movk = !!upper_low + !!upper_high; + + /* + * Build the user address: the low 32 bits are the arena + * offset, the upper 32 bits come from user_vm_start. A + * zero offset must stay NULL, so branch over the MOVKs + * when it is zero. + */ + emit(A64_MOV(0, dst, src), ctx); /* 32-bit mov clears the upper 32 bits */ + if (nr_movk) { + emit(A64_CBZ(0, dst, nr_movk + 1), ctx); + if (upper_low) + emit(A64_MOVK(1, dst, upper_low, 32), ctx); + if (upper_high) + emit(A64_MOVK(1, dst, upper_high, 48), ctx); + } break; } else if (insn_is_mov_percpu_addr(insn)) { if (dst != src) @@ -1653,6 +1715,11 @@ emit_cond_jmp: &func_addr, &func_addr_fixed); if (ret < 0) return ret; + if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { + ret = emit_kfunc_arena_args(ctx, insn); + if (ret < 0) + return ret; + } emit_call(func_addr, ctx); /* * Call to arch_bpf_timed_may_goto() is emitted by the @@ -1871,7 +1938,7 @@ emit_cond_jmp: break; } - ret = add_exception_handler(insn, ctx, dst); + ret = add_exception_handler(insn, ctx, DONT_CLEAR); if (ret) return ret; break; @@ -1938,7 +2005,7 @@ emit_cond_jmp: break; } - ret = add_exception_handler(insn, ctx, dst); + ret = add_exception_handler(insn, ctx, DONT_CLEAR); if (ret) return ret; break; @@ -1961,7 +2028,16 @@ emit_cond_jmp: return ret; if (BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { - ret = add_exception_handler(insn, ctx, dst); + /* + * A load-acquire reads into dst_reg, and a read-modify-write + * carrying BPF_FETCH reads the old value into src_reg, or into + * r0 for a BPF_CMPXCHG. Clear that register on fault, the + * remaining atomics have no destination register. + */ + int load_reg = bpf_atomic_load_reg(insn); + + ret = add_exception_handler(insn, ctx, load_reg < 0 ? + DONT_CLEAR : bpf2a64[load_reg]); if (ret) return ret; } @@ -2317,6 +2393,11 @@ bool bpf_jit_supports_stack_args(void) return true; } +bool bpf_jit_supports_arena_args(void) +{ + return true; +} + void *bpf_arch_text_copy(void *dst, void *src, size_t len) { if (!aarch64_insn_copy(dst, src, len)) @@ -2490,34 +2571,76 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes) } } +/* + * Convert an arena kernel address into the arena pointer form on its way into + * the BPF ctx, dst = (u32)(src - kern_vm_start), with @base_lo holding the low + * 32 bits of kern_vm_start. A nullable arg preserves NULL, tested on the full + * 64-bit kernel pointer. The 32-bit subtraction both truncates and clears the + * upper half, so the stored value satisfies the JIT invariant for arena + * pointer registers. + */ +static void emit_arena_arg_conv(struct jit_ctx *ctx, u8 dst, u8 src, bool nullable, u8 base_lo) +{ + if (nullable) { + if (dst != src) + emit(A64_MOV(1, dst, src), ctx); + /* skip the subtraction so that NULL stays NULL */ + emit(A64_CBZ(1, dst, 2), ctx); + src = dst; + } + emit(A64_SUB(0, dst, src, base_lo), ctx); +} + static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, - const struct btf_func_model *m, - const struct arg_aux *a, - bool for_call_origin) + const struct btf_func_model *m, const struct arg_aux *a, + bool for_call_origin, bool is_struct_ops, u64 arena_base) { - int i; - int reg; - int doff; - int soff; - int slots; u8 tmp = bpf2a64[TMP_REG_1]; + u8 base_lo = bpf2a64[TMP_REG_2]; + int i, reg, doff, soff, slots; + + /* only the low 32 bits of the base take part in the subtraction */ + if (arena_base) + emit_a64_mov_i(0, base_lo, (s32)(u32)arena_base, ctx); /* store arguments to the stack for the bpf program, or restore * arguments from stack for the original function */ - for (reg = 0; reg < a->regs_for_args; reg++) { - emit(for_call_origin ? - A64_LDR64I(reg, A64_SP, bargs_off) : - A64_STR64I(reg, A64_SP, bargs_off), - ctx); - bargs_off += 8; + for (i = 0, reg = 0; i < a->args_in_regs; i++) { + bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); + bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; + + slots = (m->arg_size[i] + 7) / 8; + while (slots-- > 0) { + if (for_call_origin) { + emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx); + } else if (arena_arg) { + emit_arena_arg_conv(ctx, tmp, reg, nullable, base_lo); + emit(A64_STR64I(tmp, A64_SP, bargs_off), ctx); + } else { + emit(A64_STR64I(reg, A64_SP, bargs_off), ctx); + } + reg++; + bargs_off += 8; + } } - soff = 32; /* on stack arguments start from FP + 32 */ + /* + * On-stack arguments start above the frame(s) pushed by the trampoline + * prologue. Entered through the fentry call from a traced function, the + * prologue saves both the parent (FP/x9) and the traced function + * (FP/LR) frames, so the arguments start at FP + 32. A struct_ops + * callback is called indirectly and only the FP/LR frame is saved, so + * they start at FP + 16. + */ + soff = is_struct_ops ? 16 : 32; doff = (for_call_origin ? oargs_off : bargs_off); /* save on stack arguments */ for (i = a->args_in_regs; i < m->nr_args; i++) { + bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); + bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; + slots = (m->arg_size[i] + 7) / 8; /* verifier ensures arg_size <= 16, so slots equals 1 or 2 */ while (slots-- > 0) { @@ -2527,6 +2650,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, */ if (slots == 0 && !for_call_origin) clear_garbage(ctx, tmp, m->arg_size[i] % 8); + /* + * No guard on for_call_origin here: only the indirect + * trampoline is given a base, and it never calls the + * original function, so arguments are never converted + * on their way back out to it. See the WARN_ON_ONCE() + * in prepare_trampoline(). + */ + if (arena_arg) + emit_arena_arg_conv(ctx, tmp, tmp, nullable, base_lo); emit(A64_STR64I(tmp, A64_SP, doff), ctx); soff += 8; doff += 8; @@ -2544,12 +2676,6 @@ static void restore_args(struct jit_ctx *ctx, int bargs_off, int nregs) } } -static bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes) -{ - return fentry_nodes->nr_nodes == 1 && - fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS; -} - static void store_func_meta(struct jit_ctx *ctx, u64 func_meta, int func_meta_off) { emit_a64_mov_i64(A64_R(10), func_meta, ctx); @@ -2592,8 +2718,21 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, bool is_struct_ops = is_struct_ops_tramp(fentry); int cookie_off, cookie_cnt, cookie_bargs_off; int fsession_cnt = bpf_fsession_cnt(tnodes); + u64 arena_base; u64 func_meta; + /* + * F_INDIRECT is only compatible with F_RET_FENTRY_RET, it is explicitly + * incompatible with F_CALL_ORIG | F_SKIP_FRAME | F_IP_ARG because + * @func_addr. Arena conversion relies on this: bpf_tramp_arena_base() + * only returns a base for the indirect trampoline, which therefore + * never calls the original function with converted arguments. + */ + WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && + (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); + + arena_base = bpf_tramp_arena_base(m, tnodes, flags); + /* trampoline stack layout: * [ parent ip ] * [ FP ] @@ -2709,7 +2848,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, store_func_meta(ctx, func_meta, func_meta_off); /* save args for bpf */ - save_args(ctx, bargs_off, oargs_off, m, a, false); + save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops, arena_base); /* save callee saved registers */ emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx); @@ -2757,8 +2896,8 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, } if (flags & BPF_TRAMP_F_CALL_ORIG) { - /* save args for original func */ - save_args(ctx, bargs_off, oargs_off, m, a, true); + /* the original func takes kernel addresses, never converted ones */ + save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops, 0); /* call original func */ emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx); emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx); diff --git a/arch/riscv/net/Makefile b/arch/riscv/net/Makefile index 9a1e5f0a94e5..6458d4d51990 100644 --- a/arch/riscv/net/Makefile +++ b/arch/riscv/net/Makefile @@ -3,7 +3,7 @@ obj-$(CONFIG_BPF_JIT) += bpf_jit_core.o ifeq ($(CONFIG_ARCH_RV64I),y) - obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o + obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o bpf_timed_may_goto.o else obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o endif diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index da0271790244..419b9d795f2a 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -81,6 +81,7 @@ struct rv_jit_context { int ex_jmp_off; unsigned long flags; int stack_size; + int tcc_offset; u64 arena_vm_start; u64 user_vm_start; }; diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index 592dd86fbf81..a9e0bd5cc81d 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -509,12 +509,15 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, } static void emit_alu_r32(const s8 *dst, const s8 *src, - struct rv_jit_context *ctx, const u8 op) + struct rv_jit_context *ctx, + const struct bpf_insn *insn) { const s8 *tmp1 = bpf2rv32[TMP_REG_1]; const s8 *tmp2 = bpf2rv32[TMP_REG_2]; const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); const s8 *rs = bpf_get_reg32(src, tmp2, ctx); + u8 op = BPF_OP(insn->code); + bool is_signed = insn->off == 1; switch (op) { case BPF_MOV: @@ -539,10 +542,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src, emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_DIV: - emit(rv_divu(lo(rd), lo(rd), lo(rs)), ctx); + emit(is_signed ? rv_div(lo(rd), lo(rd), lo(rs)) : + rv_divu(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_MOD: - emit(rv_remu(lo(rd), lo(rd), lo(rs)), ctx); + emit(is_signed ? rv_rem(lo(rd), lo(rd), lo(rs)) : + rv_remu(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_LSH: emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx); @@ -869,14 +874,58 @@ static int emit_load_r64(const s8 *dst, const s8 *src, s16 off, return 0; } -static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, - struct rv_jit_context *ctx, const u8 size, - const u8 mode) +static int emit_bpf_atomic(s8 dst, const s8 *src, const s8 *rs, + struct rv_jit_context *ctx, + const struct bpf_insn *insn) +{ + s32 imm = insn->imm; + bool is_fetch = (imm & BPF_FETCH) || (imm == BPF_XCHG); + s8 fetch_reg = is_fetch ? lo(rs) : RV_REG_ZERO; + int aq = is_fetch ? 1 : 0; + int rl = is_fetch ? 1 : 0; + + switch (imm) { + case BPF_ADD: + case BPF_ADD | BPF_FETCH: + emit(rv_amoadd_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_AND: + case BPF_AND | BPF_FETCH: + emit(rv_amoand_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_OR: + case BPF_OR | BPF_FETCH: + emit(rv_amoor_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_XOR: + case BPF_XOR | BPF_FETCH: + emit(rv_amoxor_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_XCHG: + emit(rv_amoswap_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + default: + return -1; + } + + if (is_fetch) { + emit(rv_addi(hi(rs), RV_REG_ZERO, 0), ctx); + bpf_put_reg64(src, rs, ctx); + } + return 0; +} + +static int emit_store_r64(const s8 *dst, const s8 *src, + struct rv_jit_context *ctx, + const struct bpf_insn *insn) { const s8 *tmp1 = bpf2rv32[TMP_REG_1]; const s8 *tmp2 = bpf2rv32[TMP_REG_2]; const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + u8 size = BPF_SIZE(insn->code); + u8 mode = BPF_MODE(insn->code); + s16 off = insn->off; if (mode == BPF_ATOMIC && size != BPF_W) return -1; @@ -896,9 +945,9 @@ static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, case BPF_MEM: emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx); break; - case BPF_ATOMIC: /* Only BPF_ADD supported */ - emit(rv_amoadd_w(RV_REG_ZERO, lo(rs), RV_REG_T0, 0, 0), - ctx); + case BPF_ATOMIC: + if (emit_bpf_atomic(RV_REG_T0, src, rs, ctx, insn)) + return -1; break; } break; @@ -967,6 +1016,24 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, switch (code) { case BPF_ALU64 | BPF_MOV | BPF_X: + if (insn->off != 0) { + const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); + const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + + if (insn->off == 8) { + emit(rv_slli(lo(rd), lo(rs), 24), ctx); + emit(rv_srai(lo(rd), lo(rd), 24), ctx); + } else if (insn->off == 16) { + emit(rv_slli(lo(rd), lo(rs), 16), ctx); + emit(rv_srai(lo(rd), lo(rd), 16), ctx); + } else { + emit(rv_addi(lo(rd), lo(rs), 0), ctx); + } + emit(rv_srai(hi(rd), lo(rd), 31), ctx); + bpf_put_reg64(dst, rd, ctx); + break; + } + fallthrough; case BPF_ALU64 | BPF_ADD | BPF_X: case BPF_ALU64 | BPF_ADD | BPF_K: @@ -1017,6 +1084,20 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_zext64(dst, ctx); break; } + if (insn->off != 0) { + const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); + const s8 *rs = bpf_get_reg32(src, tmp2, ctx); + + if (insn->off == 8) { + emit(rv_slli(lo(rd), lo(rs), 24), ctx); + emit(rv_srai(lo(rd), lo(rd), 24), ctx); + } else if (insn->off == 16) { + emit(rv_slli(lo(rd), lo(rs), 16), ctx); + emit(rv_srai(lo(rd), lo(rd), 16), ctx); + } + bpf_put_reg32(dst, rd, ctx); + break; + } fallthrough; case BPF_ALU | BPF_ADD | BPF_X: @@ -1041,7 +1122,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; } - emit_alu_r32(dst, src, ctx, BPF_OP(code)); + emit_alu_r32(dst, src, ctx, insn); break; case BPF_ALU | BPF_MOV | BPF_K: @@ -1065,7 +1146,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, * src is ignored---choose tmp2 as a dummy register since it * is not on the stack. */ - emit_alu_r32(dst, tmp2, ctx, BPF_OP(code)); + emit_alu_r32(dst, tmp2, ctx, insn); break; case BPF_ALU | BPF_END | BPF_FROM_LE: @@ -1266,21 +1347,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, src = tmp2; } - if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), - BPF_MODE(code))) + if (emit_store_r64(dst, src, ctx, insn)) return -1; break; case BPF_STX | BPF_ATOMIC | BPF_W: - if (insn->imm != BPF_ADD) { + if (insn->imm == BPF_CMPXCHG) { pr_info_once( "bpf-jit: not supported: atomic operation %02x ***\n", insn->imm); return -EFAULT; } - if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), - BPF_MODE(code))) + if (emit_store_r64(dst, src, ctx, insn)) return -1; break; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index f9d5347ba966..74efe4b138d2 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -18,12 +18,12 @@ #define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) -#define RV_KCFI_NINSNS (IS_ENABLED(CONFIG_CFI) ? 1 : 0) /* imm that allows emit_imm to emit max count insns */ #define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF +/* fentry and TCC init insns will be skipped on tailcall */ +#define RV_TAILCALL_OFFSET ((RV_FENTRY_NINSNS + 1) * 4) #define RV_REG_TCC RV_REG_A6 -#define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ static const int regmap[] = { @@ -57,14 +57,12 @@ static const int pt_regmap[] = { }; enum { - RV_CTX_F_SEEN_TAIL_CALL = 0, RV_CTX_F_SEEN_CALL = RV_REG_RA, RV_CTX_F_SEEN_S1 = RV_REG_S1, RV_CTX_F_SEEN_S2 = RV_REG_S2, RV_CTX_F_SEEN_S3 = RV_REG_S3, RV_CTX_F_SEEN_S4 = RV_REG_S4, RV_CTX_F_SEEN_S5 = RV_REG_S5, - RV_CTX_F_SEEN_S6 = RV_REG_S6, }; static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) @@ -77,7 +75,6 @@ static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) case RV_CTX_F_SEEN_S3: case RV_CTX_F_SEEN_S4: case RV_CTX_F_SEEN_S5: - case RV_CTX_F_SEEN_S6: __set_bit(reg, &ctx->flags); } return reg; @@ -92,7 +89,6 @@ static bool seen_reg(int reg, struct rv_jit_context *ctx) case RV_CTX_F_SEEN_S3: case RV_CTX_F_SEEN_S4: case RV_CTX_F_SEEN_S5: - case RV_CTX_F_SEEN_S6: return test_bit(reg, &ctx->flags); } return false; @@ -108,32 +104,6 @@ static void mark_call(struct rv_jit_context *ctx) __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); } -static bool seen_call(struct rv_jit_context *ctx) -{ - return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); -} - -static void mark_tail_call(struct rv_jit_context *ctx) -{ - __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); -} - -static bool seen_tail_call(struct rv_jit_context *ctx) -{ - return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); -} - -static u8 rv_tail_call_reg(struct rv_jit_context *ctx) -{ - mark_tail_call(ctx); - - if (seen_call(ctx)) { - __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); - return RV_REG_S6; - } - return RV_REG_A6; -} - static bool is_32b_int(s64 val) { return -(1L << 31) <= val && val < (1L << 31); @@ -258,23 +228,20 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); store_offset -= 8; } - if (seen_reg(RV_REG_S6, ctx)) { - emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); - store_offset -= 8; - } if (ctx->arena_vm_start) { emit_ld(RV_REG_ARENA, store_offset, RV_REG_SP, ctx); store_offset -= 8; } + /* restore TCC from stack to RV_REG_TCC */ + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); /* Set return value. */ if (!is_tail_call) emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, - /* kcfi, fentry and TCC init insns will be skipped on tailcall */ - is_tail_call ? (RV_KCFI_NINSNS + RV_FENTRY_NINSNS + 1) * 4 : 0, - ctx); + is_tail_call ? RV_TAILCALL_OFFSET : 0, ctx); } static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, @@ -355,7 +322,6 @@ static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) { int tc_ninsn, off, start_insn = ctx->ninsns; - u8 tcc = rv_tail_call_reg(ctx); /* a0: &ctx * a1: &array @@ -378,7 +344,8 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) /* if (--TCC < 0) * goto out; */ - emit_addi(RV_REG_TCC, tcc, -1, ctx); + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + emit_addi(RV_REG_TCC, RV_REG_TCC, -1, ctx); off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); @@ -394,7 +361,10 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); - /* goto *(prog->bpf_func + 4); */ + /* store updated TCC back to stack */ + emit_sd(RV_REG_SP, ctx->tcc_offset, RV_REG_TCC, ctx); + + /* goto *(prog->bpf_func + RV_TAILCALL_OFFSET); */ off = offsetof(struct bpf_prog, bpf_func); if (is_12b_check(off, insn)) return -1; @@ -1028,12 +998,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, int i, ret, offset; int *branches_off = NULL; int stack_size = 0, nr_arg_slots = 0; - int retval_off, args_off, func_meta_off, ip_off, run_ctx_off, sreg_off, stk_arg_off; + int retval_off, args_off, func_meta_off, ip_off; + int run_ctx_off, sreg_off, stk_arg_off, tcc_off; int cookie_off, cookie_cnt; struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; - bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; + bool is_struct_ops = is_struct_ops_tramp(fentry); void *orig_call = func_addr; bool save_ret; u64 func_meta; @@ -1079,6 +1050,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, * * FP - sreg_off [ callee saved reg ] * + * FP - tcc_off [ tail call count ] BPF_TRAMP_F_TAIL_CALL_CTX + * * [ pads ] pads for 16 bytes alignment * * [ stack_argN ] @@ -1126,6 +1099,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, stack_size += 8; sreg_off = stack_size; + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { + stack_size += 8; + tcc_off = stack_size; + } + if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0)) stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8; @@ -1160,6 +1138,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); } + /* store tail call count */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); + /* callee saved register S1 to pass start time */ emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); @@ -1195,7 +1177,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } if (fmod_ret->nr_nodes) { - branches_off = kzalloc_objs(int, fmod_ret->nr_nodes); + branches_off = kvzalloc_objs(int, fmod_ret->nr_nodes); if (!branches_off) return -ENOMEM; @@ -1218,9 +1200,15 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, orig_call += RV_FENTRY_NINSNS * 4; restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx); restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx); + /* restore TCC to RV_REG_TCC before calling the orig bpf func */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_ld(RV_REG_TCC, -tcc_off, RV_REG_FP, ctx); ret = emit_call((const u64)orig_call, true, ctx); if (ret) goto out; + /* store updated TCC back to stack after calling the orig bpf func */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); im->ip_after_call = ctx->ro_insns + ctx->ninsns; @@ -1273,6 +1261,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); + /* restore TCC from stack to RV_REG_TCC */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_ld(RV_REG_TCC, -tcc_off, RV_REG_FP, ctx); + if (!is_struct_ops) { /* trampoline called from function entry */ emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); @@ -1300,7 +1292,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, ret = ctx->ninsns; out: - kfree(branches_off); + kvfree(branches_off); return ret; } @@ -1831,17 +1823,31 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, for (idx = 0; idx < fm->nr_args; idx++) { u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); + bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG; - if (fm->arg_size[idx] == sizeof(int)) - emit_sextw(reg, reg, ctx); + if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) + return -EINVAL; } } + /* restore TCC to RV_REG_TCC before bpf2bpf call */ + if (aux->tail_call_reachable && insn->src_reg == BPF_PSEUDO_CALL) + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + ret = emit_call(addr, fixed_addr, ctx); if (ret) return ret; - if (insn->src_reg != BPF_PSEUDO_CALL) + /* store updated TCC back to stack after bpf2bpf call */ + if (aux->tail_call_reachable && insn->src_reg == BPF_PSEUDO_CALL) + emit_sd(RV_REG_SP, ctx->tcc_offset, RV_REG_TCC, ctx); + + /* + * arch_bpf_timed_may_goto() is emitted by the verifier and + * returns its result in BPF_REG_AX instead of BPF_REG_0, so + * skip the normal "move return register into R0". + */ + if (insn->src_reg != BPF_PSEUDO_CALL && addr != (u64)arch_bpf_timed_may_goto) emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); break; } @@ -1986,7 +1992,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, else ret = emit_atomic_rmw(rd, rs, insn, ctx); - ret = ret ?: add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); + /* ret can be 1 (skip-zext); extable entry still needs to be added */ + if (ret >= 0) { + /* + * A load-acquire reads into dst_reg, and a read-modify-write + * carrying BPF_FETCH reads the old value into src_reg, or into + * r0 for a BPF_CMPXCHG. Clear that register on fault, the + * remaining atomics have no destination register. + */ + int load_reg = bpf_atomic_load_reg(insn); + + ret = add_exception_handler(insn, load_reg < 0 ? + REG_DONT_CLEAR_MARKER : regmap[load_reg], + ctx) ?: ret; + } + if (ret) return ret; break; @@ -2020,10 +2040,9 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) stack_adjust += 8; if (seen_reg(RV_REG_S5, ctx)) stack_adjust += 8; - if (seen_reg(RV_REG_S6, ctx)) - stack_adjust += 8; if (ctx->arena_vm_start) stack_adjust += 8; + stack_adjust += 8; /* RV_REG_TCC */ stack_adjust = round_up(stack_adjust, STACK_ALIGN); stack_adjust += bpf_stack_adjust; @@ -2033,15 +2052,16 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) /* emit kcfi type preamble immediately before the first insn */ emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); + /* bpf prog starts here as kcfi skipped during prog->bpf_func setup */ + /* nops reserved for auipc+jalr pair */ for (i = 0; i < RV_FENTRY_NINSNS; i++) emit(rv_nop(), ctx); - /* First instruction is always setting the tail-call-counter - * (TCC) register. This instruction is skipped for tail calls. - * Force using a 4-byte (non-compressed) instruction. - */ - emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); + if (!is_subprog) + emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); + + /* tailcall starts here, emit insn before it must be fixed */ emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); @@ -2071,26 +2091,20 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); store_offset -= 8; } - if (seen_reg(RV_REG_S6, ctx)) { - emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); - store_offset -= 8; - } if (ctx->arena_vm_start) { emit_sd(RV_REG_SP, store_offset, RV_REG_ARENA, ctx); store_offset -= 8; } + /* store TCC from RV_REG_TCC to stack */ + emit_sd(RV_REG_SP, store_offset, RV_REG_TCC, ctx); + ctx->tcc_offset = store_offset; + emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); if (bpf_stack_adjust) emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); - /* Program contains calls and tail calls, so RV_REG_TCC need - * to be saved across calls. - */ - if (seen_tail_call(ctx) && seen_call(ctx)) - emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); - ctx->stack_size = stack_adjust; if (ctx->arena_vm_start) @@ -2157,3 +2171,13 @@ bool bpf_jit_supports_fsession(void) { return true; } + +bool bpf_jit_supports_subprog_tailcalls(void) +{ + return true; +} + +bool bpf_jit_supports_timed_may_goto(void) +{ + return true; +} diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index ce3bd3762e08..cbfcd287ea16 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -72,7 +72,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); ctx->prog = prog; - ctx->offset = kzalloc_objs(int, prog->len); + ctx->offset = kvzalloc_objs(int, prog->len); if (!ctx->offset) goto out_offset; @@ -170,7 +170,7 @@ skip_init_ctx: ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); bpf_prog_fill_jited_linfo(prog, ctx->offset); out_offset: - kfree(ctx->offset); + kvfree(ctx->offset); kfree(jit_data); prog->aux->jit_data = NULL; } @@ -234,6 +234,7 @@ void bpf_jit_free(struct bpf_prog *prog) */ if (jit_data) { bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); + kvfree(jit_data->ctx.offset); kfree(jit_data); } hdr = bpf_jit_binary_pack_hdr(prog); diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S new file mode 100644 index 000000000000..02c637d87420 --- /dev/null +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Feng Jiang <jiangfeng@kylinos.cn> */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/* + * Trampoline for the BPF timed may_goto loop bound. Custom calling convention: + * - input: stack offset in BPF_REG_AX (t0) + * - output: updated count in BPF_REG_AX (t0) + * + * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where + * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (a5, a0-a4) are saved + * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving. + */ + +SYM_FUNC_START(arch_bpf_timed_may_goto) + addi sp, sp, -(8*SZREG) + REG_S ra, 7*SZREG(sp) + REG_S s0, 6*SZREG(sp) + addi s0, sp, 8*SZREG + + /* Save BPF registers R0-R5 (a5, a0-a4) */ + REG_S a5, 5*SZREG(sp) + REG_S a0, 4*SZREG(sp) + REG_S a1, 3*SZREG(sp) + REG_S a2, 2*SZREG(sp) + REG_S a3, 1*SZREG(sp) + REG_S a4, 0*SZREG(sp) + + add a0, t0, s5 + call bpf_check_timed_may_goto + mv t0, a0 + + /* Restore BPF registers R0-R5 */ + REG_L a4, 0*SZREG(sp) + REG_L a3, 1*SZREG(sp) + REG_L a2, 2*SZREG(sp) + REG_L a1, 3*SZREG(sp) + REG_L a0, 4*SZREG(sp) + REG_L a5, 5*SZREG(sp) + + REG_L s0, 6*SZREG(sp) + REG_L ra, 7*SZREG(sp) + addi sp, sp, 8*SZREG + ret +SYM_FUNC_END(arch_bpf_timed_may_goto) diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 31749c0362ca..c46872b071ce 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -743,10 +743,12 @@ static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, { if (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && - BPF_MODE(insn->code) != BPF_PROBE_MEM32) + BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; - if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { /* lgrl %r1,kern_arena */ EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); probe->arena_reg = REG_W1; @@ -758,7 +760,8 @@ static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, static void bpf_jit_probe_store_pre(struct bpf_jit *jit, struct bpf_insn *insn, struct bpf_jit_probe *probe) { - if (BPF_MODE(insn->code) != BPF_PROBE_MEM32) + if (BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; /* lgrl %r1,kern_arena */ @@ -771,6 +774,8 @@ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, struct bpf_insn *insn, struct bpf_jit_probe *probe) { + int load_reg; + if (BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; @@ -780,6 +785,14 @@ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit, EMIT4(0xb9080000, REG_W1, insn->dst_reg); probe->arena_reg = REG_W1; probe->prg = jit->prg; + /* + * A read-modify-write carrying BPF_FETCH reads the old value into + * src_reg, or into r0 for a BPF_CMPXCHG. Clear that register on + * fault, the remaining atomics only write memory. + */ + load_reg = bpf_atomic_load_reg(insn); + if (load_reg >= 0) + probe->reg = reg2hex[load_reg]; } static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, @@ -830,6 +843,72 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, return 0; } +static int emit_ldx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) +{ + struct bpf_jit_probe probe; + + bpf_jit_probe_init(&probe); + bpf_jit_probe_load_pre(jit, insn, &probe); + + switch (BPF_SIZE(insn->code)) { + case BPF_B: /* dst = *(u8 *)(ul) (src + off) */ + /* llgc %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0090, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + case BPF_H: /* dst = *(u16 *)(ul) (src + off) */ + /* llgh %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0091, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + case BPF_W: /* dst = *(u32 *)(ul) (src + off) */ + /* llgf %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0016, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + case BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ + /* lg %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + } + + return bpf_jit_probe_post(jit, fp, &probe); +} + +static int emit_stx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) +{ + struct bpf_jit_probe probe; + + bpf_jit_probe_init(&probe); + bpf_jit_probe_store_pre(jit, insn, &probe); + + switch (BPF_SIZE(insn->code)) { + case BPF_B: /* *(u8 *)(dst + off) = src_reg */ + /* stcy %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0072, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + case BPF_H: /* (u16 *)(dst + off) = src */ + /* sthy %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0070, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + case BPF_W: /* *(u32 *)(dst + off) = src */ + /* sty %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0050, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + case BPF_DW: /* (u64 *)(dst + off) = src */ + /* stg %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + } + + return bpf_jit_probe_post(jit, fp, &probe); +} + /* * Sign- or zero-extend the register if necessary */ @@ -1477,44 +1556,13 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, */ case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */ case BPF_STX | BPF_PROBE_MEM32 | BPF_B: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* stcy %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - break; case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_H: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* sthy %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - break; case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_W: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* sty %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - break; case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* stg %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); + err = emit_stx(jit, fp, insn); if (err < 0) return err; jit->seen |= SEEN_MEM; @@ -1574,19 +1622,23 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_ATOMIC */ + case BPF_STX | BPF_ATOMIC | BPF_B: + case BPF_STX | BPF_ATOMIC | BPF_H: case BPF_STX | BPF_ATOMIC | BPF_DW: case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: { bool is32 = BPF_SIZE(insn->code) == BPF_W; /* - * Unlike loads and stores, atomics have only a base register, - * but no index register. For the non-arena case, simply use - * %dst as a base. For the arena case, use the work register - * %r1: first, load the arena base into it, and then add %dst - * to it. + * Unlike loads and stores, s390 atomics have only a base + * register, but no index register. For the non-arena case, + * simply use %dst as a base. For the arena case, use the + * work register %r1: first, load the arena base into it, + * and then add %dst to it. */ probe.arena_reg = dst_reg; @@ -1642,6 +1694,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (load_probe.prg != -1) { probe.prg = jit->prg; probe.arena_reg = load_probe.arena_reg; + probe.reg = load_probe.reg; } loop_start = jit->prg; /* 0: {csy|csg} %w0,%src,off(%arena) */ @@ -1673,6 +1726,18 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (err < 0) return err; break; + case BPF_LOAD_ACQ: + /* s390 has strong ordering, just use load */ + err = emit_ldx(jit, fp, insn); + if (err < 0) + return err; + break; + case BPF_STORE_REL: + /* s390 has strong ordering, just use store */ + err = emit_stx(jit, fp, insn); + if (err < 0) + return err; + break; default: pr_err("Unknown atomic operation %02x\n", insn->imm); return -1; @@ -1687,15 +1752,20 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEM | BPF_B: case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* llgc %dst,off(%src,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); + case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ + case BPF_LDX | BPF_PROBE_MEM | BPF_H: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: + case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ + case BPF_LDX | BPF_PROBE_MEM | BPF_W: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: + case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ + case BPF_LDX | BPF_PROBE_MEM | BPF_DW: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: + err = emit_ldx(jit, fp, insn); if (err < 0) return err; jit->seen |= SEEN_MEM; - if (insn_is_zext(&insn[1])) + if (BPF_SIZE(insn->code) != BPF_DW && insn_is_zext(&insn[1])) insn_count = 2; break; case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */ @@ -1708,20 +1778,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, return err; jit->seen |= SEEN_MEM; break; - case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ - case BPF_LDX | BPF_PROBE_MEM | BPF_H: - case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* llgh %dst,off(%src,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - if (insn_is_zext(&insn[1])) - insn_count = 2; - break; case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: bpf_jit_probe_load_pre(jit, insn, &probe); @@ -1732,20 +1788,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, return err; jit->seen |= SEEN_MEM; break; - case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ - case BPF_LDX | BPF_PROBE_MEM | BPF_W: - case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* llgf %dst,off(%src) */ - jit->seen |= SEEN_MEM; - EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - if (insn_is_zext(&insn[1])) - insn_count = 2; - break; case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: bpf_jit_probe_load_pre(jit, insn, &probe); @@ -1756,18 +1798,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (err < 0) return err; break; - case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ - case BPF_LDX | BPF_PROBE_MEM | BPF_DW: - case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* lg %dst,off(%src,%arena) */ - jit->seen |= SEEN_MEM; - EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - break; /* * BPF_JMP / CALL */ @@ -1783,8 +1813,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, insn->imm == BPF_FUNC_get_smp_processor_id) { const u32 *cpu_nr = &get_lowcore()->cpu_nr; - /* ly %b0, cpu_nr */ - EMIT6_DISP_LH(0xe3000000, 0x0058, BPF_REG_0, REG_0, REG_0, + /* llgf %b0, cpu_nr */ + EMIT6_DISP_LH(0xe3000000, 0x0016, BPF_REG_0, REG_0, REG_0, (unsigned long)cpu_nr); break; } @@ -3028,13 +3058,6 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) if (!in_arena) return true; switch (insn->code) { - case BPF_STX | BPF_ATOMIC | BPF_B: - case BPF_STX | BPF_ATOMIC | BPF_H: - case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (bpf_atomic_is_load_store(insn)) - return false; - break; case BPF_LDX | BPF_MEMSX | BPF_B: case BPF_LDX | BPF_MEMSX | BPF_H: case BPF_LDX | BPF_MEMSX | BPF_W: diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index fb67217fddcd..079f8c7e9e3c 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -1107,10 +1107,10 @@ struct execmem_info __init *execmem_arch_setup(void) .alignment = MODULE_ALIGN, }, [EXECMEM_BPF] = { - .flags = EXECMEM_KASAN_SHADOW, + .flags = flags, .start = start, .end = MODULES_END, - .pgprot = PAGE_KERNEL, + .pgprot = pgprot, .alignment = MODULE_ALIGN, }, [EXECMEM_MODULE_DATA] = { diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index de7515ea1bea..1a9fb530adc3 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1473,17 +1473,20 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, * * Bit layout of `fixup` (32-bit): * - * +-----------+--------+-----------+---------+----------+ - * | 31 | 30-24 | 23-16 | 15-8 | 7-0 | - * | | | | | | - * | ARENA_ACC | Unused | ARENA_REG | DST_REG | INSN_LEN | - * +-----------+--------+-----------+---------+----------+ + * +-----------+-------------+--------+-----------+---------+----------+ + * | 31 | 30 | 29-24 | 23-16 | 15-8 | 7-0 | + * | | | | | | | + * | ARENA_ACC | ARENA_WRITE | Unused | ARENA_REG | DST_REG | INSN_LEN | + * +-----------+-------------+--------+-----------+---------+----------+ * * - INSN_LEN (8 bits): Length of faulting insn (max x86 insn = 15 bytes (fits in 8 bits)). * - DST_REG (8 bits): Offset of dst_reg from reg2pt_regs[] (max offset = 112 (fits in 8 bits)). - * This is set to DONT_CLEAR if the insn is a store. + * This is set to DONT_CLEAR if the insn does not read into a register. * - ARENA_REG (8 bits): Offset of the register that is used to calculate the * address for load/store when accessing the arena region. + * - ARENA_WRITE (1 bit): This bit is set when the faulting instruction wrote to the arena region. + * It is independent of DST_REG, since a read-modify-write both writes to + * memory and reads the old value into a register. * - ARENA_ACCESS (1 bit): This bit is set when the faulting instruction accessed the arena region. * * Bit layout of `data` (32-bit): @@ -1502,6 +1505,7 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, #define FIXUP_INSN_LEN_MASK GENMASK(7, 0) #define FIXUP_REG_MASK GENMASK(15, 8) #define FIXUP_ARENA_REG_MASK GENMASK(23, 16) +#define FIXUP_ARENA_WRITE BIT(30) #define FIXUP_ARENA_ACCESS BIT(31) #define DATA_ARENA_OFFSET_MASK GENMASK(31, 16) @@ -1510,7 +1514,7 @@ bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs) u32 reg = FIELD_GET(FIXUP_REG_MASK, x->fixup); u32 insn_len = FIELD_GET(FIXUP_INSN_LEN_MASK, x->fixup); bool is_arena = !!(x->fixup & FIXUP_ARENA_ACCESS); - bool is_write = (reg == DONT_CLEAR); + bool is_write = !!(x->fixup & FIXUP_ARENA_WRITE); unsigned long addr; s16 off; u32 arena_reg; @@ -1678,6 +1682,50 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, return 0; } +/* + * Rebase the __arena args of a kfunc call to arena kernel addresses, + * rN = kern_vm_start + (u32)rN, with R12 holding kern_vm_start. A nullable + * arg preserves NULL by skipping the add, tested on the truncated value as + * arena NULL is offset 0. Return the number of emitted bytes. + */ +static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, + const struct bpf_insn *insn, u8 **pprog) +{ + const struct btf_func_model *fm; + u8 *prog = *pprog; + u8 *start = prog; + int i; + + fm = bpf_jit_find_kfunc_model(bpf_prog, insn); + if (!fm) + return -EINVAL; + + for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { + u8 flags = fm->arg_flags[i]; + u32 reg = BPF_REG_1 + i; + + if (!(flags & BTF_FMODEL_ARENA_ARG)) + continue; + if (WARN_ON_ONCE(!bpf_prog->aux->arena)) + return -EINVAL; + + /* mov eN, eN: truncate and clear the upper 32 bits */ + emit_mov_reg(&prog, false, reg, reg); + if (flags & BTF_FMODEL_NULLABLE_ARG) { + /* test eN, eN; jz over the 3-byte add */ + maybe_emit_mod(&prog, reg, reg, false); + EMIT2(0x85, add_2reg(0xC0, reg, reg)); + EMIT2(X86_JE, 3); + } + /* add rN, r12 */ + maybe_emit_mod(&prog, reg, X86_REG_R12, true); + EMIT2(0x01, add_2reg(0xC0, reg, X86_REG_R12)); + } + + *pprog = prog; + return prog - start; +} + static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding) { @@ -1887,7 +1935,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_mov(dst_reg, src_reg); #ifdef CONFIG_SMP /* add <dst>, gs:[<off>] */ - EMIT2(0x65, add_1mod(0x48, dst_reg)); + EMIT2(0x65, add_2mod(0x48, 0, dst_reg)); EMIT3(0x03, add_2reg(0x04, 0, dst_reg), 0x25); EMIT((u32)(unsigned long)&this_cpu_off, 4); #endif @@ -2304,6 +2352,7 @@ populate_extable: struct exception_table_entry *ex; u8 *_insn = image + proglen + (start_of_ldx - temp); u32 arena_reg, fixup_reg; + bool is_write; s64 delta; if (!bpf_prog->aux->extable) @@ -2331,19 +2380,38 @@ populate_extable: * BPF_PROBE_ATOMIC) before being used for the memory access. Pass * the reg holding the unmodified 32-bit address to * ex_handler_bpf(). + * + * A load-acquire is of BPF_STX class, but reads from src_reg + * into dst_reg like a BPF_LDX does, hence it must not be + * treated as a store here. */ - if (BPF_CLASS(insn->code) == BPF_LDX) { + if (BPF_CLASS(insn->code) == BPF_LDX || + bpf_atomic_is_load_acq(insn)) { arena_reg = reg2pt_regs[src_reg]; fixup_reg = reg2pt_regs[dst_reg]; + is_write = false; } else { + /* + * A store has no destination register to clear, + * except for a read-modify-write with BPF_FETCH, + * which also reads the old value into src_reg, or + * into r0 for a BPF_CMPXCHG. Either way the access + * is still reported as a write. + */ + int load_reg = bpf_atomic_load_reg(insn); + arena_reg = reg2pt_regs[dst_reg]; - fixup_reg = DONT_CLEAR; + fixup_reg = load_reg < 0 ? DONT_CLEAR : + reg2pt_regs[load_reg]; + is_write = true; } ex->fixup = FIELD_PREP(FIXUP_INSN_LEN_MASK, prog - start_of_ldx) | FIELD_PREP(FIXUP_ARENA_REG_MASK, arena_reg) | FIELD_PREP(FIXUP_REG_MASK, fixup_reg); ex->fixup |= FIXUP_ARENA_ACCESS; + if (is_write) + ex->fixup |= FIXUP_ARENA_WRITE; ex->data |= FIELD_PREP(DATA_ARENA_OFFSET_MASK, insn->off); } @@ -2583,6 +2651,12 @@ populate_extable: } if (!imm32) return -EINVAL; + if (src_reg == BPF_PSEUDO_KFUNC_CALL) { + err = emit_kfunc_arena_args(bpf_prog, insn, &prog); + if (err < 0) + return err; + ip += err; + } if (priv_frame_ptr) { push_r9(&prog); ip += 2; @@ -2993,11 +3067,39 @@ static int get_nr_used_regs(const struct btf_func_model *m) return nr_used_regs; } +/* + * Convert an arena kernel address into the arena pointer form on its way + * into the BPF ctx, rax = (u32)(src - kern_vm_start). A nullable arg + * preserves NULL, tested on the full 64-bit kernel pointer. The 32-bit + * subtraction both truncates and clears the upper half, so the stored + * value satisfies the JIT invariant for arena pointer registers. + */ +static void emit_arena_arg_conv(u8 **pprog, u32 src_reg, bool nullable, u32 base_lo) +{ + u8 *prog = *pprog; + + if (nullable) { + if (src_reg != BPF_REG_0) + emit_mov_reg(&prog, true, BPF_REG_0, src_reg); + /* test rax, rax; jz over the 5-byte sub */ + EMIT3(0x48, 0x85, 0xC0); + EMIT2(X86_JE, 5); + } else if (src_reg != BPF_REG_0) { + emit_mov_reg(&prog, false, BPF_REG_0, src_reg); + } + /* sub eax, base_lo */ + EMIT1_off32(0x2D, base_lo); + + *pprog = prog; +} + static void save_args(const struct btf_func_model *m, u8 **prog, - int stack_size, bool for_call_origin, u32 flags) + int stack_size, bool for_call_origin, u32 flags, + u64 arena_base) { int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0; bool use_jmp = bpf_trampoline_use_jmp(flags); + int stack_args_off = (use_jmp || (flags & BPF_TRAMP_F_INDIRECT)) ? 16 : 24; int i, j; /* Store function arguments to stack. @@ -3006,6 +3108,9 @@ static void save_args(const struct btf_func_model *m, u8 **prog, * mov QWORD PTR [rbp-0x8],rsi */ for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { + bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); + bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; + arg_regs = (m->arg_size[i] + 7) / 8; /* According to the research of Yonghong, struct members @@ -3029,16 +3134,19 @@ static void save_args(const struct btf_func_model *m, u8 **prog, /* copy function arguments from origin stack frame * into current stack frame. * - * The starting address of the arguments on-stack - * is: - * rbp + 8(push rbp) + - * 8(return addr of origin call) + - * 8(return addr of the caller) - * which means: rbp + 24 + * The arguments on-stack start above the saved rbp + * and the return addresses: two return addresses + * (origin call and caller) when the trampoline is + * entered through the fentry call, so rbp + 24, and + * a single one when it is entered with a jmp or + * called indirectly, so rbp + 16. */ for (j = 0; j < arg_regs; j++) { emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, - nr_stack_slots * 8 + 16 + (!use_jmp) * 8); + nr_stack_slots * 8 + stack_args_off); + if (arena_arg) + emit_arena_arg_conv(prog, BPF_REG_0, nullable, + (u32)arena_base); emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -stack_size); @@ -3059,9 +3167,13 @@ static void save_args(const struct btf_func_model *m, u8 **prog, /* copy the arguments from regs into stack */ for (j = 0; j < arg_regs; j++) { - emit_stx(prog, BPF_DW, BPF_REG_FP, - nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs, - -stack_size); + u32 src = nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs; + + if (arena_arg) { + emit_arena_arg_conv(prog, src, nullable, (u32)arena_base); + src = BPF_REG_0; + } + emit_stx(prog, BPF_DW, BPF_REG_FP, src, -stack_size); stack_size -= 8; nr_regs++; } @@ -3357,6 +3469,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im void *orig_call = func_addr; int cookie_off, cookie_cnt; u8 **branches = NULL; + u64 arena_base; u64 func_meta; u8 *prog; bool save_ret; @@ -3369,11 +3482,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); - /* extra registers for struct arguments */ - for (i = 0; i < m->nr_args; i++) { - if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) - nr_regs += (m->arg_size[i] + 7) / 8 - 1; - } + arena_base = bpf_tramp_arena_base(m, tnodes, flags); + + for (i = 0; i < m->nr_args; i++) + nr_regs += (m->arg_size[i] + 7) / 8 - 1; /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 * are passed through regs, the remains are through stack. @@ -3506,7 +3618,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_store_stack_imm64(&prog, BPF_REG_0, -ip_off, (long)func_addr); } - save_args(m, &prog, regs_off, false, flags); + save_args(m, &prog, regs_off, false, flags, arena_base); if (flags & BPF_TRAMP_F_CALL_ORIG) { /* arg1: mov rdi, im */ @@ -3548,7 +3660,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (flags & BPF_TRAMP_F_CALL_ORIG) { restore_regs(m, &prog, regs_off); - save_args(m, &prog, arg_stack_off, true, flags); + save_args(m, &prog, arg_stack_off, true, flags, 0); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before calling the original function, load the @@ -3703,13 +3815,12 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, int ret; /* Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). - * This will NOT cause fragmentation in direct map, as we do not - * call set_memory_*() on this buffer. * * We cannot use kvmalloc here, because we need image to be in * module memory range. + * Since it must be writable use bpf_jit_alloc_exec_rw(). */ - image = bpf_jit_alloc_exec(PAGE_SIZE); + image = bpf_jit_alloc_exec_rw(PAGE_SIZE); if (!image) return -ENOMEM; @@ -4050,6 +4161,11 @@ bool bpf_jit_supports_stack_args(void) return true; } +bool bpf_jit_supports_arena_args(void) +{ + return true; +} + void *bpf_arch_text_copy(void *dst, void *src, size_t len) { if (text_poke_copy(dst, src, len) == NULL) diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c index 536f6d01fd14..44671dbdeca8 100644 --- a/drivers/hid/bpf/hid_bpf_dispatch.c +++ b/drivers/hid/bpf/hid_bpf_dispatch.c @@ -590,11 +590,11 @@ static const struct btf_kfunc_id_set hid_bpf_kfunc_set = { /* for syscall HID-BPF */ BTF_KFUNCS_START(hid_bpf_syscall_kfunc_ids) -BTF_ID_FLAGS(func, hid_bpf_allocate_context, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, hid_bpf_release_context, KF_RELEASE) -BTF_ID_FLAGS(func, hid_bpf_hw_request) -BTF_ID_FLAGS(func, hid_bpf_hw_output_report) -BTF_ID_FLAGS(func, hid_bpf_input_report) +BTF_ID_FLAGS(func, hid_bpf_allocate_context, KF_ACQUIRE | KF_RET_NULL | KF_SLEEPABLE) +BTF_ID_FLAGS(func, hid_bpf_release_context, KF_RELEASE | KF_SLEEPABLE) +BTF_ID_FLAGS(func, hid_bpf_hw_request, KF_SLEEPABLE) +BTF_ID_FLAGS(func, hid_bpf_hw_output_report, KF_SLEEPABLE) +BTF_ID_FLAGS(func, hid_bpf_input_report, KF_SLEEPABLE) BTF_KFUNCS_END(hid_bpf_syscall_kfunc_ids) static const struct btf_kfunc_id_set hid_bpf_syscall_kfunc_set = { diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 7719f6528445..ffa5626411ac 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -145,6 +145,7 @@ struct bpf_map_ops { int (*map_direct_value_meta)(const struct bpf_map *map, u64 imm, u32 *off); int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); + vm_fault_t (*map_mmap_fault)(struct bpf_map *map, struct vm_fault *vmf); __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, struct poll_table_struct *pts); unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr, @@ -299,7 +300,6 @@ struct bpf_map_owner { struct bpf_map { u8 sha[SHA256_DIGEST_SIZE]; - u32 excl; const struct bpf_map_ops *ops; struct bpf_map *inner_map_meta; #ifdef CONFIG_SECURITY @@ -570,7 +570,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (IS_ERR_OR_NULL(rec)) { if (long_memcpy) - bpf_long_memcpy(dst, src, round_up(size, 8)); + bpf_long_memcpy(dst, src, size); else memcpy(dst, src, size); return; @@ -593,7 +593,7 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) { - bpf_obj_memcpy(map->record, dst, src, map->value_size, true); + bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true); } static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) @@ -889,8 +889,8 @@ enum bpf_arg_type { ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ ARG_PTR_TO_ARENA, - ARG_CONST_SIZE, /* number of bytes accessed from memory */ - ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ + ARG_MEM_SIZE, /* number of bytes accessed from memory */ + ARG_MEM_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ ARG_PTR_TO_CTX, /* pointer to context */ ARG_ANYTHING, /* any (initialized) argument is ok */ @@ -961,6 +961,21 @@ enum bpf_return_type { }; static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); +/* The longest tracepoint has 12 args. + * See include/trace/bpf_probe.h + * + * Also reuse this macro for maximum number of arguments a BPF function + * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via + * stack arg slots. The JIT may map some stack arg slots to registers based + * on the native calling convention (e.g., arg 6 to R9 on x86-64). + */ +#define MAX_BPF_FUNC_ARGS 12 + +/* The maximum number of arguments passed through registers + * a single function may have. + */ +#define MAX_BPF_FUNC_REG_ARGS 5 + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -985,7 +1000,7 @@ struct bpf_func_proto { enum bpf_arg_type arg4_type; enum bpf_arg_type arg5_type; }; - enum bpf_arg_type arg_type[5]; + enum bpf_arg_type arg_type[MAX_BPF_FUNC_ARGS]; }; union { struct { @@ -995,7 +1010,7 @@ struct bpf_func_proto { u32 *arg4_btf_id; u32 *arg5_btf_id; }; - u32 *arg_btf_id[5]; + u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; struct { size_t arg1_size; size_t arg2_size; @@ -1003,7 +1018,7 @@ struct bpf_func_proto { size_t arg4_size; size_t arg5_size; }; - size_t arg_size[5]; + size_t arg_size[MAX_BPF_FUNC_ARGS]; }; int *ret_btf_id; /* return value btf_id */ bool (*allowed)(const struct bpf_prog *prog); @@ -1118,21 +1133,6 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn) return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } -/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an - * atomic load or store, and false if it is a read-modify-write instruction. - */ -static inline bool -bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) -{ - switch (atomic_insn->imm) { - case BPF_LOAD_ACQ: - case BPF_STORE_REL: - return true; - default: - return false; - } -} - struct bpf_prog_ops { int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); @@ -1193,27 +1193,15 @@ struct bpf_prog_offload { u32 jited_len; }; -/* The longest tracepoint has 12 args. - * See include/trace/bpf_probe.h - * - * Also reuse this macro for maximum number of arguments a BPF function - * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via - * stack arg slots. The JIT may map some stack arg slots to registers based - * on the native calling convention (e.g., arg 6 to R9 on x86-64). - */ -#define MAX_BPF_FUNC_ARGS 12 - -/* The maximum number of arguments passed through registers - * a single function may have. - */ -#define MAX_BPF_FUNC_REG_ARGS 5 - -/* The argument is a structure or a union. */ -#define BTF_FMODEL_STRUCT_ARG BIT(0) - /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) +/* The argument is an arena pointer. */ +#define BTF_FMODEL_ARENA_ARG BIT(2) + +/* The argument is nullable. */ +#define BTF_FMODEL_NULLABLE_ARG BIT(3) + struct btf_func_model { u8 ret_size; u8 ret_flags; @@ -1287,6 +1275,15 @@ struct bpf_tramp_nodes { int nr_nodes; }; +/* + * The arena base against which a struct_ops trampoline converts the + * arguments marked with BTF_FMODEL_ARENA_ARG while saving them into the BPF + * ctx, ctx[arg] = (u32)(kaddr - kern_vm_start). Zero when the trampoline + * converts nothing. + */ +u64 bpf_tramp_arena_base(const struct btf_func_model *m, + struct bpf_tramp_nodes *tnodes, u32 flags); + struct bpf_tramp_run_ctx; /* Different use cases for BPF trampoline: @@ -1522,8 +1519,9 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, struct bpf_tracing_multi_link *link); -int bpf_trampoline_multi_detach(struct bpf_prog *prog, - struct bpf_tracing_multi_link *link); +void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link); +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags); /* * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn @@ -1642,11 +1640,11 @@ static inline int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, { return -ENOTSUPP; } -static inline int bpf_trampoline_multi_detach(struct bpf_prog *prog, - struct bpf_tracing_multi_link *link) +static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link) { - return -ENOTSUPP; } +static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif struct bpf_func_info_aux { @@ -1865,8 +1863,9 @@ struct bpf_prog_aux { struct bpf_prog { u16 pages; /* Number of allocated pages */ - u16 jited:1, /* Is our filter JIT'ed? */ + u32 jited:1, /* Is our filter JIT'ed? */ jit_requested:1,/* archs need to JIT the prog */ + jit_required:1, /* program strictly requires JIT compiler */ gpl_compatible:1, /* Is filter GPL compatible? */ cb_access:1, /* Is control block accessed? */ dst_needed:1, /* Do we need dst entry? */ @@ -2196,6 +2195,12 @@ static inline bool is_tracing_multi(enum bpf_attach_type type) type == BPF_TRACE_FSESSION_MULTI; } +static inline bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes) +{ + return fentry_nodes->nr_nodes == 1 && + fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS; +} + #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) /* This macro helps developer to register a struct_ops type and generate * type information correctly. Developers should use this macro to register @@ -3146,7 +3151,7 @@ int btf_struct_access(struct bpf_verifier_log *log, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict); + bool strict, bool walk_flex_arrays); int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf *btf, @@ -3170,7 +3175,6 @@ const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); @@ -3509,11 +3513,6 @@ static inline void bpf_task_storage_free(struct task_struct *task) { } -static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return false; -} - static inline const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) @@ -4148,8 +4147,16 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog) } const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off); -void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo, - const char **filep, const char **linep, int *nump); +struct bpf_linfo_source { + const char *file; + const char *line; + u32 file_name_off; + int line_num; + int line_col; +}; + +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src); int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump); struct bpf_prog *bpf_prog_find_from_stack(void); @@ -4169,7 +4176,7 @@ bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image) } #endif -static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type) +static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) { switch (map_type) { case BPF_MAP_TYPE_PERCPU_ARRAY: @@ -4196,7 +4203,7 @@ static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 all return -EINVAL; if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) { - if (!bpf_map_supports_cpu_flags(map->map_type)) + if (!bpf_map_is_percpu_map(map->map_type)) return -EINVAL; if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS)) return -EINVAL; diff --git a/include/linux/bpf_ksock.h b/include/linux/bpf_ksock.h new file mode 100644 index 000000000000..cb387fb75e43 --- /dev/null +++ b/include/linux/bpf_ksock.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2026 Isovalent */ + +#ifndef _BPF_KSOCK_H +#define _BPF_KSOCK_H + +#include <linux/types.h> +#include <linux/in.h> +#include <linux/in6.h> + +/** + * struct bpf_ksock_create_opts - BPF kernel socket creation parameters + * @family: Address family: AF_INET or AF_INET6. + * @type: Socket type: only SOCK_DGRAM supported for now. + * @protocol: Protocol number (e.g. IPPROTO_UDP), or 0 for the default protocol + * of the given type. + * @reserved: Must be zero. Reserved for future use. + */ +struct bpf_ksock_create_opts { + __u8 family; + __u8 type; + __u8 protocol; + __u8 reserved; +}; + +/** + * union bpf_ksock_addr - IPv4 or IPv6 socket address + * @sin: IPv4 socket address. + * @sin6: IPv6 socket address. + */ +union bpf_ksock_addr { + struct sockaddr_in sin; + struct sockaddr_in6 sin6; +}; + +#endif /* _BPF_KSOCK_H */ diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 39a851e690ec..5fad59fdab0d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -15,7 +15,7 @@ * ensures that umax_value + (int)off + (int)size cannot overflow a u64. */ #define BPF_MAX_VAR_OFF (1 << 29) -/* Maximum variable size permitted for ARG_CONST_SIZE[_OR_ZERO]. This ensures +/* Maximum variable size permitted for ARG_MEM_SIZE[_OR_ZERO]. This ensures * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) @@ -162,11 +162,6 @@ struct bpf_reg_state { * pointing to bpf_func_state. */ u32 frameno; - /* Tracks subreg definition. The stored value is the insn_idx of the - * writing insn. This is safe because subreg_def is used before any insn - * patching which only happens after main verification finished. - */ - s32 subreg_def; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ bool precise; }; @@ -359,6 +354,11 @@ struct bpf_func_state { * 0 = main function, 1 = first callee. */ u32 frameno; + /* + * Unique diagnostic identity for this function invocation. Frame depth is + * reused after returns, while this ID is preserved across state clones. + */ + u32 diag_frame_id; /* subprog number == index within subprog_info * zero == main subprog */ @@ -385,6 +385,8 @@ struct bpf_func_state { * | number of simulations is tracked in frame N */ u32 callback_depth; + /* Instructions processed in this frame and callees on the current path. */ + u32 insns_subtotal; /* The following fields should be last. See copy_func_state() */ /* The state of the stack. Each element of the array describes BPF_REG_SIZE @@ -803,7 +805,8 @@ struct bpf_subprog_info { u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */ u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; - u32 insn_processed; + u32 insns_total; + u32 insns_self; /* offsets in range [stack_depth .. fastcall_stack_off) * are used for bpf_fastcall spills and fills. */ @@ -835,6 +838,7 @@ static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) return 0; } +struct bpf_diag; struct bpf_verifier_env; struct backtrack_state { @@ -898,6 +902,14 @@ struct bpf_scc_info { struct bpf_liveness; +struct bpf_fd_array { + union { + struct bpf_map *map; + struct btf *btf; + unsigned long val; + }; +}; + /* single container for all structs * one verifier_env per bpf_check() call */ @@ -939,9 +951,12 @@ struct bpf_verifier_env { bool bypass_spec_v4; bool seen_direct_write; bool seen_exception; + bool signature; + u32 insn_aux_data_len; struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; + struct bpf_diag *diag; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; @@ -989,7 +1004,19 @@ struct bpf_verifier_env { u32 free_list_size; u32 explored_states_size; u32 num_backedges; - bpfptr_t fd_array; + /* + * The program's fd_array comes in two shapes, told apart by whether + * the caller passed fd_array_cnt. They are mutually exclusive: + * - continuous (fd_array_cnt given): ->fd_array holds every entry + * resolved to its object up front, indexed by fd_array position, + * with ->fd_array_cnt slots; ->fd_array_raw is unused. + * - sparse (no fd_array_cnt): ->fd_array is NULL, and entries are + * read from ->fd_array_raw (the caller's fd_array) and resolved + * on the spot at each reference. + */ + struct bpf_fd_array *fd_array; + u32 fd_array_cnt; + bpfptr_t fd_array_raw; /* bit mask to keep track of whether a register has been accessed * since the last time the function state was printed @@ -1156,8 +1183,8 @@ static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id) *btf_id = key & 0x7FFFFFFF; } -int bpf_check_btf_info_early(struct bpf_verifier_env *env, - const union bpf_attr *attr, bpfptr_t uattr); +int bpf_prepare_btf_info(struct bpf_verifier_env *env, + const union bpf_attr *attr, bpfptr_t uattr); int bpf_check_btf_info(struct bpf_verifier_env *env, const union bpf_attr *attr, bpfptr_t uattr); @@ -1243,6 +1270,11 @@ static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, bt->stack_masks[frame] |= 1ull << slot; } +static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask) +{ + bt->stack_masks[frame] |= mask; +} + static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) { bt->stack_arg_masks[frame] |= 1 << slot; @@ -1276,7 +1308,37 @@ static inline u32 type_flag(u32 type) return type & ~BPF_BASE_TYPE_MASK; } -/* only use after check_attach_btf_id() */ +static inline bool bpf_is_ptr_to_mem_or_btf_id(enum bpf_reg_type type) +{ + switch (base_type(type)) { + case PTR_TO_MEM: + case PTR_TO_BTF_ID: + return true; + default: + return false; + } +} + +static inline bool bpf_may_fault_on_deref(enum bpf_reg_type type) +{ + /* + * The pointer types which must not be dereferenced without fault + * protection, that is, the ones bpf_convert_ctx_accesses() has to + * turn a BPF_LDX into a BPF_PROBE_MEM one for. + */ + return type == PTR_TO_BTF_ID || (type_flag(type) & PTR_UNTRUSTED); +} + +static inline bool bpf_prog_has_arena_ctx_arg(const struct bpf_prog *prog) +{ + int i; + + for (i = 0; i < prog->aux->ctx_arg_info_size; i++) + if (base_type(prog->aux->ctx_arg_info[i].reg_type) == PTR_TO_ARENA) + return true; + return false; +} + static inline enum bpf_prog_type resolve_prog_type(const struct bpf_prog *prog) { return (prog->type == BPF_PROG_TYPE_EXT && prog->aux->saved_dst_prog_type) ? @@ -1315,6 +1377,18 @@ static inline bool type_is_non_owning_ref(u32 type) return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF; } +static inline bool type_is_map_ptr(enum bpf_reg_type type) +{ + switch (base_type(type)) { + case CONST_PTR_TO_MAP: + case PTR_TO_MAP_KEY: + case PTR_TO_MAP_VALUE: + return true; + default: + return false; + } +} + static inline bool type_is_pkt_pointer(enum bpf_reg_type type) { type = base_type(type); @@ -1399,8 +1473,10 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, u32 frameno); u32 bpf_vlog_alignment(u32 pos); +const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn); struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off); +const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog); int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); @@ -1438,23 +1514,46 @@ struct ref_obj_desc { u8 cnt; }; -struct bpf_kfunc_call_arg_meta { - /* In parameters */ +/* + * A memory argument a call fills in. The verifier allows the stack to be uninitialized if + * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access(). + */ +struct arg_raw_mem_desc { + u8 regno; + int size; +}; + +/* Size of PTR_TO_MEM returned, taken from a constant allocation-size argument */ +struct ret_mem_desc { + u32 size; + bool found; +}; + +/* A constant scalar argument; Populated by process_const_arg() */ +struct arg_constant_desc { + u64 value; + bool found; +}; + +struct bpf_call_arg_meta { + /* Common */ struct btf *btf; u32 func_id; - u32 kfunc_flags; - const struct btf_type *func_proto; - const char *func_name; - /* Out parameters */ + const struct bpf_func_proto *fn; u8 release_regno; - bool r0_rdonly; u32 ret_btf_id; - u64 r0_size; u32 subprogno; - struct { - u64 value; - bool found; - } arg_constant; + struct bpf_map_desc map; + struct bpf_dynptr_desc dynptr; + struct ref_obj_desc ref_obj; + struct ret_mem_desc ret_mem; + + /* Only set by kfunc */ + bool r0_rdonly; + u32 kfunc_flags; + const struct btf_type *func_proto; + const char *func_name; + struct arg_constant_desc arg_constant; /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, * generally to pass info about user-defined local kptr types to later @@ -1481,28 +1580,31 @@ struct bpf_kfunc_call_arg_meta { u8 spi; u8 frameno; } iter; - struct bpf_map_desc map; - struct bpf_dynptr_desc dynptr; - struct ref_obj_desc ref_obj; - u64 mem_size; + + /* Only set by helper */ + u64 msize_max_value; + s64 const_map_key; + struct btf *ret_btf; + struct btf_field *kptr_field; + struct arg_raw_mem_desc arg_raw_mem; }; int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, const struct bpf_func_proto **ptr); int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, - s16 offset, struct bpf_kfunc_call_arg_meta *meta); + s16 offset, struct bpf_call_arg_meta *meta); bool bpf_is_async_callback_calling_insn(struct bpf_insn *insn); bool bpf_is_sync_callback_calling_insn(struct bpf_insn *insn); -static inline bool bpf_is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static inline bool bpf_is_iter_next_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ITER_NEXT; } -static inline bool bpf_is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta) +static inline bool bpf_is_kfunc_sleepable(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_SLEEPABLE; } -bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta); +bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta); struct bpf_iarray *bpf_iarray_realloc(struct bpf_iarray *old, size_t n_elem); int bpf_copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off); bool bpf_insn_is_cond_jump(u8 code); @@ -1566,6 +1668,7 @@ enum bpf_reg_arg_type { struct bpf_kfunc_desc { struct btf_func_model func_model; + struct bpf_func_proto proto; u32 func_id; s32 imm; u16 offset; @@ -1573,17 +1676,18 @@ struct bpf_kfunc_desc { }; struct bpf_kfunc_desc_tab { + u32 nr_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during * verification. JITs do lookups by bpf_insn, where func_id may not be * available, therefore at the end of verification do_misc_fixups() * sorts this by imm and offset. + * + * Grown one entry at a time by bpf_add_kfunc_call(). */ - struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS]; - u32 nr_descs; + struct bpf_kfunc_desc descs[]; }; /* Functions exported from verifier.c, used by fixups.c */ -bool bpf_is_reg64(struct bpf_insn *insn, u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t); void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog); bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env); @@ -1607,5 +1711,6 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env); int bpf_jit_subprogs(struct bpf_verifier_env *env); int bpf_fixup_call_args(struct bpf_verifier_env *env); int bpf_do_misc_fixups(struct bpf_verifier_env *env); +int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn); #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/include/linux/btf.h b/include/linux/btf.h index 8f8d3f7a8929..89d5a5c4f117 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -79,6 +79,7 @@ #define KF_ARENA_ARG1 (1 << 14) /* kfunc takes an arena pointer as its first argument */ #define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ +#define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the @@ -213,6 +214,7 @@ int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, void *obj, */ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, char *buf, int len, u64 flags); +int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len); int btf_get_fd_by_id(u32 id); u32 btf_obj_id(const struct btf *btf); diff --git a/include/linux/filter.h b/include/linux/filter.h index 14acb2455746..4a9bc6a848f2 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -383,6 +383,61 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn) /* Legacy alias */ #define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF) +/* + * Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an + * atomic load or store, and false if it is a read-modify-write instruction. + */ +static inline bool +bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) +{ + switch (atomic_insn->imm) { + case BPF_LOAD_ACQ: + case BPF_STORE_REL: + return true; + default: + return false; + } +} + +/* + * A load-acquire is the only BPF_STX class instruction that reads into + * dst_reg from src_reg + off16, i.e. it has the operand roles of a BPF_LDX. + * Unlike bpf_atomic_is_load_store(), @insn is not assumed to be a BPF_ATOMIC + * instruction here, so that callers which walk all instruction classes can + * use this directly. + */ +static inline bool bpf_atomic_is_load_acq(const struct bpf_insn *insn) +{ + return BPF_CLASS(insn->code) == BPF_STX && + (BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && + insn->imm == BPF_LOAD_ACQ; +} + +/* + * Given an instruction @insn, return the number of the BPF register that a + * BPF_ATOMIC reads the value at its memory operand into, or -1 if there is + * no such register. That is the register a BPF_PROBE_ATOMIC has to clear when + * the access faults. Like bpf_atomic_is_load_acq(), @insn is not assumed to + * be a BPF_ATOMIC here. + */ +static inline int bpf_atomic_load_reg(const struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_STX || + (BPF_MODE(insn->code) != BPF_ATOMIC && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)) + return -1; + + switch (insn->imm) { + case BPF_LOAD_ACQ: + return insn->dst_reg; + case BPF_CMPXCHG: + return BPF_REG_0; + default: + return (insn->imm & BPF_FETCH) ? insn->src_reg : -1; + } +} + /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ #define BPF_ST_MEM(SIZE, DST, OFF, IMM) \ @@ -1183,6 +1238,7 @@ bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); bool bpf_jit_supports_stack_args(void); +bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); @@ -1211,25 +1267,12 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, #ifdef CONFIG_BPF_SYSCALL struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len); -struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env); -void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux); #else static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len) { return ERR_PTR(-ENOTSUPP); } - -static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) -{ - return NULL; -} - -static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux) -{ -} #endif /* CONFIG_BPF_SYSCALL */ int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt); @@ -1333,6 +1376,7 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, void bpf_jit_binary_free(struct bpf_binary_header *hdr); u64 bpf_jit_alloc_exec_limit(void); void *bpf_jit_alloc_exec(unsigned long size); +void *bpf_jit_alloc_exec_rw(unsigned long size); void bpf_jit_free_exec(void *addr); void bpf_jit_free(struct bpf_prog *fp); struct bpf_binary_header * diff --git a/include/linux/socket.h b/include/linux/socket.h index 2a8d7b14f1d1..5a5eb1250103 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -461,6 +461,8 @@ extern struct file *__sys_socket_file(int family, int type, int protocol); extern int __sys_bind(int fd, struct sockaddr __user *umyaddr, int addrlen); extern int __sys_bind_socket(struct socket *sock, struct sockaddr_storage *address, int addrlen); +int connect_socket(struct socket *sock, struct sockaddr_storage *addr, + int addrlen, int flags); extern int __sys_connect_file(struct file *file, struct sockaddr_storage *addr, int addrlen, int file_flags); extern int __sys_connect(int fd, struct sockaddr __user *uservaddr, diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 89b36de5fdbb..732b35cc08d1 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -3038,8 +3038,34 @@ union bpf_attr { * * * **BPF_F_ADJ_ROOM_DECAP_L3_IPV4**, * **BPF_F_ADJ_ROOM_DECAP_L3_IPV6**: - * Indicate the new IP header version after decapsulating the outer - * IP header. Used when the inner and outer IP versions are different. + * Indicate the new IP header version after decapsulating the + * outer IP header. Used when the inner and outer IP versions + * are different. These flags only trigger a protocol change + * without clearing any tunnel-specific GSO flags. + * + * * **BPF_F_ADJ_ROOM_DECAP_L4_GRE**: + * Clear GRE tunnel GSO flags (SKB_GSO_GRE and SKB_GSO_GRE_CSUM) + * when decapsulating a GRE tunnel. + * + * * **BPF_F_ADJ_ROOM_DECAP_L4_UDP**: + * Clear UDP tunnel GSO flags (SKB_GSO_UDP_TUNNEL and + * SKB_GSO_UDP_TUNNEL_CSUM) when decapsulating a UDP tunnel. + * + * * **BPF_F_ADJ_ROOM_DECAP_IPXIP4**: + * Clear IPIP/SIT tunnel GSO flag (SKB_GSO_IPXIP4) when decapsulating + * a tunnel with an outer IPv4 header (IPv4-in-IPv4 or IPv6-in-IPv4). + * + * * **BPF_F_ADJ_ROOM_DECAP_IPXIP6**: + * Clear IPv6 encapsulation tunnel GSO flag (SKB_GSO_IPXIP6) when + * decapsulating a tunnel with an outer IPv6 header (IPv6-in-IPv6 + * or IPv4-in-IPv6). + * + * When using the decapsulation flags above, the skb->encapsulation + * flag is automatically cleared if all tunnel-specific GSO flags + * (SKB_GSO_UDP_TUNNEL, SKB_GSO_UDP_TUNNEL_CSUM, SKB_GSO_GRE, + * SKB_GSO_GRE_CSUM, SKB_GSO_IPXIP4, SKB_GSO_IPXIP6) have been + * removed from the packet. This handles cases where all tunnel + * layers have been decapsulated. * * A call to this helper is susceptible to change the underlying * packet buffer. Therefore, at load time, all checks on pointers @@ -3532,6 +3558,47 @@ union bpf_attr { * Use the mark present in *params*->mark for the fib lookup. * This option should not be used with BPF_FIB_LOOKUP_DIRECT, * as it only has meaning for full lookups. + * **BPF_FIB_LOOKUP_VLAN** + * If the fib lookup resolves to a VLAN device whose + * parent is a real (non-VLAN) device, set + * *params*->h_vlan_proto and *params*->h_vlan_TCI from + * the VLAN device and replace *params*->ifindex with the + * parent's ifindex. *params*->h_vlan_TCI carries the VID + * only, with PCP and DEI bits zero; a consumer wanting to + * set egress priority writes PCP itself. *params*->smac is + * the VLAN device's own address, which can differ from the + * parent's. Only the immediate parent is resolved; if it + * is itself a VLAN device (QinQ) or in another namespace, + * the egress cannot be reduced to a physical device plus + * one tag and the lookup returns + * **BPF_FIB_LKUP_RET_VLAN_FAILURE** with *params*->ifindex + * left at the input. To obtain the VLAN device's own + * ifindex, repeat the lookup without + * **BPF_FIB_LOOKUP_VLAN**, re-initializing *params* + * first: output fields overwrite the inputs they share + * storage with. The swap and the vlan fields + * are written only on success; other output fields keep + * the helper's existing behaviour, so a frag-needed result + * still reports the route mtu in *params*->mtu_result. + * This flag is only valid for XDP programs; tc programs + * receive -EINVAL since they can redirect to the VLAN + * device directly. + * **BPF_FIB_LOOKUP_VLAN_INPUT** + * Treat *params*->h_vlan_proto and *params*->h_vlan_TCI + * as an input VLAN tag and run the lookup as if ingress + * had happened on the VLAN subinterface carrying that tag + * on *params*->ifindex. The VID is the low 12 bits of + * *params*->h_vlan_TCI; *params*->h_vlan_proto must be + * ETH_P_8021Q or ETH_P_8021AD in network byte order, else + * **-EINVAL**. If *params*->ifindex is itself a VLAN + * device, its inner (QinQ) subinterface is matched; for a + * bond or team, pass the master's ifindex. An unmatched + * tag, a down device, or one in another namespace returns + * **BPF_FIB_LKUP_RET_NOT_FWDED**, mirroring real ingress. + * A VID of 0 is looked up literally, so do not set this + * flag for priority-tagged frames. Cannot be combined with + * **BPF_FIB_LOOKUP_TBID** or **BPF_FIB_LOOKUP_OUTPUT** + * (returns **-EINVAL**). * * *ctx* is either **struct xdp_md** for XDP programs or * **struct sk_buff** tc cls_act programs. @@ -4694,6 +4761,7 @@ union bpf_attr { * * **BPF_RB_RING_SIZE**: The size of ring buffer. * * **BPF_RB_CONS_POS**: Consumer position (can wrap around). * * **BPF_RB_PROD_POS**: Producer(s) position (can wrap around). + * * **BPF_RB_OVERWRITE_POS**: Overwrite position (can wrap around). * * Data returned is just a momentary snapshot of actual values * and could be inaccurate, so this facility should be used to @@ -5079,17 +5147,19 @@ union bpf_attr { * Description * Redirect the packet to another net device of index *ifindex*. * This helper is somewhat similar to **bpf_redirect**\ (), except - * that the redirection happens to the *ifindex*' peer device and - * the netns switch takes place from ingress to ingress without - * going through the CPU's backlog queue. + * that the redirection happens to the *ifindex*' peer device. If + * *flags* is 0, the netns switch takes place from ingress to + * ingress without going through the CPU's backlog queue. If the + * **BPF_F_EGRESS** flag is provided then redirection happens in + * the egress direction of the peer device. * * *skb*\ **->mark** and *skb*\ **->tstamp** are not cleared during * the netns switch. * - * The *flags* argument is reserved and must be 0. The helper is - * currently only supported for tc BPF program types at the - * ingress hook and for veth and netkit target device types. The - * peer device must reside in a different network namespace. + * If the *flags* argument is 0, the helper is currently only + * supported for tc BPF program types at the ingress hook and for + * veth and netkit target device types. The peer device must reside + * in a different network namespace. * Return * The helper returns **TC_ACT_REDIRECT** on success or * **TC_ACT_SHOT** on error. @@ -6239,7 +6309,7 @@ enum { }; /* BPF_FUNC_skb_adjust_room flags. */ -enum { +enum bpf_adj_room_flags { BPF_F_ADJ_ROOM_FIXED_GSO = (1ULL << 0), BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 = (1ULL << 1), BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 = (1ULL << 2), @@ -6249,6 +6319,10 @@ enum { BPF_F_ADJ_ROOM_ENCAP_L2_ETH = (1ULL << 6), BPF_F_ADJ_ROOM_DECAP_L3_IPV4 = (1ULL << 7), BPF_F_ADJ_ROOM_DECAP_L3_IPV6 = (1ULL << 8), + BPF_F_ADJ_ROOM_DECAP_L4_GRE = (1ULL << 9), + BPF_F_ADJ_ROOM_DECAP_L4_UDP = (1ULL << 10), + BPF_F_ADJ_ROOM_DECAP_IPXIP4 = (1ULL << 11), + BPF_F_ADJ_ROOM_DECAP_IPXIP6 = (1ULL << 12), }; enum { @@ -6336,9 +6410,10 @@ enum { /* Flags for bpf_redirect and bpf_redirect_map helpers */ enum { BPF_F_INGRESS = (1ULL << 0), /* used for skb path */ + BPF_F_EGRESS = (1ULL << 1), /* used for skb path */ BPF_F_BROADCAST = (1ULL << 3), /* used for XDP path */ BPF_F_EXCLUDE_INGRESS = (1ULL << 4), /* used for XDP path */ -#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS) +#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_EGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS) }; #define __bpf_md_ptr(type, name) \ @@ -6840,6 +6915,15 @@ struct bpf_link_info { __u32 pid; } uprobe_multi; struct { + __u32 attach_type; + __u32 count; /* in/out: tracing_multi target count */ + __u32 btf_obj_id; + __u32 :32; + __aligned_u64 ids; + __aligned_u64 addrs; + __aligned_u64 cookies; + } tracing_multi; + struct { __u32 type; /* enum bpf_perf_event_type */ __u32 :32; union { @@ -7327,6 +7411,8 @@ enum { BPF_FIB_LOOKUP_TBID = (1U << 3), BPF_FIB_LOOKUP_SRC = (1U << 4), BPF_FIB_LOOKUP_MARK = (1U << 5), + BPF_FIB_LOOKUP_VLAN = (1U << 6), + BPF_FIB_LOOKUP_VLAN_INPUT = (1U << 7), }; enum { @@ -7340,6 +7426,7 @@ enum { BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */ BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */ BPF_FIB_LKUP_RET_NO_SRC_ADDR, /* failed to derive IP src addr */ + BPF_FIB_LKUP_RET_VLAN_FAILURE, /* VLAN egress, parent unresolvable */ }; struct bpf_fib_lookup { @@ -7393,7 +7480,13 @@ struct bpf_fib_lookup { union { struct { - /* output */ + /* + * output with BPF_FIB_LOOKUP_VLAN: set from the + * resolved egress VLAN device (see the flag); zeroed + * on other successful lookups. input with + * BPF_FIB_LOOKUP_VLAN_INPUT: the VLAN tag to scope + * the lookup by. + */ __be16 h_vlan_proto; __be16 h_vlan_TCI; }; diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 4dc41bf5780c..90255d80e5be 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 80b7b8a69446..7b6847200b43 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -55,8 +55,10 @@ struct bpf_arena { struct vm_struct *kern_vm; struct page *scratch_page; struct range_tree rt; - /* protects rt */ + /* protects rt and nr_pages */ rqspinlock_t spinlock; + /* number of pages currently populated in the arena */ + u64 nr_pages; struct list_head vma_list; /* protects vma_list */ struct mutex lock; @@ -143,14 +145,14 @@ static long compute_pgoff(struct bpf_arena *arena, long uaddr) } struct apply_range_data { + struct bpf_arena *arena; struct page **pages; - struct page *scratch_page; int i; }; struct clear_range_data { + struct bpf_arena *arena; struct llist_head *free_pages; - struct page *scratch_page; }; static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) @@ -180,7 +182,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) if (pte_none(old)) continue; - if (WARN_ON_ONCE(pte_page(old) != d->scratch_page)) + if (WARN_ON_ONCE(pte_page(old) != d->arena->scratch_page)) return -EBUSY; ptep_get_and_clear(&init_mm, addr, pte); flush_tlb_before_set(addr); @@ -196,6 +198,7 @@ static int apply_range_set_cb(pte_t *pte, unsigned long addr, void *data) set_pte_at(&init_mm, addr, pte, pteval); #endif d->i++; + WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages + 1); return 0; } @@ -227,10 +230,11 @@ static int apply_range_clear_cb(pte_t *pte, unsigned long addr, void *data) * scratches its PTE. A later bpf_arena_free_pages() over that range walks * here. Without the skip, scratch_page would be freed. */ - if (page == d->scratch_page) + if (page == d->arena->scratch_page) return 0; __llist_add(&page->pcp_llist, d->free_pages); + WRITE_ONCE(d->arena->nr_pages, d->arena->nr_pages - 1); return 0; } @@ -413,7 +417,9 @@ static int arena_map_check_btf(struct bpf_map *map, const struct btf *btf, static u64 arena_map_mem_usage(const struct bpf_map *map) { - return 0; + struct bpf_arena *arena = container_of(map, struct bpf_arena, map); + + return (u64)READ_ONCE(arena->nr_pages) << PAGE_SHIFT; } struct vma_list { @@ -484,8 +490,12 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) kaddr = kbase + (u32)(vmf->address); if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) - /* Make a reasonable effort to address impossible case */ - return VM_FAULT_RETRY; + /* + * A failed lock means a possible deadlock was detected. Don't + * return VM_FAULT_RETRY: this handler never took mmap_lock, but + * the fault path would re-take it on retry and deadlock. Fail. + */ + return VM_FAULT_SIGBUS; page = vmalloc_to_page((void *)kaddr); if (page) { @@ -506,8 +516,7 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) if (ret) goto out_sigsegv_memcg; - struct apply_range_data data = { .pages = &page, .i = 0, - .scratch_page = arena->scratch_page }; + struct apply_range_data data = { .arena = arena, .pages = &page, .i = 0 }; /* Account into memcg of the process that created bpf_arena */ ret = bpf_map_alloc_pages(map, NUMA_NO_NODE, 1, &page); if (ret) { @@ -696,8 +705,8 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt bpf_map_memcg_exit(old_memcg, new_memcg); return 0; } + data.arena = arena; data.pages = pages; - data.scratch_page = arena->scratch_page; if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) goto out_free_pages; @@ -853,6 +862,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, uaddr &= PAGE_MASK; kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); if (full_uaddr >= uaddr_end) return; @@ -873,8 +884,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, range_tree_set(&arena->rt, pgoff, page_cnt); init_llist_head(&free_pages); + cdata.arena = arena; cdata.free_pages = &free_pages; - cdata.scratch_page = arena->scratch_page; /* clear ptes and collect struct pages */ apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, apply_range_clear_cb, &cdata); @@ -981,8 +992,8 @@ static void arena_free_worker(struct work_struct *work) bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); init_llist_head(&free_pages); + cdata.arena = arena; cdata.free_pages = &free_pages; - cdata.scratch_page = arena->scratch_page; arena_vm_start = bpf_arena_get_kern_vm_start(arena); user_vm_start = bpf_arena_get_user_vm_start(arena); @@ -1107,9 +1118,9 @@ __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_c __bpf_kfunc_end_defs(); BTF_KFUNCS_START(arena_kfuncs) -BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_ARENA_RET | KF_ARENA_ARG2) -BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_ARENA_ARG2) -BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_ARENA_RET | KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) BTF_KFUNCS_END(arena_kfuncs) static const struct btf_kfunc_id_set common_kfunc_set = { diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 248b4818178c..ef315b168b29 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -259,6 +259,37 @@ static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) return this_cpu_ptr(array->pptrs[index & array->index_mask]); } +static int percpu_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + + if (!bpf_jit_supports_percpu_insn()) + return -EOPNOTSUPP; + if (map->max_entries != 1) + return -EOPNOTSUPP; + if (off >= map->value_size) + return -EINVAL; + + *imm = (u64)(__force unsigned long) array->pptrs[0]; + return 0; +} + +static int percpu_array_map_direct_value_meta(const struct bpf_map *map, u64 imm, u32 *off) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + u64 base = (u64)(__force unsigned long) array->pptrs[0]; + + if (!bpf_jit_supports_percpu_insn()) + return -EOPNOTSUPP; + if (map->max_entries != 1) + return -EOPNOTSUPP; + if (imm < base || imm >= base + array->elem_size) + return -ENOENT; + + *off = imm - base; + return 0; +} + /* emit BPF instructions equivalent to C code of percpu_array_map_lookup_elem() */ static int percpu_array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) { @@ -551,9 +582,10 @@ static int array_map_check_btf(struct bpf_map *map, const struct btf_type *key_type, const struct btf_type *value_type) { - /* One exception for keyless BTF: .bss/.data/.rodata map */ + /* One exception for keyless BTF: .bss/.data/.rodata/.percpu map */ if (btf_type_is_void(key_type)) { - if (map->map_type != BPF_MAP_TYPE_ARRAY || + if ((map->map_type != BPF_MAP_TYPE_ARRAY && + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) || map->max_entries != 1) return -EINVAL; @@ -576,17 +608,42 @@ static int array_map_check_btf(struct bpf_map *map, static int array_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) { struct bpf_array *array = container_of(map, struct bpf_array, map); - pgoff_t pgoff = PAGE_ALIGN(sizeof(*array)) >> PAGE_SHIFT; if (!(map->map_flags & BPF_F_MMAPABLE)) return -EINVAL; - if (vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > + /* use u64 math so the offset cannot overflow on 32-bit archs */ + if ((u64)vma->vm_pgoff * PAGE_SIZE + (vma->vm_end - vma->vm_start) > PAGE_ALIGN((u64)array->map.max_entries * array->elem_size)) return -EINVAL; - return remap_vmalloc_range(vma, array_map_vmalloc_addr(array), - vma->vm_pgoff + pgoff); + /* + * Pages are faulted in on demand by array_map_mmap_fault(). Set the + * same flags that the eager remap_vmalloc_range() path used to set + * via vm_insert_page(), so that e.g. NUMA balancing keeps skipping + * these VMAs. + */ + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP | VM_MIXEDMAP); + + return 0; +} + +static vm_fault_t array_map_mmap_fault(struct bpf_map *map, + struct vm_fault *vmf) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + struct page *page; + + page = vmalloc_to_page(array->value + ((u64)vmf->pgoff << PAGE_SHIFT)); + if (!page) + return VM_FAULT_SIGBUS; + + /* the eager remap_vmalloc_range() flushed via vm_insert_page() */ + flush_dcache_folio(page_folio(page)); + get_page(page); + vmf->page = page; + + return 0; } static bool array_map_meta_equal(const struct bpf_map *meta0, @@ -812,6 +869,7 @@ const struct bpf_map_ops array_map_ops = { .map_direct_value_addr = array_map_direct_value_addr, .map_direct_value_meta = array_map_direct_value_meta, .map_mmap = array_map_mmap, + .map_mmap_fault = array_map_mmap_fault, .map_seq_show_elem = array_map_seq_show_elem, .map_check_btf = array_map_check_btf, .map_lookup_batch = generic_map_lookup_batch, @@ -832,6 +890,8 @@ const struct bpf_map_ops percpu_array_map_ops = { .map_get_next_key = bpf_array_get_next_key, .map_lookup_elem = percpu_array_map_lookup_elem, .map_gen_lookup = percpu_array_map_gen_lookup, + .map_direct_value_addr = percpu_array_map_direct_value_addr, + .map_direct_value_meta = percpu_array_map_direct_value_meta, .map_update_elem = array_map_update_elem, .map_delete_elem = array_map_delete_elem, .map_lookup_percpu_elem = percpu_array_map_lookup_percpu_elem, diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 2e4ae0ef0860..a2b18a9f1694 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -214,7 +214,6 @@ static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg) return bt->reg_masks[bt->frame] & (1 << reg); } - /* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */ static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask) { @@ -254,7 +253,6 @@ void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) } } - /* For given verifier state backtrack_insn() is called from the last insn to * the first insn. Its purpose is to compute a bitmask of registers and * stack slots that needs precision in the parent verifier state. @@ -285,6 +283,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, verbose(env, "stack=%s before ", env->tmp_str_buf); verbose(env, "%d: ", idx); bpf_verbose_insn(env, insn); + verbose(env, "\n"); } /* If there is a history record that some registers gained range at this insn, @@ -636,7 +635,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * r5 += 1 * ... * call bpf_perf_event_output#25 - * where .arg5_type = ARG_CONST_SIZE_OR_ZERO + * where .arg5_type = ARG_MEM_SIZE_OR_ZERO * * and this case: * r6 = 1 diff --git a/kernel/bpf/bloom_filter.c b/kernel/bpf/bloom_filter.c index b73336c976b7..c6e7559b07de 100644 --- a/kernel/bpf/bloom_filter.c +++ b/kernel/bpf/bloom_filter.c @@ -41,7 +41,7 @@ static long bloom_map_peek_elem(struct bpf_map *map, void *value) for (i = 0; i < bloom->nr_hash_funcs; i++) { h = hash(bloom, value, map->value_size, i); - if (!test_bit(h, bloom->bitset)) + if (!test_bit(h % BITS_PER_LONG, bloom->bitset + BIT_WORD(h))) return -ENOENT; } @@ -57,9 +57,13 @@ static long bloom_map_push_elem(struct bpf_map *map, void *value, u64 flags) if (flags != BPF_ANY) return -EINVAL; + /* + * On 32-bit architectures, hashes larger than INT_MAX would be + * treated as negative by set_bit(). + */ for (i = 0; i < bloom->nr_hash_funcs; i++) { h = hash(bloom, value, map->value_size, i); - set_bit(h, bloom->bitset); + set_bit(h % BITS_PER_LONG, bloom->bitset + BIT_WORD(h)); } return 0; @@ -94,9 +98,10 @@ static int bloom_map_alloc_check(union bpf_attr *attr) static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) { - u32 bitset_bytes, bitset_mask, nr_hash_funcs, nr_bits; + u32 bitset_mask, nr_hash_funcs, nr_bits; int numa_node = bpf_map_attr_numa_node(attr); struct bpf_bloom_filter *bloom; + u64 bitset_bytes; if (attr->key_size != 0 || attr->value_size == 0 || attr->max_entries == 0 || @@ -127,22 +132,16 @@ static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) if (check_mul_overflow(attr->max_entries, nr_hash_funcs, &nr_bits) || check_mul_overflow(nr_bits / 5, (u32)7, &nr_bits) || nr_bits > (1UL << 31)) { - /* The bit array size is 2^32 bits but to avoid overflowing the - * u32, we use U32_MAX, which will round up to the equivalent - * number of bytes - */ - bitset_bytes = BITS_TO_BYTES(U32_MAX); bitset_mask = U32_MAX; } else { if (nr_bits <= BITS_PER_LONG) nr_bits = BITS_PER_LONG; else nr_bits = roundup_pow_of_two(nr_bits); - bitset_bytes = BITS_TO_BYTES(nr_bits); bitset_mask = nr_bits - 1; } - bitset_bytes = roundup(bitset_bytes, sizeof(unsigned long)); + bitset_bytes = BITS_TO_LONGS((u64)bitset_mask + 1) * sizeof(unsigned long); bloom = bpf_map_area_alloc(sizeof(*bloom) + bitset_bytes, numa_node); if (!bloom) diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index f5eaeb2493d4..14a5fdfa0421 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -782,8 +782,8 @@ __bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) return -EINVAL; } - /* avoid overflows, e.g., if start == INT_MIN and end == INT_MAX */ - if ((s64)end - (s64)start > BPF_MAX_LOOPS) { + /* start <= end here, so end - start fits in a u32 without overflow */ + if ((u32)(end - start) > BPF_MAX_LOOPS) { s->cur = s->end = 0; return -E2BIG; } @@ -802,12 +802,11 @@ __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) { struct bpf_iter_num_kern *s = (void *)it; - /* check failed initialization or if we are done (same behavior); - * need to be careful about overflow, so convert to s64 for checks, - * e.g., if s->cur == s->end == INT_MAX, we can't just do - * s->cur + 1 >= s->end + /* + * s->cur < s->end while iterating, else s->cur == s->end == 0; the signed + * s->cur + 1 >= s->end holds even when s->cur + 1 wraps (start == INT_MIN). */ - if ((s64)(s->cur + 1) >= s->end) { + if (s->cur + 1 >= s->end) { s->cur = s->end = 0; return NULL; } @@ -819,9 +818,7 @@ __bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it) __bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it) { - struct bpf_iter_num_kern *s = (void *)it; - - s->cur = s->end = 0; + /* no-op */ } __bpf_kfunc_end_defs(); diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 1433809bb166..82c5988417a0 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -186,7 +186,7 @@ static const struct bpf_func_proto bpf_ima_inode_hash_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_ima_inode_hash_btf_ids[0], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .allowed = bpf_ima_inode_hash_allowed, }; @@ -205,7 +205,7 @@ static const struct bpf_func_proto bpf_ima_file_hash_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_ima_file_hash_btf_ids[0], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .allowed = bpf_ima_inode_hash_allowed, }; @@ -295,7 +295,6 @@ BTF_ID(func, bpf_lsm_bpf_map_create) BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) -BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_bpf_token_create) BTF_ID(func, bpf_lsm_bpf_token_free) BTF_ID(func, bpf_lsm_bpf_token_cmd) diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 51b16e5f5534..d7c3030bc63b 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -147,6 +147,8 @@ void bpf_struct_ops_image_free(void *image) #define MAYBE_NULL_SUFFIX "__nullable" #define REFCOUNTED_SUFFIX "__ref" +#define ARENA_SUFFIX "__arena" +#define ARENA_MAYBE_NULL_SUFFIX "__arena__nullable" /* Prepare argument info for every nullable argument of a member of a * struct_ops type. @@ -159,7 +161,7 @@ void bpf_struct_ops_image_free(void *image) * to provide an array of struct bpf_ctx_arg_aux, which in turn provides * the information that used by the verifier to check the arguments of the * BPF struct_ops program assigned to the member. Here, we only care about - * the arguments that are marked as __nullable. + * the arguments that are marked as __nullable, __ref or __arena. * * The array of struct bpf_ctx_arg_aux is eventually assigned to * prog->aux->ctx_arg_info of BPF struct_ops programs and passed to the @@ -172,10 +174,12 @@ static int prepare_arg_info(struct btf *btf, const char *st_ops_name, const char *member_name, const struct btf_type *func_proto, void *stub_func_addr, + struct btf_func_model *model, struct bpf_struct_ops_arg_info *arg_info) { const struct btf_type *stub_func_proto, *pointed_type; - bool is_nullable = false, is_refcounted = false; + bool is_nullable = false, is_refcounted = false, is_arena = false; + bool is_arena_nullable = false; const struct btf_param *stub_args, *args; struct bpf_ctx_arg_aux *info, *info_buf; u32 nargs, arg_no, info_cnt = 0; @@ -225,27 +229,39 @@ static int prepare_arg_info(struct btf *btf, /* Prepare info for every nullable argument */ info = info_buf; for (arg_no = 0; arg_no < nargs; arg_no++) { - /* Skip arguments that is not suffixed with - * "__nullable or __ref". + bool ptr_to_arena, ptr_to_struct; + + /* + * Skip arguments that are not suffixed with "__arena__nullable", + * "__arena", "__nullable", or "__ref". */ - is_nullable = btf_param_match_suffix(btf, &stub_args[arg_no], - MAYBE_NULL_SUFFIX); + is_arena_nullable = btf_param_match_suffix(btf, &stub_args[arg_no], + ARENA_MAYBE_NULL_SUFFIX); + is_arena = btf_param_match_suffix(btf, &stub_args[arg_no], ARENA_SUFFIX); + is_nullable = !is_arena_nullable && + btf_param_match_suffix(btf, &stub_args[arg_no], MAYBE_NULL_SUFFIX); is_refcounted = btf_param_match_suffix(btf, &stub_args[arg_no], REFCOUNTED_SUFFIX); - if (is_nullable) + if (is_arena_nullable) + suffix = ARENA_MAYBE_NULL_SUFFIX; + else if (is_arena) + suffix = ARENA_SUFFIX; + else if (is_nullable) suffix = MAYBE_NULL_SUFFIX; else if (is_refcounted) suffix = REFCOUNTED_SUFFIX; else continue; - /* Should be a pointer to struct */ - pointed_type = btf_type_resolve_ptr(btf, - args[arg_no].type, - &arg_btf_id); - if (!pointed_type || - !btf_type_is_struct(pointed_type)) { + /* + * Should be a pointer to struct, or any pointer for __arena or + * __arena__nullable. + */ + pointed_type = btf_type_resolve_ptr(btf, args[arg_no].type, &arg_btf_id); + ptr_to_arena = pointed_type && (is_arena || is_arena_nullable); + ptr_to_struct = pointed_type && btf_type_is_struct(pointed_type); + if (!ptr_to_arena && !ptr_to_struct) { pr_warn("stub function %s has %s tagging to an unsupported type\n", stub_fname, suffix); goto err_out; @@ -268,7 +284,18 @@ static int prepare_arg_info(struct btf *btf, info->btf_id = arg_btf_id; info->btf = btf; info->offset = offset; - if (is_nullable) { + if (is_arena || is_arena_nullable) { + /* + * Both types get PTR_TO_ARENA. In verifier state, + * PTR_TO_ARENA encompasses potential NULL values, but + * we do not force the program to check it, or maintain + * precision around it, since it has no safety implication. + */ + info->reg_type = PTR_TO_ARENA; + model->arg_flags[arg_no] |= BTF_FMODEL_ARENA_ARG; + if (is_arena_nullable) + model->arg_flags[arg_no] |= BTF_FMODEL_NULLABLE_ARG; + } else if (is_nullable) { info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID | PTR_MAYBE_NULL; } else if (is_refcounted) { info->reg_type = PTR_TRUSTED | PTR_TO_BTF_ID; @@ -445,9 +472,22 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, goto errout; } + /* + * A >8 byte return value is passed back in a register pair, + * which the struct_ops trampoline does not preserve (only + * 8 bytes of the return value are saved and restored). + */ + if (st_ops->func_models[i].ret_size > 8) { + pr_warn("func ptr %s in struct %s has a >8 byte return value, which is not supported\n", + mname, st_ops->name); + err = -EOPNOTSUPP; + goto errout; + } + stub_func_addr = *(void **)(st_ops->cfi_stubs + moff); err = prepare_arg_info(btf, st_ops->name, mname, func_proto, stub_func_addr, + &st_ops->func_models[i], arg_info + i); if (err) goto errout; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index f56437e626c9..da36d4b9d31a 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1169,19 +1169,19 @@ static const char *btf_show_name(struct btf_show *show) id = t->type; break; default: - id = 0; - break; + goto resolved; } + t = btf_type_skip_qualifiers(show->btf, id); if (!id) break; - t = btf_type_skip_qualifiers(show->btf, id); } /* We may not be able to represent this type; bail to be safe */ if (i == BTF_SHOW_MAX_ITER) return ""; +resolved: if (!name) - name = btf_name_by_offset(show->btf, t->name_off); + name = btf_type_is_void(t) ? "void" : btf_name_by_offset(show->btf, t->name_off); switch (BTF_INFO_KIND(t->info)) { case BTF_KIND_STRUCT: @@ -2534,7 +2534,6 @@ static void btf_bitfield_show(void *data, u8 bits_offset, btf_int128_print(show, print_num); } - static void btf_int_bits_show(const struct btf *btf, const struct btf_type *t, void *data, u8 bits_offset, @@ -3669,7 +3668,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, { BPF_RB_NODE, "bpf_rb_node", false }, - { BPF_REFCOUNT, "bpf_refcount", false }, + { BPF_REFCOUNT, "bpf_refcount", true }, }; int type = 0, i; const char *name = __btf_name_by_offset(btf, var_type->name_off); @@ -3751,7 +3750,7 @@ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level); + u32 level, u32 *seen_mask); /* Find special fields in the struct type of a field. * @@ -3762,7 +3761,7 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_nested_struct(const struct btf *btf, const struct btf_type *t, u32 off, u32 nelems, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, err, i; @@ -3770,7 +3769,7 @@ static int btf_find_nested_struct(const struct btf *btf, const struct btf_type * if (level >= MAX_RESOLVE_DEPTH) return -E2BIG; - ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level); + ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level, seen_mask); if (ret <= 0) return ret; @@ -3827,7 +3826,7 @@ static int btf_find_field_one(const struct btf *btf, if (expected_size && expected_size != sz * nelems) return 0; ret = btf_find_nested_struct(btf, var_type, off, nelems, field_mask, - &info[0], info_cnt, level); + &info[0], info_cnt, level, seen_mask); return ret; } @@ -3892,11 +3891,11 @@ static int btf_find_field_one(const struct btf *btf, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level) + u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_member *member; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_member(i, t, member) { const struct btf_type *member_type = btf_type_by_id(btf, @@ -3910,7 +3909,7 @@ static int btf_find_struct_field(const struct btf *btf, ret = btf_find_field_one(btf, t, member_type, i, off, 0, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) return ret; @@ -3921,11 +3920,11 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_var_secinfo *vsi; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_vsi(i, t, vsi) { const struct btf_type *var = btf_type_by_id(btf, vsi->type); @@ -3933,7 +3932,7 @@ static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, off = vsi->offset; ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) @@ -3947,10 +3946,12 @@ static int btf_find_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt) { + u32 seen_mask = 0; + if (__btf_type_is_struct(t)) - return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0); + return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0, &seen_mask); else if (btf_type_is_datasec(t)) - return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0); + return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0, &seen_mask); return -EINVAL; } @@ -4168,7 +4169,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type rec->spin_lock_off = rec->fields[i].offset; break; case BPF_RES_SPIN_LOCK: - WARN_ON_ONCE(rec->spin_lock_off >= 0); + WARN_ON_ONCE(rec->res_spin_lock_off >= 0); /* Cache offset for faster lookup at runtime */ rec->res_spin_lock_off = rec->fields[i].offset; break; @@ -6451,7 +6452,7 @@ struct btf *btf_parse_vmlinux(void) if (IS_ERR(btf)) goto err_out; - /* btf_parse_vmlinux() runs under bpf_verifier_lock */ + /* btf_parse_vmlinux() runs under btf_vmlinux_lock */ bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]); err = btf_alloc_id(btf); if (err) { @@ -6954,15 +6955,19 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, return false; } - /* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */ + /* + * Check for PTR_TO_RDONLY_BUF_OR_NULL, PTR_TO_RDWR_BUF_OR_NULL or + * PTR_TO_ARENA (both nullable and non-nullable cases). + */ for (i = 0; i < prog->aux->ctx_arg_info_size; i++) { const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i]; u32 type, flag; type = base_type(ctx_arg_info->reg_type); flag = type_flag(ctx_arg_info->reg_type); - if (ctx_arg_info->offset == off && type == PTR_TO_BUF && - (flag & PTR_MAYBE_NULL)) { + if (ctx_arg_info->offset == off && + (type == PTR_TO_ARENA || + (type == PTR_TO_BUF && (flag & PTR_MAYBE_NULL)))) { info->reg_type = ctx_arg_info->reg_type; return true; } @@ -7101,7 +7106,7 @@ enum bpf_struct_walk_result { static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, const struct btf_type *t, int off, int size, u32 *next_btf_id, enum bpf_type_flag *flag, - const char **field_name) + const char **field_name, bool walk_flex_arrays) { u32 i, moff, mtrue_end, msize = 0, total_nelems = 0; const struct btf_type *mtype, *elem_type = NULL; @@ -7128,11 +7133,14 @@ again: *flag |= PTR_UNTRUSTED; if (off + size > t->size) { + struct btf_array *array_elem; + + if (!walk_flex_arrays) + goto error; + /* If the last element is a variable size array, we may * need to relax the rule. */ - struct btf_array *array_elem; - if (vlen == 0) goto error; @@ -7397,7 +7405,8 @@ int btf_struct_access(struct bpf_verifier_log *log, t = btf_type_by_id(btf, id); do { - err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, field_name); + err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, + field_name, !type_is_alloc(reg->type)); switch (err) { case WALK_PTR: @@ -7456,7 +7465,7 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict) + bool strict, bool walk_flex_arrays) { const struct btf_type *type; enum bpf_type_flag flag = 0; @@ -7475,7 +7484,8 @@ again: type = btf_type_by_id(btf, id); if (!type) return false; - err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL); + err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL, + walk_flex_arrays); if (err != WALK_STRUCT) return false; @@ -7519,14 +7529,28 @@ static u8 __get_type_fmodel_flags(const struct btf_type *t) { u8 flags = 0; - if (btf_type_is_struct(t)) - flags |= BTF_FMODEL_STRUCT_ARG; if (btf_type_is_signed_int(t)) flags |= BTF_FMODEL_SIGNED_ARG; return flags; } +static u8 __get_arg_fmodel_flags(const struct btf *btf, + const struct btf_param *arg, + const struct btf_type *t) +{ + u8 flags = __get_type_fmodel_flags(t); + + if (btf_param_match_suffix(btf, arg, "__arena__nullable")) + flags |= BTF_FMODEL_ARENA_ARG | BTF_FMODEL_NULLABLE_ARG; + else if (btf_param_match_suffix(btf, arg, "__arena")) + flags |= BTF_FMODEL_ARENA_ARG; + else if (btf_param_match_suffix(btf, arg, "__nullable")) + flags |= BTF_FMODEL_NULLABLE_ARG; + + return flags; +} + int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf *btf, const struct btf_type *func, @@ -7592,7 +7616,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } m->arg_size[i] = ret; - m->arg_flags[i] = __get_type_fmodel_flags(t); + m->arg_flags[i] = __get_arg_fmodel_flags(btf, &args[i], t); } m->nr_args = nargs; return 0; @@ -8285,6 +8309,16 @@ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, return ssnprintf.len; } +int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len) +{ + struct btf_show show = { + .btf = btf, + .state.type_id = type_id, + }; + + return snprintf(buf, len, "%s", btf_show_name(&show)); +} + #ifdef CONFIG_PROC_FS static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) { @@ -8695,7 +8729,7 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 26d37066465f..0f13c13f4133 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -5,6 +5,8 @@ #include <linux/filter.h> #include <linux/sort.h> +#include "diagnostics.h" + #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) /* non-recursive DFS pseudo code @@ -47,7 +49,6 @@ enum { BRANCH = 2, }; - static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off) { struct bpf_subprog_info *subprog; @@ -113,6 +114,10 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) if (w < 0 || w >= env->prog->len) { verbose_linfo(env, t, "%d: ", t); verbose(env, "jump out of range from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "jump out of range", "Keep branch targets inside the program.", + "Instruction %d jumps to instruction %d, but the program only contains instructions 0 through %d.", + t, w, env->prog->len - 1); return -EINVAL; } @@ -136,6 +141,11 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) verbose_linfo(env, t, "%d: ", t); verbose_linfo(env, w, "%d: ", w); verbose(env, "back-edge from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "back-edge is not allowed", + "Load with privileges that allow this back-edge, or rewrite the control flow so it does not branch backward.", + "Instruction %d branches back to instruction %d. This program is being rejected without the privilege needed for this back-edge.", + t, w); return -EINVAL; } else if (insn_state[w] == EXPLORED) { /* forward- or cross-edge */ @@ -316,6 +326,11 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, if (!jt) { verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); + bpf_diag_program_structure( + env, subprog_start, "missing jump table", + "Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.", + "No jump table was found for the subprogram that starts at instruction %u.", + subprog_start); return ERR_PTR(-EINVAL); } @@ -343,6 +358,11 @@ create_jt(int t, struct bpf_verifier_env *env) if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) { verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n", t, subprog_start, subprog_end); + bpf_diag_program_structure( + env, t, "jump table target out of range", + "Keep every jump-table target inside the same subprogram.", + "The jump table for instruction %d points outside subprogram range [%u,%u).", + t, subprog_start, subprog_end); kvfree(jt); return ERR_PTR(-EINVAL); } @@ -374,6 +394,11 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) w = jt->items[i]; if (w < 0 || w >= env->prog->len) { verbose(env, "indirect jump out of range from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "indirect jump out of range", + "Keep indirect jump targets inside the program.", + "Instruction %d can jump indirectly to instruction %d, but the program only contains instructions 0 through %d.", + t, w, env->prog->len - 1); return -EINVAL; } @@ -491,7 +516,7 @@ static int visit_insn(int t, struct bpf_verifier_env *env) return ret; } } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; ret = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta); if (ret == 0 && bpf_is_iter_next_kfunc(&meta)) { @@ -624,12 +649,21 @@ walk_cfg: if (insn_state[i] != EXPLORED) { verbose(env, "unreachable insn %d\n", i); + bpf_diag_program_structure( + env, i, "unreachable instruction", + "Remove the unreachable instruction or add valid control flow that reaches it.", + "Instruction %d is not reachable from the program entry point.", i); ret = -EINVAL; goto err_free; } if (bpf_is_ldimm64(insn)) { if (insn_state[i + 1] != 0) { verbose(env, "jump into the middle of ldimm64 insn %d\n", i); + bpf_diag_program_structure( + env, i, "jump into ldimm64 immediate", + "Target the first instruction of the ldimm64 pair, or restructure the jump target.", + "Control flow reaches the second half of the ldimm64 instruction pair that starts at instruction %d.", + i); ret = -EINVAL; goto err_free; } diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 4355ccb78a9c..149672c76c49 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -813,8 +813,10 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *old_prog = NULL; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; + struct bpf_cgroup_storage *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_prog *new_prog = prog ? : link->link.prog; enum cgroup_bpf_attach_type atype; + u32 old_flags, old_pl_flags; struct bpf_prog_list *pl; struct hlist_head *progs; int err; @@ -865,6 +867,8 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, if (pl) { old_prog = pl->prog; + old_pl_flags = pl->flags; + bpf_cgroup_storages_assign(old_storage, pl->storage); } else { pl = kmalloc_obj(*pl); if (!pl) { @@ -884,6 +888,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, pl->link = link; pl->flags = flags; bpf_cgroup_storages_assign(pl->storage, storage); + old_flags = cgrp->bpf.flags[atype]; cgrp->bpf.flags[atype] = saved_flags; if (type == BPF_LSM_CGROUP) { @@ -915,12 +920,15 @@ cleanup: if (old_prog) { pl->prog = old_prog; pl->link = NULL; + pl->flags = old_pl_flags; + bpf_cgroup_storages_assign(pl->storage, old_storage); } bpf_cgroup_storages_free(new_storage); if (!old_prog) { hlist_del(&pl->node); kfree(pl); } + cgrp->bpf.flags[atype] = old_flags; return err; } @@ -1018,6 +1026,20 @@ static void replace_effective_prog(struct cgroup *cgrp, } } +static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog, + struct bpf_prog *new_prog) +{ + enum bpf_cgroup_storage_type stype; + + for_each_cgroup_storage_type(stype) { + if (old_prog->aux->cgroup_storage[stype] != + new_prog->aux->cgroup_storage[stype]) + return false; + } + + return true; +} + /** * __cgroup_bpf_replace() - Replace link's program and propagate the change * to descendants @@ -1056,6 +1078,9 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, if (!found) return -ENOENT; + if (!cgroup_bpf_storages_compatible(link->link.prog, new_prog)) + return -EINVAL; + cgrp->bpf.revisions[atype] += 1; old_prog = xchg(&link->link.prog, new_prog); replace_effective_prog(cgrp, atype, pl); @@ -2235,7 +2260,7 @@ int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, if (ret < 0) return ret; - if (ctx.optlen > *optlen) + if (ctx.optlen > *optlen || ctx.optlen < 0) return -EFAULT; /* BPF programs can shrink the buffer, export the modifications. @@ -2305,7 +2330,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -2347,7 +2372,7 @@ static const struct bpf_func_proto bpf_sysctl_get_current_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sysctl_get_new_value, struct bpf_sysctl_kern *, ctx, char *, buf, @@ -2367,7 +2392,7 @@ static const struct bpf_func_proto bpf_sysctl_get_new_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx, @@ -2393,7 +2418,7 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto * diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c index 93bebe6fe12e..0e8b3ccc7a5b 100644 --- a/kernel/bpf/check_btf.c +++ b/kernel/bpf/check_btf.c @@ -28,9 +28,9 @@ static int check_abnormal_return(struct bpf_verifier_env *env) #define MIN_BPF_FUNCINFO_SIZE 8 #define MAX_FUNCINFO_REC_SIZE 252 -static int check_btf_func_early(struct bpf_verifier_env *env, - const union bpf_attr *attr, - bpfptr_t uattr) +static int prepare_btf_func(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) { u32 krec_size = sizeof(struct bpf_func_info); const struct btf_type *type, *func_proto; @@ -407,9 +407,9 @@ static int check_core_relo(struct bpf_verifier_env *env, return err; } -int bpf_check_btf_info_early(struct bpf_verifier_env *env, - const union bpf_attr *attr, - bpfptr_t uattr) +int bpf_prepare_btf_info(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) { struct btf *btf; int err; @@ -429,7 +429,7 @@ int bpf_check_btf_info_early(struct bpf_verifier_env *env, } env->prog->aux->btf = btf; - err = check_btf_func_early(env, attr, uattr); + err = prepare_btf_func(env, attr, uattr); if (err) return err; return 0; diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index b2a19acadb91..7f1b30059cc8 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -182,7 +182,6 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * u64 val = 0; if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || - map->map_type == BPF_MAP_TYPE_INSN_ARRAY || off < 0 || off + size > map->value_size || bpf_map_direct_read(map, off, size, &val, is_ldsx)) { *dst = unknown; @@ -199,14 +198,9 @@ process_call: ci_out[r] = unknown; break; case BPF_STX: - if (mode != BPF_ATOMIC) - break; - if (insn->imm == BPF_CMPXCHG) - ci_out[BPF_REG_0] = unknown; - else if (insn->imm == BPF_LOAD_ACQ) - *dst = unknown; - else if (insn->imm & BPF_FETCH) - *src = unknown; + r = bpf_atomic_load_reg(insn); + if (r >= 0) + ci_out[r] = unknown; break; } } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6e19a030da6f..d55e737ed75a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -126,6 +126,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag fp->aux->main_prog_aux = aux; fp->aux->prog = fp; fp->jit_requested = ebpf_jit_enabled(); + fp->jit_required = IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON); fp->blinding_requested = bpf_jit_blinding_enabled(fp); #ifdef CONFIG_CGROUP_BPF aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID; @@ -305,7 +306,7 @@ int bpf_prog_calc_tag(struct bpf_prog *fp) bool was_ld_map; u32 i; - dst = vmalloc(size); + dst = __vmalloc(size, GFP_KERNEL_ACCOUNT); if (!dst) return -ENOMEM; @@ -915,6 +916,11 @@ static LIST_HEAD(pack_list); #define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE) +static bool bpf_jit_mem_is_rox(void) +{ + return execmem_is_rox(EXECMEM_BPF); +} + static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns) { struct bpf_prog_pack *pack; @@ -926,16 +932,18 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins pack->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE); if (!pack->ptr) goto out; - bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE); if (static_branch_unlikely(&bpf_pred_flush_enabled)) pack->arch_flush_needed = true; - set_vm_flush_reset_perms(pack->ptr); - err = set_memory_rox((unsigned long)pack->ptr, - BPF_PROG_PACK_SIZE / PAGE_SIZE); - if (err) - goto out; + if (!bpf_jit_mem_is_rox()) { + bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE); + set_vm_flush_reset_perms(pack->ptr); + err = set_memory_rox((unsigned long)pack->ptr, + BPF_PROG_PACK_SIZE / PAGE_SIZE); + if (err) + goto out; + } list_add_tail(&pack->list, &pack_list); return pack; @@ -964,7 +972,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); - if (ptr) { + if (ptr && !bpf_jit_mem_is_rox()) { int err; bpf_fill_ill_insns(ptr, size); @@ -1115,12 +1123,17 @@ void bpf_jit_uncharge_modmem(u32 size) atomic_long_sub(size, &bpf_jit_current); } -void *__weak bpf_jit_alloc_exec(unsigned long size) +void *bpf_jit_alloc_exec(unsigned long size) { return execmem_alloc(EXECMEM_BPF, size); } -void __weak bpf_jit_free_exec(void *addr) +void *bpf_jit_alloc_exec_rw(unsigned long size) +{ + return execmem_alloc_rw(EXECMEM_BPF, size); +} + +void bpf_jit_free_exec(void *addr) { execmem_free(addr); } @@ -2621,22 +2634,10 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc { #ifdef CONFIG_BPF_JIT struct bpf_prog *orig_prog; - struct bpf_insn_aux_data *orig_insn_aux; if (!bpf_prog_need_blind(prog)) return bpf_int_jit_compile(env, prog); - if (env) { - /* - * If env is not NULL, we are called from the end of bpf_check(), at this - * point, only insn_aux_data is used after failure, so it should be restored - * on failure. - */ - orig_insn_aux = bpf_dup_insn_aux_data(env); - if (!orig_insn_aux) - return prog; - } - orig_prog = prog; prog = bpf_jit_blind_constants(env, prog); /* @@ -2649,8 +2650,6 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc prog = bpf_int_jit_compile(env, prog); if (prog->jited) { bpf_jit_prog_release_other(prog, orig_prog); - if (env) - vfree(orig_insn_aux); return prog; } @@ -2658,8 +2657,6 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc out_restore: prog = orig_prog; - if (env) - bpf_restore_insn_aux_data(env, orig_insn_aux); #endif return prog; } @@ -2670,15 +2667,11 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct /* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ - bool jit_needed = false; + bool jit_needed = fp->jit_required; if (fp->bpf_func) goto finalize; - if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) || - bpf_prog_has_kfunc_call(fp)) - jit_needed = true; - if (!bpf_prog_select_interpreter(fp)) jit_needed = true; @@ -3299,6 +3292,11 @@ bool __weak bpf_jit_supports_stack_args(void) return false; } +bool __weak bpf_jit_supports_arena_args(void) +{ + return false; +} + bool __weak bpf_jit_supports_far_kfunc_call(void) { return false; @@ -3463,24 +3461,14 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); #ifdef CONFIG_BPF_SYSCALL -void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo, - const char **filep, const char **linep, int *nump) +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src) { - /* Get base component of the file path. */ - if (filep) { - *filep = btf_name_by_offset(btf, linfo->file_name_off); - *filep = kbasename(*filep); - } - - /* Obtain the source line, and strip whitespace in prefix. */ - if (linep) { - *linep = btf_name_by_offset(btf, linfo->line_off); - while (isspace(**linep)) - *linep += 1; - } - - if (nump) - *nump = BPF_LINE_INFO_LINE_NUM(linfo->line_col); + src->file = kbasename(btf_name_by_offset(btf, linfo->file_name_off)); + src->line = btf_name_by_offset(btf, linfo->line_off); + src->file_name_off = linfo->file_name_off; + src->line_num = BPF_LINE_INFO_LINE_NUM(linfo->line_col); + src->line_col = BPF_LINE_INFO_LINE_COL(linfo->line_col); } const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off) @@ -3523,6 +3511,7 @@ const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump) { + struct bpf_linfo_source src; int idx = -1, insn_start, insn_end, len; struct bpf_line_info *linfo; void **jited_linfo; @@ -3554,7 +3543,15 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * if (idx == -1) return -ENOENT; - bpf_get_linfo_file_line(btf, &linfo[idx], filep, linep, nump); + bpf_get_linfo_source(btf, &linfo[idx], &src); + while (isspace(*src.line)) + src.line++; + if (filep) + *filep = src.file; + if (linep) + *linep = src.line; + if (nump) + *nump = src.line_num; return 0; } diff --git a/kernel/bpf/cpumask.c b/kernel/bpf/cpumask.c index b8c805b4b06a..1336a4efa755 100644 --- a/kernel/bpf/cpumask.c +++ b/kernel/bpf/cpumask.c @@ -449,12 +449,12 @@ __bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask) * @src__sz: Length of the BPF memory region in bytes. * * Return: - * * 0 if the struct cpumask * instance was populated successfully. + * * 0 if the struct bpf_cpumask * instance was populated successfully. * * -EACCES if the memory region is too small to populate the cpumask. * * -EINVAL if the memory region is not aligned to the size of a long * and the architecture does not support efficient unaligned accesses. */ -__bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) +__bpf_kfunc int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) { unsigned long source = (unsigned long)src; @@ -467,7 +467,7 @@ __bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t !IS_ALIGNED(source, sizeof(long))) return -EINVAL; - bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids); + bitmap_copy(cpumask_bits(&cpumask->cpumask), src, nr_cpu_ids); return 0; } diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c new file mode 100644 index 000000000000..b682fd2be443 --- /dev/null +++ b/kernel/bpf/diagnostics.c @@ -0,0 +1,2359 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + +#include <linux/bpf.h> +#include <linux/bpf_verifier.h> +#include <linux/btf.h> +#include <linux/ctype.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/seq_buf.h> +#include <linux/overflow.h> +#include <linux/slab.h> +#include <linux/stdarg.h> +#include <linux/string.h> + +#include "disasm.h" +#include "diagnostics.h" + +#define REGISTER_TYPE_SAFETY "Register Type Safety" +#define MEMORY_SAFETY "Memory Safety" +#define RESOURCE_LIFETIME_SAFETY "Resource Lifetime Safety" +#define CALL_TYPE_SAFETY "Call Type Safety" +#define EXECUTION_CONTEXT_SAFETY "Execution Context Safety" +#define PROGRAM_STRUCTURE "Program Structure" +#define POLICY "Policy" + +#define BPF_DIAG_TEXT_WIDTH 100 +#define BPF_DIAG_TEXT_INDENT " " +#define BPF_DIAG_CONTEXT 2 +#define BPF_DIAG_CONTEXT_CNT (1 + BPF_DIAG_CONTEXT * 2) +#define BPF_DIAG_HISTORY_RENDER_MAX 64 +#define BPF_DIAG_SOURCE_LANE_WIDTH 88 +#define BPF_DIAG_TAB_WIDTH 8 +#define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk)) +#define BPF_DIAG_FMT_BUF_SIZE 256 +#define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20) +#define DISASM_LINE_LEN 160 + +enum bpf_diag_mod_target_kind { + BPF_DIAG_MOD_TARGET_NONE, + BPF_DIAG_MOD_TARGET_REG, + BPF_DIAG_MOD_TARGET_STACK_ARG, + BPF_DIAG_MOD_TARGET_STACK_SLOT, + BPF_DIAG_MOD_TARGET_STACK_RANGE, +}; + +struct bpf_diag_mod_target { + u32 frame_id; + union { + struct { + s16 min_off; + s16 max_off; + } range; + u16 spi; + u8 regno; + u8 stack_arg; + }; + u8 frameno; + u8 kind; +}; + +static struct bpf_diag_mod_target diag_reg_target(u32 frame_id, u8 frameno, u8 regno) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_REG, + .regno = regno, + }; +} + +static struct bpf_diag_mod_target diag_stack_arg_target(u32 frame_id, u8 frameno, u8 slot) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_ARG, + .stack_arg = slot, + }; +} + +static struct bpf_diag_mod_target diag_stack_slot_target(u32 frame_id, u8 frameno, u16 spi) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_SLOT, + .spi = spi, + }; +} + +static struct bpf_diag_mod_target diag_stack_range_target(u32 frame_id, u8 frameno, + s16 min_off, s16 max_off) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_RANGE, + .range.min_off = min_off, + .range.max_off = max_off, + }; +} + +struct bpf_diag_reg_snapshot { + u32 type; + u32 btf_id; + const struct bpf_map *map_ptr; + const struct btf *btf; + struct tnum var_off; + struct cnum64 r64; +}; + +enum bpf_diag_history_kind { + BPF_DIAG_HISTORY_BRANCH, + BPF_DIAG_HISTORY_MOD, + BPF_DIAG_HISTORY_REF_ACQUIRE, + BPF_DIAG_HISTORY_REF_RELEASE, + BPF_DIAG_HISTORY_CONTEXT, +}; + +struct bpf_diag_history_event { + u32 insn_idx : 24; + u32 kind : 8; + u8 in_lineage : 1; + union { + struct { + bool cond_true; + } branch; + struct { + struct bpf_diag_mod_target target; + struct bpf_diag_mod_target origin; + struct bpf_diag_reg_snapshot old, new; + u8 reason; + bool origin_valid; + } mod; + struct { + u32 ref_id; + } ref; + struct { + u32 depth; + u8 kind; + bool enter; + } ctx; + }; +}; + +enum bpf_diag_history_scope { + BPF_DIAG_HISTORY_SCOPE_REG, + BPF_DIAG_HISTORY_SCOPE_STACK_ARG, + BPF_DIAG_HISTORY_SCOPE_REF, + BPF_DIAG_HISTORY_SCOPE_CONTEXT, +}; + +struct bpf_diag_history_opts { + enum bpf_diag_history_scope scope; + u32 frame_id; + u32 frameno; + int regno; + int stack_arg_slot; + u32 ref_id; + enum bpf_diag_context_kind ctx_kind; + u32 ctx_depth; +}; + +static void diag_print_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_opts *opts); +static bool diag_target_matches(const struct bpf_diag_mod_target *event_target, + const struct bpf_diag_mod_target *target); +static const char *diag_context_name(enum bpf_diag_context_kind kind); +struct disasm_line { + char text[DISASM_LINE_LEN]; + int idx; + bool valid; +}; + +struct disasm_ctx { + struct bpf_verifier_env *env; + struct seq_buf seq; +}; + +struct diag_fmt_chunk { + struct list_head node; + struct seq_buf seq; + char data[]; +}; + +struct diag_fmt_mark { + struct diag_fmt_chunk *chunk; + size_t len; +}; + +struct bpf_diag_log { + struct bpf_diag_history_event *events; + /* Sequence number of the oldest retained event on the active path. */ + u64 first_seq; + u32 cnt; + u32 cap; + u32 head; + bool growth_failed; +}; + +struct bpf_diag_scratch { + struct bpf_linfo_source source_lines[BPF_DIAG_CONTEXT_CNT]; + struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT]; +}; + +struct bpf_diag_mod_scope { + struct bpf_reg_state target_reg_snapshot; + struct bpf_diag_mod_target target; + struct bpf_diag_mod_target origin; + enum bpf_diag_mod_reason reason; + u32 insn_idx; + bool active; + bool origin_valid; +}; + +struct bpf_diag { + struct bpf_diag_log log; + struct bpf_diag_scratch scratch; + struct list_head fmt_chunks; + struct bpf_diag_mod_scope mod; + u32 frame_id_gen; +}; + +bool bpf_diag_enabled(const struct bpf_verifier_env *env) +{ + return env->log.level & BPF_LOG_LEVEL; +} + +static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); + +int bpf_diag_init(struct bpf_verifier_env *env) +{ + if (!bpf_diag_enabled(env)) + return 0; + + env->diag = kzalloc_obj(struct bpf_diag, GFP_KERNEL_ACCOUNT); + if (!env->diag) + return -ENOMEM; + + INIT_LIST_HEAD(&env->diag->fmt_chunks); + return 0; +} + +void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state) +{ + if (env->diag) + state->diag_frame_id = ++env->diag->frame_id_gen; +} + +static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_chunk *chunk; + size_t capacity, available; + char *buf; + + if (!diag || !size || size > INT_MAX) + return NULL; + + if (!list_empty(&diag->fmt_chunks)) { + chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + available = seq_buf_get_buf(&chunk->seq, &buf); + if (available >= size) + goto commit; + } + + capacity = max_t(size_t, BPF_DIAG_FMT_CHUNK_SIZE, size); + chunk = kmalloc(struct_size(chunk, data, capacity), GFP_KERNEL_ACCOUNT); + if (!chunk) + return NULL; + + seq_buf_init(&chunk->seq, chunk->data, capacity); + list_add_tail(&chunk->node, &diag->fmt_chunks); + available = seq_buf_get_buf(&chunk->seq, &buf); + if (WARN_ON_ONCE(available < size)) + return NULL; + +commit: + seq_buf_commit(&chunk->seq, size); + return buf; +} + +char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size) +{ + char *buf; + + buf = diag_fmt_alloc(env, size); + if (buf) + buf[0] = '\0'; + return buf; +} + +const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) +{ + va_list copy; + char *buf; + int len; + + va_copy(copy, args); + len = vsnprintf(NULL, 0, fmt, copy); + va_end(copy); + if (len < 0 || len == INT_MAX) + return ""; + + buf = diag_fmt_alloc(env, len + 1); + if (buf) + vsnprintf(buf, len + 1, fmt, args); + return buf ?: ""; +} + +const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) +{ + const char *buf; + va_list args; + + va_start(args, fmt); + buf = bpf_diag_vfmt(env, fmt, args); + va_end(args); + return buf; +} + +static struct diag_fmt_mark diag_fmt_save(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_mark mark = {}; + + if (!diag || list_empty(&diag->fmt_chunks)) + return mark; + + mark.chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + mark.len = mark.chunk->seq.len; + return mark; +} + +static void diag_fmt_restore(struct bpf_verifier_env *env, struct diag_fmt_mark mark) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_chunk *chunk; + + if (!diag) + return; + + while (!list_empty(&diag->fmt_chunks)) { + chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + if (chunk == mark.chunk) + break; + list_del(&chunk->node); + kfree(chunk); + } + + if (mark.chunk) { + mark.chunk->seq.len = mark.len; + seq_buf_str(&mark.chunk->seq); + } +} + +void bpf_diag_free(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + + if (!diag) + return; + + diag_fmt_restore(env, (struct diag_fmt_mark){}); + kvfree(diag->log.events); + kfree(diag); + env->diag = NULL; +} + +static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + va_start(args, fmt); + bpf_verifier_vlog(&env->log, fmt, args); + va_end(args); +} + +static u64 log_end(const struct bpf_diag_log *log) +{ + return log->first_seq + log->cnt; +} + +static u32 log_pos(const struct bpf_diag_log *log, u32 idx) +{ + u32 pos = log->head + idx; + + return pos < log->cap ? pos : pos - log->cap; +} + +u64 bpf_diag_event_log_save(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + + return diag ? log_end(&diag->log) : 0; +} + +void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos) +{ + struct bpf_diag *diag = env->diag; + struct bpf_diag_log *log; + u64 end_seq; + + if (!diag) + return; + + log = &diag->log; + end_seq = log_end(log); + if (WARN_ON_ONCE(log_pos > end_seq)) + log_pos = end_seq; + + /* + * A deep abandoned path may have rotated away the shared prefix. In + * that case, restart with an empty retained suffix and remember that + * every event before the restored mark is unavailable. + */ + if (log_pos <= log->first_seq) { + log->first_seq = log_pos; + log->head = 0; + log->cnt = 0; + return; + } + + log->cnt = log_pos - log->first_seq; +} + +u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state) +{ + u32 depth = 0; + int i; + + for (i = 0; i < state->acquired_refs; i++) { + if (state->refs[i].type == REF_TYPE_IRQ) + depth++; + } + + return depth; +} + +static void diag_append_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + struct bpf_diag_history_event *events; + struct bpf_diag *diag = env->diag; + struct bpf_diag_log *log; + u32 cap, max_events; + + if (!diag) + return; + log = &diag->log; + + if (log->cnt < log->cap) { + log->events[log_pos(log, log->cnt++)] = *event; + return; + } + + max_events = BPF_DIAG_EVENT_LOG_MAX_SIZE / sizeof(*events); + if (log->growth_failed || log->cap == max_events) + goto rotate; + + cap = min(log->cap ? log->cap * 2 : 64, max_events); + events = kvrealloc(log->events, array_size(cap, sizeof(*events)), GFP_KERNEL_ACCOUNT); + if (!events) { + log->growth_failed = true; + goto rotate; + } + log->events = events; + log->cap = cap; + log->events[log->cnt++] = *event; + return; + +rotate: + if (log->cap) { + log->events[log->head++] = *event; + if (log->head == log->cap) + log->head = 0; + } + log->first_seq++; +} + +static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char *first_prefix, + const char *next_prefix, const char *text) +{ + const char *prefix = first_prefix; + + while (*text) { + const char *line = text; + int prefix_len = strlen(prefix); + int text_width = BPF_DIAG_TEXT_WIDTH - prefix_len; + int len = 0, last_space = -1; + + if (text_width < 1) + text_width = 1; + + while (line[len] && line[len] != '\n' && len < text_width) { + if (line[len] == ' ') + last_space = len; + len++; + } + + if (line[len] && line[len] != '\n' && line[len] != ' ' && last_space > 0) + len = last_space; + + diag_write(env, "%s%.*s\n", prefix, len, line); + + text = line + len; + while (*text == ' ') + text++; + if (*text == '\n') + text++; + + prefix = next_prefix; + } +} + +const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id) +{ + char *buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + size_t len; + int ret; + + if (!buf) + return ""; + + buf[0] = '\0'; + ret = btf_type_name_to_buf(btf, type_id, buf, BPF_DIAG_FMT_BUF_SIZE); + if (ret < 0 || !buf[0]) { + scnprintf(buf, BPF_DIAG_FMT_BUF_SIZE, "BTF type ID %u", type_id); + return buf; + } + + len = strlen(buf); + if (len && buf[len - 1] == '{') + buf[len - 1] = '\0'; + return buf; +} + +static void diag_vprint_indented(struct bpf_verifier_env *env, const char *fmt, va_list args) + __printf(2, 0); + +static void diag_vprint_indented(struct bpf_verifier_env *env, const char *fmt, va_list args) +{ + char *buf; + + if (!bpf_diag_enabled(env)) + return; + + buf = kvasprintf(GFP_KERNEL_ACCOUNT, fmt, args); + if (!buf) { + diag_write(env, "%s<failed to allocate diagnostic text>\n", BPF_DIAG_TEXT_INDENT); + return; + } + + diag_print_wrapped_prefixed(env, BPF_DIAG_TEXT_INDENT, BPF_DIAG_TEXT_INDENT, buf); + kfree(buf); +} + +static int diag_line_width(unsigned int line) +{ + int width = 1; + + while (line >= 10) { + line /= 10; + width++; + } + + return width; +} + +static int diag_line_indent(const char *line) +{ + int indent = 0; + + while (*line == ' ' || *line == '\t') { + if (*line == '\t') + indent = round_up(indent + 1, BPF_DIAG_TAB_WIDTH); + else + indent++; + line++; + } + + return indent; +} + +static void disasm_print(void *private_data, const char *fmt, ...) __printf(2, 3); + +static void disasm_print(void *private_data, const char *fmt, ...) +{ + struct disasm_ctx *ctx = private_data; + va_list args; + + va_start(args, fmt); + seq_buf_vprintf(&ctx->seq, fmt, args); + va_end(args); +} + +static const char *disasm_kfunc_name(void *private_data, const struct bpf_insn *insn) +{ + struct disasm_ctx *ctx = private_data; + + return bpf_disasm_kfunc_name(ctx->env, insn); +} + +static void format_disasm_line(struct bpf_verifier_env *env, int insn_idx, + struct disasm_line *line) +{ + struct disasm_ctx ctx = { .env = env }; + struct bpf_insn *insn; + const struct bpf_insn_cbs cbs = { + .cb_call = disasm_kfunc_name, + .cb_print = disasm_print, + .private_data = &ctx, + }; + + line->idx = insn_idx; + line->valid = false; + seq_buf_init(&ctx.seq, line->text, sizeof(line->text)); + + if (insn_idx < 0 || insn_idx >= env->prog->len) + return; + + if (insn_idx > 0 && bpf_is_ldimm64(&env->prog->insnsi[insn_idx - 1])) + return; + + insn = &env->prog->insnsi[insn_idx]; + if (bpf_is_ldimm64(insn) && insn_idx + 1 >= env->prog->len) + return; + + print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); + seq_buf_str(&ctx.seq); + ctx.seq.len = strnlen(line->text, sizeof(line->text)); + while (ctx.seq.len && line->text[ctx.seq.len - 1] == '\n') + seq_buf_pop(&ctx.seq); + seq_buf_str(&ctx.seq); + + line->valid = true; +} + +static void diag_format_source_text(char *buf, size_t size, const char *line, int width) +{ + int col = 0, len = 0; + + if (!size) + return; + if (width <= 0) { + buf[0] = '\0'; + return; + } + + line = line ?: "..."; + while (*line && col < width && len + 1 < size) { + if (*line == '\t') { + int next = round_up(col + 1, BPF_DIAG_TAB_WIDTH); + + while (col < next && col < width && len + 1 < size) { + buf[len++] = ' '; + col++; + } + line++; + continue; + } + + buf[len++] = *line++; + col++; + } + + if (*line) { + int ellipsis_len = min(3, width); + + while (len > 0 && col > width - ellipsis_len) { + len--; + col--; + } + while (ellipsis_len-- && len + 1 < size) + buf[len++] = '.'; + } + + buf[len] = '\0'; +} + +static void diag_format_source_lane(char *buf, size_t size, const char *source_prefix, + int source_line_width, int line_num, const char *line) +{ + int len, text_width; + + if (line_num <= 0) { + buf[0] = '\0'; + return; + } + + len = scnprintf(buf, size, "%s%*d | ", source_prefix, source_line_width, line_num); + text_width = BPF_DIAG_SOURCE_LANE_WIDTH - len; + diag_format_source_text(buf + len, size - len, line, text_width); +} + +static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, + const char *problem) +{ + char first; + + if (!bpf_diag_enabled(env)) + return; + + category = category ?: "Verifier Error"; + problem = problem ?: ""; + + if (!problem[0]) { + diag_write(env, "\nVerification failed: %s\n", category); + return; + } + + first = toupper(problem[0]); + diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1); +} + +static void diag_reason(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +static void diag_suggestion(struct bpf_verifier_env *env, const char *fmt, ...) + __printf(2, 3); + +static void diag_section(struct bpf_verifier_env *env, const char *title) +{ + if (!bpf_diag_enabled(env)) + return; + + diag_write(env, "\n%s:\n", title); +} + +static void diag_reason(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + diag_section(env, "Reason"); + + va_start(args, fmt); + diag_vprint_indented(env, fmt, args); + va_end(args); +} + +static void diag_suggestion(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + diag_section(env, "Suggestion"); + + va_start(args, fmt); + diag_vprint_indented(env, fmt, args); + va_end(args); + diag_write(env, "\n"); +} + +static void diag_print_source_annotation(struct bpf_verifier_env *env, int line_width, int indent, + const char *label, const char *msg) +{ + const char *first_prefix, *next_prefix, *text; + + indent = min_t(int, indent, max_t(int, 0, BPF_DIAG_SOURCE_LANE_WIDTH - line_width - 8)); + text = bpf_diag_fmt(env, "%s: %s", label, msg); + first_prefix = bpf_diag_fmt(env, " %*s | %*s^-- ", line_width + 4, "", indent, ""); + next_prefix = bpf_diag_fmt(env, " %*s | %*s ", line_width + 4, "", indent, ""); + + diag_print_wrapped_prefixed(env, first_prefix, next_prefix, text); +} + +static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, + struct disasm_line *disasm_lines) +{ + int insn_width = diag_line_width(env->prog->len ? env->prog->len - 1 : 0); + int i; + + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + int row = i - BPF_DIAG_CONTEXT; + + format_disasm_line(env, insn_idx + row, &disasm_lines[i]); + } + + diag_write(env, " Instruction context:\n"); + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + struct disasm_line *line = &disasm_lines[i]; + + if (line->valid) + diag_write(env, " %s%*d | %s\n", + line->idx == insn_idx ? ">>> " : " ", + insn_width, line->idx, line->text); + } +} + +static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, + const char *fmt, ...) +{ + struct bpf_diag_scratch *scratch; + struct bpf_linfo_source *source_lines; + struct disasm_line *disasm_lines; + struct bpf_linfo_source src = {}; + struct diag_fmt_mark mark; + const struct bpf_line_info *linfo; + const struct bpf_subprog_info *subprog; + struct btf *btf = env->prog->aux->btf; + char *source_lane; + const char *msg; + const char *func; + int start_line, end_line, width, indent, subprogno, linfo_start, linfo_end, i; + va_list args; + + if (!bpf_diag_enabled(env)) + return; + if (!env->diag) + return; + + mark = diag_fmt_save(env); + label = label ?: "note"; + scratch = &env->diag->scratch; + source_lines = scratch->source_lines; + disasm_lines = scratch->disasm_lines; + memset(source_lines, 0, sizeof(scratch->source_lines)); + memset(disasm_lines, 0, sizeof(scratch->disasm_lines)); + + va_start(args, fmt); + msg = bpf_diag_vfmt(env, fmt, args); + va_end(args); + if (!*msg) + msg = "<failed to allocate diagnostic text>"; + + linfo = bpf_find_linfo(env->prog, insn_idx); + if (btf && linfo) + bpf_get_linfo_source(btf, linfo, &src); + if (!src.file || !*src.file) { + diag_write(env, " insn %u\n", insn_idx); + goto out_annotation; + } + + subprog = bpf_find_containing_subprog(env, insn_idx); + subprogno = subprog ? subprog - env->subprog_info : -ENOENT; + func = subprogno >= 0 ? bpf_subprog_name(env, subprogno) : NULL; + if (func && *func) + diag_write(env, " %s @ %s:%d:%d\n", func, src.file, src.line_num, src.line_col); + else + diag_write(env, " %s:%d:%d\n", src.file, src.line_num, src.line_col); + if (!src.line || !*src.line) + goto out_annotation; + + start_line = src.line_num - BPF_DIAG_CONTEXT; + end_line = src.line_num + BPF_DIAG_CONTEXT; + width = diag_line_width(end_line); + indent = diag_line_indent(src.line); + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) + source_lines[i].line_num = start_line + i; + + linfo = env->prog->aux->linfo; + linfo_start = subprog ? subprog->linfo_idx : 0; + linfo_end = subprogno >= 0 && subprogno + 1 < env->subprog_cnt ? + env->subprog_info[subprogno + 1].linfo_idx : env->prog->aux->nr_linfo; + for (i = linfo_start; i < linfo_end; i++) { + struct bpf_linfo_source line_src; + int idx; + + bpf_get_linfo_source(btf, &linfo[i], &line_src); + if (line_src.file_name_off != src.file_name_off || + line_src.line_num < start_line || line_src.line_num > end_line || + !line_src.line || !*line_src.line) + continue; + + idx = line_src.line_num - start_line; + if (!source_lines[idx].line) + source_lines[idx] = line_src; + } + + diag_write(env, " Source context:\n"); + source_lane = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + if (!source_lane) + goto out_restore; + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + const char *source_prefix; + + source_prefix = source_lines[i].line_num == src.line_num ? ">>> " : " "; + diag_format_source_lane(source_lane, BPF_DIAG_FMT_BUF_SIZE, source_prefix, width, + source_lines[i].line_num, source_lines[i].line); + diag_write(env, " %s\n", source_lane); + if (source_lines[i].line_num == src.line_num) + diag_print_source_annotation(env, width, indent, label, msg); + } + diag_print_insn_context(env, insn_idx, disasm_lines); + goto out_restore; + +out_annotation: + diag_print_source_annotation(env, 0, 0, label, msg); + diag_print_insn_context(env, insn_idx, disasm_lines); +out_restore: + diag_fmt_restore(env, mark); +} + +static const struct bpf_func_state *diag_current_frame(const struct bpf_verifier_env *env) +{ + return env->cur_state->frame[env->cur_state->curframe]; +} + +void bpf_diag_register_type(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *problem, const char *reason, const char *suggestion) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + if (regno >= 0) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type) +{ + switch (base_type(type)) { + case NOT_INIT: + return "an uninitialized value"; + case SCALAR_VALUE: + return "an integer scalar"; + case PTR_TO_CTX: + return "a context pointer"; + case PTR_TO_STACK: + return "a stack pointer"; + case PTR_TO_MAP_VALUE: + if (type_may_be_null(type)) + return "a nullable map value pointer"; + return "a map value pointer"; + case PTR_TO_MEM: + if (type_may_be_null(type)) + return "a nullable memory pointer"; + return "a memory pointer"; + case PTR_TO_BTF_ID: + if (type_may_be_null(type)) + return "a nullable kernel object pointer"; + if (type_is_non_owning_ref(type)) + return "a borrowed allocated object pointer"; + if (type_is_ptr_alloc_obj(type)) + return "an owned allocated object pointer"; + if (type_flag(type) & PTR_UNTRUSTED) + return "an untrusted kernel object pointer"; + return "a kernel object pointer"; + default: + return reg_type_str(env, type); + } +} + +static const char *diag_arg_ordinal(int argno) +{ + switch (argno) { + case 1: + return "first"; + case 2: + return "second"; + case 3: + return "third"; + case 4: + return "fourth"; + case 5: + return "fifth"; + case 6: + return "sixth"; + case 7: + return "seventh"; + case 8: + return "eighth"; + case 9: + return "ninth"; + case 10: + return "tenth"; + case 11: + return "eleventh"; + case 12: + return "twelfth"; + default: + return NULL; + } +} + +void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno, + int stack_arg_slot, const char *call_name, const char *arg_name, + const char *reason, const char *suggestion) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + }; + const char *ordinal = diag_arg_ordinal(argno); + const char *arg_desc; + bool print_history = true; + + if (regno >= 0) { + opts.scope = BPF_DIAG_HISTORY_SCOPE_REG; + opts.regno = regno; + } else if (stack_arg_slot >= 0) { + opts.scope = BPF_DIAG_HISTORY_SCOPE_STACK_ARG; + opts.stack_arg_slot = stack_arg_slot; + } else { + print_history = false; + } + + if (ordinal && arg_name) + arg_desc = bpf_diag_fmt(env, "%s argument (%s)", ordinal, arg_name); + else if (ordinal) + arg_desc = bpf_diag_fmt(env, "%s argument", ordinal); + else if (arg_name) + arg_desc = bpf_diag_fmt(env, "argument %s", arg_name); + else + arg_desc = "argument"; + + bpf_diag_header(env, CALL_TYPE_SAFETY, "invalid call argument"); + diag_reason(env, "The %s to %s does not satisfy the verifier contract: %s.", + arg_desc, call_name, reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "invalid %s for %s", arg_desc, call_name); + + if (print_history) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +static const char *diag_context_constraint(enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return "RCU read-side critical sections cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_PREEMPT: + return "preemption-disabled code cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_IRQ: + return "IRQ-disabled code cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_LOCK: + return "code holding a BPF spin lock cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_NONE: + default: + return NULL; + } +} + +static const char *diag_active_context(struct bpf_verifier_env *env, u32 depth, + const char *context) +{ + if (depth == 1) + return bpf_diag_fmt(env, "an active %s (depth 1)", context); + return bpf_diag_fmt(env, "%u active %ss (depth %u)", depth, context, depth); +} + +static u32 diag_context_depth(struct bpf_verifier_env *env, enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return env->cur_state->active_rcu_locks; + case BPF_DIAG_CONTEXT_PREEMPT: + return env->cur_state->active_preempt_locks; + case BPF_DIAG_CONTEXT_IRQ: + return bpf_diag_irq_depth(env->cur_state); + case BPF_DIAG_CONTEXT_LOCK: + return env->cur_state->active_locks; + case BPF_DIAG_CONTEXT_NONE: + default: + return 0; + } +} + +void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, const char *suggestion) +{ + struct bpf_diag_history_opts opts; + enum bpf_diag_context_kind ctx_kind; + const char *constraint, *context; + u32 depth; + + if (env->cur_state->active_rcu_locks) + ctx_kind = BPF_DIAG_CONTEXT_RCU; + else if (env->cur_state->active_preempt_locks) + ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; + else if (env->cur_state->active_irq_id) + ctx_kind = BPF_DIAG_CONTEXT_IRQ; + else if (env->cur_state->active_locks) + ctx_kind = BPF_DIAG_CONTEXT_LOCK; + else + ctx_kind = BPF_DIAG_CONTEXT_NONE; + + depth = diag_context_depth(env, ctx_kind); + opts = (struct bpf_diag_history_opts) { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + .ctx_depth = depth, + }; + constraint = diag_context_constraint(ctx_kind); + context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, + "operation is not allowed in this context"); + if (constraint) { + if (depth) { + diag_reason( + env, "The operation %s cannot be used in %s because %s. This path is still inside %s.", + operation, context, constraint, diag_active_context(env, depth, context)); + } else { + diag_reason(env, "The operation %s cannot be used in %s because %s.", + operation, context, constraint); + } + } else { + diag_reason(env, "The operation %s cannot be used in %s.", operation, + context); + } + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not allowed in %s", operation, + context); + + if (ctx_kind != BPF_DIAG_CONTEXT_NONE) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_active(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion) +{ + u32 depth = diag_context_depth(env, ctx_kind); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + .ctx_depth = depth, + }; + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, + "operation is not allowed in this context"); + diag_reason( + env, "The operation %s cannot be used while this path is still inside %s. Leave the region before this operation.", + operation, diag_active_context(env, depth, context)); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not allowed before leaving %s", + operation, context); + + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_required(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion) +{ + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, "required context is not active"); + diag_reason(env, "The operation %s requires an active %s, but this path is outside one.", + operation, context); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s requires %s", operation, context); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, enum bpf_diag_context_kind ctx_kind, + const char *suggestion) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + }; + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, "unmatched context exit"); + diag_reason( + env, "The operation %s tries to leave %s, but this path has no active %s to leave. The current depth is 0.", + operation, context, context); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s has no matching enter on this path", + operation); + + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx, + const char *problem, const char *suggestion, + const char *reason_fmt, ...) +{ + va_list args; + + bpf_diag_header(env, PROGRAM_STRUCTURE, problem); + diag_section(env, "Reason"); + + va_start(args, reason_fmt); + diag_vprint_indented(env, reason_fmt, args); + va_end(args); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_policy(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + const char *reason, const char *suggestion) +{ + bpf_diag_header(env, POLICY, "operation is not allowed"); + diag_reason(env, "The %s is not allowed: %s.", operation, reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "policy check failed for %s", operation); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const struct bpf_reg_state *reg, + enum bpf_diag_invalid_deref_kind kind, s64 offset) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const char *type_name = bpf_diag_reg_type_plain(env, reg->type); + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "invalid dereference"); + + switch (kind) { + case BPF_DIAG_DEREF_SCALAR: + diag_reason(env, "%s is an integer scalar here, not a pointer to memory.", + reg_name); + break; + case BPF_DIAG_DEREF_NULLABLE_PTR: + diag_reason( + env, "%s may be NULL here (%s). The program could dereference NULL on this path, so the verifier cannot prove this access is safe.", + reg_name, type_name); + break; + case BPF_DIAG_DEREF_MODIFIED_PTR: + diag_reason( + env, "%s has offset %lld here, but this pointer type must be dereferenced in its original form.", + reg_name, offset); + break; + case BPF_DIAG_DEREF_INVALID_PTR: + default: + diag_reason( + env, "%s has type %s here, which is not valid for this memory access.", + reg_name, type_name); + break; + } + + diag_section(env, "At"); + if (kind == BPF_DIAG_DEREF_MODIFIED_PTR) + bpf_diag_source(env, insn_idx, "error", + "dereference requires the original %s pointer", type_name); + else + bpf_diag_source(env, insn_idx, "error", "invalid dereference of %s (%s)", + reg_name, type_name); + + if (regno >= 0) + diag_print_history(env, &opts); + + switch (kind) { + case BPF_DIAG_DEREF_NULLABLE_PTR: + diag_suggestion( + env, "Add a NULL check before the access and dereference the pointer only on the non-NULL path."); + break; + case BPF_DIAG_DEREF_MODIFIED_PTR: + diag_suggestion( + env, "Preserve the original pointer in another register, or use only offsets this pointer type permits before dereferencing it."); + break; + case BPF_DIAG_DEREF_SCALAR: + case BPF_DIAG_DEREF_INVALID_PTR: + default: + diag_suggestion( + env, "Preserve a pointer-valued register where needed, or reload and revalidate the pointer after scalar arithmetic, helper calls, or other operations that can invalidate it."); + break; + } +} + +void bpf_diag_unreadable_reg(struct bpf_verifier_env *env, u32 insn_idx, int regno) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const struct bpf_diag_log *log = env->diag ? &env->diag->log : NULL; + struct bpf_diag_mod_target target; + bool invalidated = false; + int i; + + target = diag_reg_target(opts.frame_id, opts.frameno, regno); + for (i = log ? log->cnt : 0; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + + if (event->kind != BPF_DIAG_HISTORY_MOD || + !diag_target_matches(&event->mod.target, &target)) + continue; + invalidated = event->mod.new.type == NOT_INIT; + break; + } + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "unreadable register"); + if (invalidated) + diag_reason( + env, "R%d is not readable here. A previous operation invalidated this register, so the verifier cannot use it as an input.", + regno); + else if (log && !log->first_seq) + diag_reason(env, + "R%d has never been initialized on this path, so the verifier cannot use it as an input.", + regno); + else + diag_reason( + env, "R%d is not readable here. It may never have been initialized, or an earlier operation may have invalidated it.", + regno); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "R%d is not readable", regno); + + if (regno >= 0) + diag_print_history(env, &opts); + + if (invalidated) + diag_suggestion( + env, "Avoid using the register after it is invalidated, or initialize it again before this instruction."); + else if (log && !log->first_seq) + diag_suggestion(env, "Initialize R%d on every path before this instruction.", regno); + else + diag_suggestion( + env, "Initialize the register on every path, or initialize it again after any operation that invalidates it."); +} + +static int diag_stack_argno(u8 slot) +{ + return MAX_BPF_FUNC_REG_ARGS + slot + 1; +} + +static void diag_format_stack_arg(char *buf, size_t size, u8 slot, const char *arg_name) +{ + int argno = diag_stack_argno(slot); + const char *ordinal = diag_arg_ordinal(argno); + + if (ordinal && arg_name) + scnprintf(buf, size, "outgoing stack argument %u (%s argument, %s)", slot + 1, + ordinal, arg_name); + else if (ordinal) + scnprintf(buf, size, "outgoing stack argument %u (%s argument)", slot + 1, ordinal); + else if (arg_name) + scnprintf(buf, size, "outgoing stack argument %u (%s)", slot + 1, arg_name); + else + scnprintf(buf, size, "outgoing stack argument %u", slot + 1); +} + +void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int nargs, + int stack_arg_slot, const char *callee_name, + const char *arg_name) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_STACK_ARG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .stack_arg_slot = stack_arg_slot, + }; + const char *arg_buf; + + arg_buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + if (arg_buf) + diag_format_stack_arg((char *)arg_buf, BPF_DIAG_FMT_BUF_SIZE, stack_arg_slot, + arg_name); + else + arg_buf = ""; + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "missing stack argument"); + if (callee_name && *callee_name) + diag_reason( + env, "Function %s expects %d arguments, but %s is not initialized at this call.", + callee_name, nargs, arg_buf); + else + diag_reason( + env, "The callee expects %d arguments, but %s is not initialized at this call.", + nargs, arg_buf); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not initialized", arg_buf); + + if (stack_arg_slot >= 0) + diag_print_history(env, &opts); + + diag_suggestion( + env, "Write the outgoing stack argument after any operation that may invalidate stored pointer values, and before making this call."); +} + +void bpf_diag_memory(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion) +{ + bpf_diag_header(env, MEMORY_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_BRANCH, + .branch = { + .cond_true = cond_true, + }, + }; + + diag_append_history(env, &event); +} + +static void diag_snapshot_reg(struct bpf_diag_reg_snapshot *snapshot, + const struct bpf_reg_state *reg) +{ + snapshot->type = reg->type; + if (type_is_map_ptr(reg->type)) + snapshot->map_ptr = reg->map_ptr; + if (base_type(reg->type) == PTR_TO_BTF_ID && reg->btf && reg->btf_id) { + snapshot->btf_id = reg->btf_id; + snapshot->btf = reg->btf; + } + snapshot->var_off = reg->var_off; + snapshot->r64 = reg->r64; +} + +static bool diag_mod_insn_origin(struct bpf_verifier_env *env, u32 insn_idx, + const struct bpf_diag_mod_target *target, + struct bpf_diag_mod_target *origin) +{ + const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + u8 class = BPF_CLASS(insn->code); + const struct bpf_func_state *state; + + if (target->kind == BPF_DIAG_MOD_TARGET_REG && (class == BPF_ALU || class == BPF_ALU64) && + BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X) { + *origin = diag_reg_target(target->frame_id, target->frameno, insn->src_reg); + return true; + } + + if ((target->kind != BPF_DIAG_MOD_TARGET_STACK_ARG && + target->kind != BPF_DIAG_MOD_TARGET_STACK_SLOT) || + class != BPF_STX) + return false; + + state = env->cur_state->frame[env->cur_state->curframe]; + *origin = diag_reg_target(state->diag_frame_id, state->frameno, insn->src_reg); + return true; +} + +static bool diag_mod_keeps_lineage(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + const struct bpf_insn *insn; + u8 class; + + if (event->mod.reason != BPF_DIAG_MOD_WRITE || + event->mod.target.kind != BPF_DIAG_MOD_TARGET_REG) + return false; + + insn = &env->prog->insnsi[event->insn_idx]; + class = BPF_CLASS(insn->code); + if (class != BPF_ALU && class != BPF_ALU64) + return false; + + switch (BPF_OP(insn->code)) { + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_OR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + case BPF_XOR: + case BPF_NEG: + case BPF_END: + return true; + default: + return false; + } +} + +static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, + struct bpf_diag_mod_target target, + enum bpf_diag_mod_reason reason, + const struct bpf_reg_state *old_reg, + const struct bpf_reg_state *new_reg, + const struct bpf_diag_mod_target *origin) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_MOD, + .mod = { + .target = target, + .reason = reason, + }, + }; + + if (old_reg) + diag_snapshot_reg(&event.mod.old, old_reg); + if (new_reg) + diag_snapshot_reg(&event.mod.new, new_reg); + if (origin) { + event.mod.origin = *origin; + event.mod.origin_valid = true; + } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { + event.mod.origin_valid = true; + } + if (old_reg && new_reg && + (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && + !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && + !event.mod.origin_valid && + diag_mod_keeps_lineage(env, &event)) + return; + + diag_append_history(env, &event); +} + +static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, + const struct bpf_diag_mod_target *target) +{ + struct bpf_verifier_state *vstate = env->cur_state; + struct bpf_func_state *state; + + state = target->frameno <= vstate->curframe ? vstate->frame[target->frameno] : NULL; + + if (!state) + return NULL; + if (state->diag_frame_id != target->frame_id) + return NULL; + + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + if (target->regno >= MAX_BPF_REG) + return NULL; + return &state->regs[target->regno]; + case BPF_DIAG_MOD_TARGET_STACK_ARG: + if (target->stack_arg >= state->out_stack_arg_cnt) + return NULL; + return &state->stack_arg_regs[target->stack_arg]; + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + if (target->spi >= state->allocated_stack / BPF_REG_SIZE) + return NULL; + return &state->stack[target->spi].spilled_ptr; + default: + return NULL; + } +} + +static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + struct bpf_diag_mod_target *target) +{ + struct bpf_verifier_state *vstate = env->cur_state; + unsigned long addr = (unsigned long)reg; + int frame; + + for (frame = 0; frame <= vstate->curframe; frame++) { + struct bpf_func_state *state = vstate->frame[frame]; + unsigned long start, end; + u32 nslots = state->allocated_stack / BPF_REG_SIZE; + int spi; + + start = (unsigned long)state->regs; + end = (unsigned long)(state->regs + MAX_BPF_REG); + if (addr >= start && addr < end) { + *target = diag_reg_target(state->diag_frame_id, state->frameno, + reg - state->regs); + return true; + } + + start = (unsigned long)state->stack_arg_regs; + end = (unsigned long)(state->stack_arg_regs + state->out_stack_arg_cnt); + if (state->out_stack_arg_cnt && addr >= start && addr < end) { + *target = diag_stack_arg_target(state->diag_frame_id, state->frameno, + reg - state->stack_arg_regs); + return true; + } + + start = (unsigned long)state->stack; + end = (unsigned long)(state->stack + nslots); + if (nslots && addr >= start && addr < end) { + spi = ((const char *)reg - (const char *)state->stack) / + sizeof(*state->stack); + *target = diag_stack_slot_target(state->diag_frame_id, state->frameno, spi); + return true; + } + } + return false; +} + +void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason) +{ + struct bpf_diag *diag = env->diag; + + if (!diag) + return; + diag->mod.active = reg_to_target(env, reg, &diag->mod.target); + if (!diag->mod.active) + return; + diag->mod.target_reg_snapshot = *reg; + diag->mod.insn_idx = env->insn_idx; + diag->mod.reason = reason; + diag->mod.origin_valid = origin && reg_to_target(env, origin, &diag->mod.origin); +} + +void bpf_diag_mod_end(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + const struct bpf_reg_state *new_reg; + + if (!diag || !diag->mod.active) + return; + diag->mod.active = false; + /* + * Resolve the target again because the enclosing function state's stack + * may have been reallocated while the modification was in progress. + */ + new_reg = target_to_reg(env, &diag->mod.target); + if (!new_reg) + return; + diag_record_mod(env, diag->mod.insn_idx, diag->mod.target, diag->mod.reason, + &diag->mod.target_reg_snapshot, new_reg, + diag->mod.origin_valid ? &diag->mod.origin : NULL); +} + +void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + enum bpf_diag_mod_reason reason) +{ + struct bpf_diag_mod_target target; + + if (!env->diag || reg->type == NOT_INIT || !reg_to_target(env, reg, &target)) + return; + diag_record_mod(env, env->insn_idx, target, reason, reg, NULL, NULL); +} + +void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, + const struct bpf_func_state *state, s16 min_off, s16 max_off, + enum bpf_diag_mod_reason reason) +{ + diag_record_mod(env, env->insn_idx, + diag_stack_range_target(state->diag_frame_id, state->frameno, min_off, max_off), + reason, NULL, NULL, NULL); +} + +static void diag_record_ref(struct bpf_verifier_env *env, u32 insn_idx, u8 kind, u32 ref_id) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = kind, + .ref = { + .ref_id = ref_id, + }, + }; + + diag_append_history(env, &event); +} + +void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) +{ + diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_ACQUIRE, ref_id); +} + +void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) +{ + diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_RELEASE, ref_id); +} + +void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx, + enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth) +{ + /* + * Keep leave events so context rendering can stop at a depth-zero exit + * and show nested-region depth accurately for the active path. + */ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_CONTEXT, + .ctx = { + .kind = ctx_kind, + .enter = enter, + .depth = depth, + }, + }; + + diag_append_history(env, &event); +} + +static int diag_history_context_start_idx(const struct bpf_diag_log *log, + const struct bpf_diag_history_opts *opts) +{ + int i; + + if (!opts->ctx_depth) + return 0; + + /* Find the most recent outermost entry, or a depth-zero exit. */ + for (i = log->cnt; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + + if (event->kind != BPF_DIAG_HISTORY_CONTEXT || event->ctx.kind != opts->ctx_kind) + continue; + + if (event->ctx.enter && event->ctx.depth == 1) + return i - 1; + if (!event->ctx.enter && event->ctx.depth == 0) + return 0; + } + + return 0; +} + +struct bpf_diag_history_filter { + const struct bpf_diag_history_opts *opts; + u32 lineage_start; + bool lineage_valid; +}; + +static bool diag_target_matches(const struct bpf_diag_mod_target *event_target, + const struct bpf_diag_mod_target *target) +{ + int slot_off; + + if (event_target->frame_id != target->frame_id || event_target->frameno != target->frameno) + return false; + + if (event_target->kind == BPF_DIAG_MOD_TARGET_STACK_RANGE && + target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT) { + slot_off = -(target->spi + 1) * BPF_REG_SIZE; + return event_target->range.min_off < slot_off + BPF_REG_SIZE && + event_target->range.max_off > slot_off; + } + + if (event_target->kind != target->kind) + return false; + + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + return event_target->regno == target->regno; + case BPF_DIAG_MOD_TARGET_STACK_ARG: + return event_target->stack_arg == target->stack_arg; + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + return event_target->spi == target->spi; + default: + return false; + } +} + +static void diag_build_lineage(struct bpf_verifier_env *env, struct bpf_diag_log *log, + struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + struct bpf_diag_mod_target target; + int i; + + for (i = 0; i < log->cnt; i++) + log->events[log_pos(log, i)].in_lineage = false; + + if (opts->scope == BPF_DIAG_HISTORY_SCOPE_REG) + target = diag_reg_target(opts->frame_id, opts->frameno, opts->regno); + else if (opts->scope == BPF_DIAG_HISTORY_SCOPE_STACK_ARG) + target = diag_stack_arg_target(opts->frame_id, opts->frameno, + opts->stack_arg_slot); + else + return; + + /* + * Find the nearest mutation of the active target. A fill or spill changes + * the target to its origin, so the same walk follows register/stack + * lineage recursively until it reaches the write that created the value. + */ + for (i = log->cnt; i > 0; i--) { + struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + if (event->kind != BPF_DIAG_HISTORY_MOD || + !diag_target_matches(&event->mod.target, &target)) + continue; + + event->in_lineage = true; + filter->lineage_start = i - 1; + filter->lineage_valid = true; + + if (event->mod.origin_valid) { + target = event->mod.origin; + continue; + } + if (event->mod.reason != BPF_DIAG_MOD_WRITE && + event->mod.reason != BPF_DIAG_MOD_SPILL) + continue; + if (diag_mod_keeps_lineage(env, event)) + continue; + break; + } +} + +static int diag_history_start_idx(const struct bpf_diag_log *log, + const struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + int i; + + if (opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT) + return diag_history_context_start_idx(log, opts); + if (filter->lineage_valid) + return filter->lineage_start; + if (opts->scope != BPF_DIAG_HISTORY_SCOPE_REF) + return 0; + + for (i = log->cnt; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + if (event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE && + event->ref.ref_id == opts->ref_id) + return i - 1; + } + + return 0; +} + +static bool diag_history_event_visible(const struct bpf_diag_history_event *event, + const struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + + switch (event->kind) { + case BPF_DIAG_HISTORY_BRANCH: + return true; + case BPF_DIAG_HISTORY_MOD: + return filter->lineage_valid && event->in_lineage; + case BPF_DIAG_HISTORY_REF_ACQUIRE: + case BPF_DIAG_HISTORY_REF_RELEASE: + return opts->scope == BPF_DIAG_HISTORY_SCOPE_REF && + event->ref.ref_id == opts->ref_id; + case BPF_DIAG_HISTORY_CONTEXT: + return opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT && + event->ctx.kind == opts->ctx_kind; + default: + return false; + } +} + +static const char *diag_s64_bound_name(s64 value) +{ + if (value == S64_MIN) + return "S64_MIN"; + if (value == S64_MAX) + return "S64_MAX"; + return NULL; +} + +static const char *diag_u64_bound_name(u64 value) +{ + if (value == U64_MAX) + return "U64_MAX"; + return NULL; +} + +static const char *diag_s64_str(struct bpf_verifier_env *env, s64 value) +{ + return diag_s64_bound_name(value) ?: bpf_diag_fmt(env, "%lld", value); +} + +static const char *diag_u64_str(struct bpf_verifier_env *env, u64 value) +{ + return diag_u64_bound_name(value) ?: bpf_diag_fmt(env, "%llu", value); +} + +static bool diag_cnum64_unknown(struct cnum64 range) +{ + return cnum64_smin(range) == S64_MIN && cnum64_smax(range) == S64_MAX && + cnum64_umin(range) == 0 && cnum64_umax(range) == U64_MAX; +} + +static bool diag_snapshot_unknown(const struct bpf_diag_reg_snapshot *snapshot) +{ + return tnum_is_unknown(snapshot->var_off) && diag_cnum64_unknown(snapshot->r64); +} + +static const char *diag_scalar_range(struct bpf_verifier_env *env, struct cnum64 range) +{ + return bpf_diag_fmt(env, "signed range [%s, %s], unsigned range [%s, %s]", + diag_s64_str(env, cnum64_smin(range)), + diag_s64_str(env, cnum64_smax(range)), + diag_u64_str(env, cnum64_umin(range)), + diag_u64_str(env, cnum64_umax(range))); +} + +const char *bpf_diag_fmt_s64_sum(struct bpf_verifier_env *env, s64 value, int addend) +{ + s64 sum; + + if (check_add_overflow(value, (s64)addend, &sum)) + return bpf_diag_fmt(env, "%lld plus %d (%s)", value, addend, + addend < 0 ? "below S64_MIN" : "above S64_MAX"); + + return bpf_diag_fmt(env, "%lld", sum); +} + +static const char *diag_access_offset(struct bpf_verifier_env *env, int off, + const struct bpf_reg_state *reg) +{ + if (tnum_is_const(reg->var_off)) + return bpf_diag_fmt(env, "constant %s", + bpf_diag_fmt_s64_sum(env, (s64)reg->var_off.value, off)); + + if (tnum_is_unknown(reg->var_off) && diag_cnum64_unknown(reg->r64)) + return bpf_diag_fmt(env, "unbounded"); + + if (off) + return bpf_diag_fmt(env, + "variable: known bits %#llx, unknown mask %#llx, plus fixed offset %d; %s", + (u64)reg->var_off.value, reg->var_off.mask, off, + diag_scalar_range(env, reg->r64)); + return bpf_diag_fmt(env, "variable: known bits %#llx, unknown mask %#llx; %s", + (u64)reg->var_off.value, reg->var_off.mask, + diag_scalar_range(env, reg->r64)); +} + +void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const char *type_name, const char *proof, + int off, int size, u32 mem_size, const struct bpf_reg_state *reg) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const char *offset_desc; + + if (!bpf_diag_enabled(env)) + return; + + offset_desc = diag_access_offset(env, off, reg); + + bpf_diag_header(env, MEMORY_SAFETY, "access outside bounds"); + diag_reason( + env, "The verifier cannot prove offset + access_size <= object_size. Here, %s. %s is %s; offset is %s; access_size is %d; object_size is %u.", + proof, reg_name, type_name, offset_desc, size, mem_size); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "access may be outside object bounds"); + + if (regno >= 0) + diag_print_history(env, &opts); + + diag_suggestion( + env, "Add or adjust a bounds check that proves offset + access_size stays within the object."); +} + +static const char *diag_lock_name(const struct bpf_reference_state *lock) +{ + switch (lock->type) { + case REF_TYPE_LOCK: + return "bpf_spin_lock"; + case REF_TYPE_RES_LOCK: + return "resource spin lock"; + case REF_TYPE_RES_LOCK_IRQ: + return "IRQ-saving resource spin lock"; + default: + return "lock"; + } +} + +static void diag_res_report(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason) +{ + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); +} + +void bpf_diag_res(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion) +{ + diag_res_report(env, insn_idx, problem, reason); + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_lock(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, + const struct bpf_reference_state *active_lock) +{ + diag_res_report(env, insn_idx, problem, reason); + + if (active_lock) { + diag_section(env, "Active lock"); + bpf_diag_source(env, active_lock->insn_idx, "acquired", + "active %s has verifier identity %d", + diag_lock_name(active_lock), active_lock->id); + } + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, u32 depth) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = BPF_DIAG_CONTEXT_IRQ, + .ctx_depth = depth, + }; + + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + if (depth) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REF, + .ref_id = ref_id, + }; + + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, "unreleased resource"); + diag_reason( + env, "Owned resource (id=%u) was acquired at instruction %u and still needs to be released before this exit path.", + ref_id, alloc_insn); + + diag_section(env, "At"); + bpf_diag_source(env, fail_insn, "error", + "owned resource (id=%u) still needs release", ref_id); + + diag_print_history(env, &opts); + + diag_suggestion( + env, "Release or transfer ownership of the acquired resource on every path before the program exits."); +} + +static const char *diag_var_offset(struct bpf_verifier_env *env, + const struct bpf_diag_reg_snapshot *snapshot) +{ + if (tnum_is_const(snapshot->var_off)) + return bpf_diag_fmt(env, "at offset %lld", (s64)snapshot->var_off.value); + + if (diag_snapshot_unknown(snapshot)) + return bpf_diag_fmt(env, "with unknown offset"); + + return bpf_diag_fmt(env, + "with variable offset: known bits %#llx, unknown mask %#llx, %s", + snapshot->var_off.value, snapshot->var_off.mask, + diag_scalar_range(env, snapshot->r64)); +} + +static const char *diag_reg_map_name(const struct bpf_map *map) +{ + if (!map || !map->name[0]) + return NULL; + + return map->name; +} + +static const char *diag_reg_snapshot(struct bpf_verifier_env *env, + const struct bpf_diag_reg_snapshot *snapshot) +{ + const char *type_name = reg_type_str(env, snapshot->type); + const char *offset = diag_var_offset(env, snapshot); + const char *btf = snapshot->btf && snapshot->btf_id ? + bpf_diag_fmt_btf_type(env, snapshot->btf, snapshot->btf_id) : NULL; + const char *map_name; + + if (snapshot->type == SCALAR_VALUE) { + if (tnum_is_const(snapshot->var_off)) + return bpf_diag_fmt(env, "integer scalar value %lld", + (s64)snapshot->var_off.value); + if (diag_snapshot_unknown(snapshot)) + return bpf_diag_fmt(env, "integer scalar with unknown value"); + if (cnum64_is_const(snapshot->r64)) + return bpf_diag_fmt(env, "integer scalar value %lld", + cnum64_smin(snapshot->r64)); + return bpf_diag_fmt(env, "integer scalar with %s", + diag_scalar_range(env, snapshot->r64)); + } + + if (snapshot->type == NOT_INIT) + return bpf_diag_fmt(env, "uninitialized value"); + + if (base_type(snapshot->type) == PTR_TO_CTX) + return bpf_diag_fmt(env, "context pointer %s", offset); + + if (base_type(snapshot->type) == PTR_TO_STACK) + return bpf_diag_fmt(env, "stack pointer %s", offset); + + if (base_type(snapshot->type) == PTR_TO_MAP_VALUE) { + const char *kind = type_may_be_null(snapshot->type) ? "nullable map value" : + "map value"; + + map_name = diag_reg_map_name(snapshot->map_ptr); + if (map_name) + return bpf_diag_fmt(env, "%s from %s %s", kind, map_name, offset); + return bpf_diag_fmt(env, "%s %s", kind, offset); + } + + if (base_type(snapshot->type) == CONST_PTR_TO_MAP) { + map_name = diag_reg_map_name(snapshot->map_ptr); + if (map_name) + return bpf_diag_fmt(env, "map pointer for map %s", map_name); + return bpf_diag_fmt(env, "map pointer"); + } + + if (type_is_non_owning_ref(snapshot->type)) { + if (btf) + return bpf_diag_fmt(env, "borrowed allocated object pointer type=%s", btf); + return bpf_diag_fmt(env, "borrowed allocated object pointer"); + } + + if (type_is_ptr_alloc_obj(snapshot->type)) { + if (btf) + return bpf_diag_fmt(env, "owned allocated object pointer type=%s", btf); + return bpf_diag_fmt(env, "owned allocated object pointer"); + } + + if (base_type(snapshot->type) == PTR_TO_BTF_ID && btf) + return bpf_diag_fmt(env, "%s type=%s %s", type_name, btf, offset); + + return bpf_diag_fmt(env, "%s %s", type_name, offset); +} + +static const char *diag_mod_target_desc(struct bpf_verifier_env *env, + const struct bpf_diag_mod_target *target) +{ + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + return bpf_diag_fmt(env, "R%u", target->regno); + case BPF_DIAG_MOD_TARGET_STACK_ARG: + return bpf_diag_fmt(env, "*(R11-%u)", (target->stack_arg + 1) * BPF_REG_SIZE); + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + return bpf_diag_fmt(env, "stack slot fp%d", -(target->spi + 1) * BPF_REG_SIZE); + default: + return "value"; + } +} + +static void diag_print_mod(struct bpf_verifier_env *env, const struct bpf_diag_history_event *event) +{ + const struct bpf_diag_mod_target *target = &event->mod.target; + const char *target_desc, *reason = NULL, *old, *new; + const char *label = "update"; + + if (target->kind == BPF_DIAG_MOD_TARGET_STACK_RANGE) { + bpf_diag_source( + env, event->insn_idx, "invalidated", + "variable-offset stack write may affect bytes fp%d through fp%d", + target->range.min_off, target->range.max_off - 1); + return; + } + + old = diag_reg_snapshot(env, &event->mod.old); + new = diag_reg_snapshot(env, &event->mod.new); + target_desc = diag_mod_target_desc(env, target); + + switch (event->mod.reason) { + case BPF_DIAG_MOD_REF_RELEASE: + reason = target->kind == BPF_DIAG_MOD_TARGET_REG ? "resource release invalidated " + "this pointer" : + "resource release invalidated " + "this value"; + break; + case BPF_DIAG_MOD_PKT_DATA_CHANGE: + reason = "packet data may have moved"; + break; + case BPF_DIAG_MOD_NON_OWN_REF: + reason = "leaving the protected region invalidated this borrowed pointer"; + break; + case BPF_DIAG_MOD_CALLER_SAVED: + reason = target->kind == BPF_DIAG_MOD_TARGET_STACK_ARG ? + "call invalidated this outgoing stack argument" : + "call invalidated this caller-saved register"; + break; + case BPF_DIAG_MOD_WRITE: + if (target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT) + reason = "a later stack write overwrote this spilled value"; + break; + case BPF_DIAG_MOD_SPILL: + label = "spilled"; + break; + case BPF_DIAG_MOD_VAR_WRITE: + default: + break; + } + + if (reason) { + bpf_diag_source(env, event->insn_idx, "invalidated", + "%s: %s; previous value was %s", target_desc, reason, old); + return; + } + + bpf_diag_source(env, event->insn_idx, label, "%s changed from %s to %s", target_desc, + old, new); +} + +static void diag_print_ref_event(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + const char *label; + + label = event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE ? "acquired" : "released"; + bpf_diag_source(env, event->insn_idx, label, "owned resource (id=%u)", + event->ref.ref_id); +} + +static const char *diag_context_name(enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return "RCU read lock region"; + case BPF_DIAG_CONTEXT_PREEMPT: + return "non-preemptible region"; + case BPF_DIAG_CONTEXT_IRQ: + return "IRQ-disabled region"; + case BPF_DIAG_CONTEXT_LOCK: + return "lock region"; + case BPF_DIAG_CONTEXT_NONE: + default: + return "non-sleepable program"; + } +} + +static void diag_print_context_event(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + bpf_diag_source(env, event->insn_idx, "context", "%s %s; depth is now %u", + event->ctx.enter ? "entered" : "left", + diag_context_name(event->ctx.kind), event->ctx.depth); +} + +static void diag_print_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_opts *opts) +{ + const struct bpf_diag_history_event *event; + struct bpf_diag_history_filter filter = { + .opts = opts, + }; + struct bpf_diag_log *log; + struct diag_fmt_mark mark; + bool first = true; + int start_idx; + u32 i, visible_cnt = 0, visible_idx = 0; + + if (!bpf_diag_enabled(env)) + return; + + if (!env->diag) + return; + log = &env->diag->log; + + diag_build_lineage(env, log, &filter); + + start_idx = diag_history_start_idx(log, &filter); + for (i = start_idx; i < log->cnt; i++) { + event = &log->events[log_pos(log, i)]; + if (diag_history_event_visible(event, &filter)) + visible_cnt++; + } + + if (!visible_cnt && !log->first_seq && opts->scope == BPF_DIAG_HISTORY_SCOPE_STACK_ARG) + return; + + diag_section(env, "Causal path"); + mark = diag_fmt_save(env); + for (i = start_idx; i < log->cnt; i++) { + event = &log->events[log_pos(log, i)]; + if (!diag_history_event_visible(event, &filter)) + continue; + + diag_fmt_restore(env, mark); + if (visible_cnt > BPF_DIAG_HISTORY_RENDER_MAX && + visible_idx >= BPF_DIAG_HISTORY_RENDER_MAX / 2 && + visible_idx < visible_cnt - BPF_DIAG_HISTORY_RENDER_MAX / 2) { + if (visible_idx++ != BPF_DIAG_HISTORY_RENDER_MAX / 2) + continue; + if (!first) + diag_write(env, "\n"); + first = false; + diag_write(env, " %u intermediate causal-history events omitted\n", + visible_cnt - BPF_DIAG_HISTORY_RENDER_MAX); + continue; + } + visible_idx++; + + if (!first) + diag_write(env, "\n"); + first = false; + + switch (event->kind) { + case BPF_DIAG_HISTORY_BRANCH: + bpf_diag_source(env, event->insn_idx, "branch", + "took the %s branch of this conditional, goto %s", + event->branch.cond_true ? "true" : "false", + event->branch.cond_true ? "followed" : "not followed"); + break; + case BPF_DIAG_HISTORY_MOD: + diag_print_mod(env, event); + break; + case BPF_DIAG_HISTORY_REF_ACQUIRE: + case BPF_DIAG_HISTORY_REF_RELEASE: + diag_print_ref_event(env, event); + break; + case BPF_DIAG_HISTORY_CONTEXT: + diag_print_context_event(env, event); + break; + default: + break; + } + } + + if (!visible_cnt) + diag_write(env, " no retained diagnostic events on this path\n"); + if (log->first_seq) + diag_write(env, " %llu older causal-history event%s not retained because diagnostic " + "event storage reached capacity\n", + log->first_seq, log->first_seq == 1 ? "" : "s"); + diag_fmt_restore(env, mark); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h new file mode 100644 index 000000000000..d1b79945008a --- /dev/null +++ b/kernel/bpf/diagnostics.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#ifndef __BPF_DIAGNOSTICS_H +#define __BPF_DIAGNOSTICS_H + +#include <linux/bpf.h> +#include <linux/compiler_attributes.h> +#include <linux/stdarg.h> +#include <linux/types.h> + +struct bpf_reference_state; +struct bpf_func_state; +struct bpf_reg_state; +struct bpf_verifier_env; +struct bpf_verifier_state; +struct btf; + +const char *bpf_diag_fmt_s64_sum(struct bpf_verifier_env *env, s64 value, int addend); +enum bpf_diag_mod_reason { + BPF_DIAG_MOD_WRITE, + BPF_DIAG_MOD_SPILL, + BPF_DIAG_MOD_VAR_WRITE, + BPF_DIAG_MOD_REF_RELEASE, + BPF_DIAG_MOD_PKT_DATA_CHANGE, + BPF_DIAG_MOD_NON_OWN_REF, + BPF_DIAG_MOD_CALLER_SAVED, +}; + +enum bpf_diag_context_kind { + BPF_DIAG_CONTEXT_NONE, + BPF_DIAG_CONTEXT_RCU, + BPF_DIAG_CONTEXT_PREEMPT, + BPF_DIAG_CONTEXT_IRQ, + BPF_DIAG_CONTEXT_LOCK, +}; + +enum bpf_diag_invalid_deref_kind { + BPF_DIAG_DEREF_SCALAR, + BPF_DIAG_DEREF_NULLABLE_PTR, + BPF_DIAG_DEREF_MODIFIED_PTR, + BPF_DIAG_DEREF_INVALID_PTR, +}; + +bool bpf_diag_enabled(const struct bpf_verifier_env *env); +int bpf_diag_init(struct bpf_verifier_env *env); +void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state); +char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size); +const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) + __printf(2, 0); +const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id); +const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type); +u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); +void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); +u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state); +void bpf_diag_free(struct bpf_verifier_env *env); +void bpf_diag_register_type(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *problem, const char *reason, const char *suggestion); +void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const struct bpf_reg_state *reg, + enum bpf_diag_invalid_deref_kind kind, s64 offset); +void bpf_diag_unreadable_reg(struct bpf_verifier_env *env, u32 insn_idx, int regno); +void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int nargs, + int stack_arg_slot, const char *callee_name, + const char *arg_name); +void bpf_diag_memory(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion); +void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const char *type_name, const char *proof, + int off, int size, u32 mem_size, const struct bpf_reg_state *reg); +void bpf_diag_res(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion); +void bpf_diag_lock(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, + const struct bpf_reference_state *active_lock); +void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, u32 depth); +void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn); +void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno, + int stack_arg_slot, const char *call_name, const char *arg_name, + const char *reason, const char *suggestion); +void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, const char *suggestion); +void bpf_diag_ctx_active(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion); +void bpf_diag_ctx_required(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion); +void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, enum bpf_diag_context_kind ctx_kind, + const char *suggestion); +void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx, + const char *problem, const char *suggestion, + const char *reason_fmt, ...) __printf(5, 6); +void bpf_diag_policy(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + const char *reason, const char *suggestion); +void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); +void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); +void bpf_diag_mod_end(struct bpf_verifier_env *env); +void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + enum bpf_diag_mod_reason reason); +void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, + const struct bpf_func_state *state, s16 min_off, s16 max_off, + enum bpf_diag_mod_reason reason); +void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id); +void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id); +void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx, + enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth); + +#endif /* __BPF_DIAGNOSTICS_H */ diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 0391b3bc0073..50b3ca5149a0 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -139,7 +139,7 @@ static void print_bpf_end_insn(bpf_insn_print_t verbose, void *private_data, const struct bpf_insn *insn) { - verbose(private_data, "(%02x) r%d = %s%d r%d\n", + verbose(private_data, "(%02x) r%d = %s%d r%d", insn->code, insn->dst_reg, BPF_SRC(insn->code) == BPF_TO_BE ? "be" : "le", insn->imm, insn->dst_reg); @@ -149,7 +149,7 @@ static void print_bpf_bswap_insn(bpf_insn_print_t verbose, void *private_data, const struct bpf_insn *insn) { - verbose(private_data, "(%02x) r%d = bswap%d r%d\n", + verbose(private_data, "(%02x) r%d = bswap%d r%d", insn->code, insn->dst_reg, insn->imm, insn->dst_reg); } @@ -197,19 +197,19 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, else print_bpf_end_insn(verbose, cbs->private_data, insn); } else if (BPF_OP(insn->code) == BPF_NEG) { - verbose(cbs->private_data, "(%02x) %c%d = -%c%d\n", + verbose(cbs->private_data, "(%02x) %c%d = -%c%d", insn->code, class == BPF_ALU ? 'w' : 'r', insn->dst_reg, class == BPF_ALU ? 'w' : 'r', insn->dst_reg); } else if (is_addr_space_cast(insn)) { - verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %u, %u)\n", + verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %u, %u)", insn->code, insn->dst_reg, insn->src_reg, ((u32)insn->imm) >> 16, (u16)insn->imm); } else if (is_mov_percpu_addr(insn)) { - verbose(cbs->private_data, "(%02x) r%d = &(void __percpu *)(r%d)\n", + verbose(cbs->private_data, "(%02x) r%d = &(void __percpu *)(r%d)", insn->code, insn->dst_reg, insn->src_reg); } else if (BPF_SRC(insn->code) == BPF_X) { - verbose(cbs->private_data, "(%02x) %c%d %s %s%c%d\n", + verbose(cbs->private_data, "(%02x) %c%d %s %s%c%d", insn->code, class == BPF_ALU ? 'w' : 'r', insn->dst_reg, is_sdiv_smod(insn) ? bpf_alu_sign_string[BPF_OP(insn->code) >> 4] @@ -218,7 +218,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, class == BPF_ALU ? 'w' : 'r', insn->src_reg); } else { - verbose(cbs->private_data, "(%02x) %c%d %s %d\n", + verbose(cbs->private_data, "(%02x) %c%d %s %d", insn->code, class == BPF_ALU ? 'w' : 'r', insn->dst_reg, is_sdiv_smod(insn) ? bpf_alu_sign_string[BPF_OP(insn->code) >> 4] @@ -227,7 +227,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, } } else if (class == BPF_STX) { if (BPF_MODE(insn->code) == BPF_MEM) - verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d\n", + verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, @@ -235,7 +235,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, else if (BPF_MODE(insn->code) == BPF_ATOMIC && (insn->imm == BPF_ADD || insn->imm == BPF_AND || insn->imm == BPF_OR || insn->imm == BPF_XOR)) { - verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d\n", + verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, @@ -246,7 +246,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->imm == (BPF_AND | BPF_FETCH) || insn->imm == (BPF_OR | BPF_FETCH) || insn->imm == (BPF_XOR | BPF_FETCH))) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)\n", + verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)", insn->code, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_atomic_alu_string[BPF_OP(insn->imm) >> 4], @@ -254,7 +254,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->dst_reg, insn->off, insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_CMPXCHG) { - verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)\n", + verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)", insn->code, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], @@ -262,44 +262,44 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_XCHG) { - verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)\n", + verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)", insn->code, insn->src_reg, BPF_SIZE(insn->code) == BPF_DW ? "64" : "", bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_LOAD_ACQ) { - verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))\n", + verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))", insn->code, insn->dst_reg, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->src_reg, insn->off); } else if (BPF_MODE(insn->code) == BPF_ATOMIC && insn->imm == BPF_STORE_REL) { - verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)\n", + verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->src_reg); } else { - verbose(cbs->private_data, "BUG_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_%02x", insn->code); } } else if (class == BPF_ST) { if (BPF_MODE(insn->code) == BPF_MEM) { - verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d\n", + verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = %d", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->dst_reg, insn->off, insn->imm); } else if (BPF_MODE(insn->code) == 0xc0 /* BPF_NOSPEC, no UAPI */) { - verbose(cbs->private_data, "(%02x) nospec\n", insn->code); + verbose(cbs->private_data, "(%02x) nospec", insn->code); } else { - verbose(cbs->private_data, "BUG_st_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_st_%02x", insn->code); } } else if (class == BPF_LDX) { if (BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) { - verbose(cbs->private_data, "BUG_ldx_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_ldx_%02x", insn->code); return; } - verbose(cbs->private_data, "(%02x) r%d = *(%s *)(r%d %+d)\n", + verbose(cbs->private_data, "(%02x) r%d = *(%s *)(r%d %+d)", insn->code, insn->dst_reg, BPF_MODE(insn->code) == BPF_MEM ? bpf_ldst_string[BPF_SIZE(insn->code) >> 3] : @@ -307,12 +307,12 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg, insn->off); } else if (class == BPF_LD) { if (BPF_MODE(insn->code) == BPF_ABS) { - verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[%d]\n", + verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[%d]", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->imm); } else if (BPF_MODE(insn->code) == BPF_IND) { - verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[r%d + %d]\n", + verbose(cbs->private_data, "(%02x) r0 = *(%s *)skb[r%d + %d]", insn->code, bpf_ldst_string[BPF_SIZE(insn->code) >> 3], insn->src_reg, insn->imm); @@ -332,12 +332,12 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, if (is_ptr && !allow_ptr_leaks) imm = 0; - verbose(cbs->private_data, "(%02x) r%d = %s\n", + verbose(cbs->private_data, "(%02x) r%d = %s", insn->code, insn->dst_reg, __func_imm_name(cbs, insn, imm, tmp, sizeof(tmp))); } else { - verbose(cbs->private_data, "BUG_ld_%02x\n", insn->code); + verbose(cbs->private_data, "BUG_ld_%02x", insn->code); return; } } else if (class == BPF_JMP32 || class == BPF_JMP) { @@ -347,35 +347,35 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, char tmp[64]; if (insn->src_reg == BPF_PSEUDO_CALL) { - verbose(cbs->private_data, "(%02x) call pc%s\n", + verbose(cbs->private_data, "(%02x) call pc%s", insn->code, __func_get_name(cbs, insn, tmp, sizeof(tmp))); } else { strcpy(tmp, "unknown"); - verbose(cbs->private_data, "(%02x) call %s#%d\n", insn->code, + verbose(cbs->private_data, "(%02x) call %s#%d", insn->code, __func_get_name(cbs, insn, tmp, sizeof(tmp)), insn->imm); } } else if (insn->code == (BPF_JMP | BPF_JA)) { - verbose(cbs->private_data, "(%02x) goto pc%+d\n", + verbose(cbs->private_data, "(%02x) goto pc%+d", insn->code, insn->off); } else if (insn->code == (BPF_JMP | BPF_JA | BPF_X)) { - verbose(cbs->private_data, "(%02x) gotox r%d\n", + verbose(cbs->private_data, "(%02x) gotox r%d", insn->code, insn->dst_reg); } else if (insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_MAY_GOTO) { - verbose(cbs->private_data, "(%02x) may_goto pc%+d\n", + verbose(cbs->private_data, "(%02x) may_goto pc%+d", insn->code, insn->off); } else if (insn->code == (BPF_JMP32 | BPF_JA)) { - verbose(cbs->private_data, "(%02x) gotol pc%+d\n", + verbose(cbs->private_data, "(%02x) gotol pc%+d", insn->code, insn->imm); } else if (insn->code == (BPF_JMP | BPF_EXIT)) { - verbose(cbs->private_data, "(%02x) exit\n", insn->code); + verbose(cbs->private_data, "(%02x) exit", insn->code); } else if (BPF_SRC(insn->code) == BPF_X) { verbose(cbs->private_data, - "(%02x) if %c%d %s %c%d goto pc%+d\n", + "(%02x) if %c%d %s %c%d goto pc%+d", insn->code, class == BPF_JMP32 ? 'w' : 'r', insn->dst_reg, bpf_jmp_string[BPF_OP(insn->code) >> 4], @@ -383,14 +383,14 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, insn->src_reg, insn->off); } else { verbose(cbs->private_data, - "(%02x) if %c%d %s 0x%x goto pc%+d\n", + "(%02x) if %c%d %s 0x%x goto pc%+d", insn->code, class == BPF_JMP32 ? 'w' : 'r', insn->dst_reg, bpf_jmp_string[BPF_OP(insn->code) >> 4], (u32)insn->imm, insn->off); } } else { - verbose(cbs->private_data, "(%02x) %s\n", + verbose(cbs->private_data, "(%02x) %s", insn->code, bpf_class_string[class]); } } diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index ea2d60dc1fee..79f0c222c583 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -148,7 +148,10 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false); if (!d->image) goto out; - d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); + /* d->rw_image doesn't need to be in module memory range, so we + * can use vzalloc. + */ + d->rw_image = vzalloc(PAGE_SIZE); if (!d->rw_image) { bpf_prog_pack_free(d->image, PAGE_SIZE); d->image = NULL; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 3cf2cc6e3ab6..65b441e4a351 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -20,6 +20,26 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) insn->imm == BPF_CMPXCHG; } +/* Returns true if 'insn' is an address space cast instruction translated as BPF_ALU op */ +static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *insn) +{ + struct bpf_map *arena = (struct bpf_map *)prog->aux->arena; + + if (insn->code != (BPF_ALU64 | BPF_MOV | BPF_X) || insn->off != BPF_ADDR_SPACE_CAST) + return false; + + /* cast from as(1) to as(0) */ + if (insn->imm == 1) + return true; + + /* cast from as(0) to as(1) */ + if (insn->imm == 1 << 16) + return arena && arena->map_flags & BPF_F_NO_USER_CONV; + + /* non-BPF_F_NO_USER_CONV cast from as(0) to as(1) should be handled by JIT */ + return false; +} + /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -29,30 +49,66 @@ static int insn_def_regno(const struct bpf_insn *insn) case BPF_ST: return -1; case BPF_STX: - if (BPF_MODE(insn->code) == BPF_ATOMIC || - BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { - if (insn->imm == BPF_CMPXCHG) - return BPF_REG_0; - else if (insn->imm == BPF_LOAD_ACQ) - return insn->dst_reg; - else if (insn->imm & BPF_FETCH) - return insn->src_reg; - } - return -1; + return bpf_atomic_load_reg(insn); default: return insn->dst_reg; } } -/* Return TRUE if INSN has defined any 32-bit value explicitly. */ -static bool insn_has_def32(struct bpf_insn *insn) +/* + * For use only in combination with insn_def_regno() >= 0. + * Returns TRUE if the destination register operates on 64-bit, + * otherwise return FALSE. + */ +static bool bpf_is_reg64(struct bpf_prog *prog, struct bpf_insn *insn) { - int dst_reg = insn_def_regno(insn); + u8 class = BPF_CLASS(insn->code); + u8 mode = BPF_MODE(insn->code); + u8 size = BPF_SIZE(insn->code); + u8 op = BPF_OP(insn->code); + bool mode_mem; + + /* subregister endiness swap */ + if ((class == BPF_ALU || class == BPF_ALU64) && op == BPF_END && insn->imm != 64) + return false; + + /* w0 += 1 */ + if (class == BPF_ALU && op != BPF_END) + return false; + + /* address space casts converted to BPF_ALU, see bpf_do_misc_fixups() */ + if (is_addr_space_cast32(prog, insn)) + return false; - if (dst_reg == -1) + /* non 64-bit, non signed extended loads */ + mode_mem = mode == BPF_MEM || mode == BPF_PROBE_MEM || mode == BPF_PROBE_MEM32; + if (class == BPF_LDX && mode_mem && size != BPF_DW) return false; - return !bpf_is_reg64(insn, dst_reg, NULL, DST_OP); + /* atomics, see insn_def_regno() */ + if (class == BPF_STX && size != BPF_DW) + return false; + + /* both LD_IND and LD_ABS return 32-bit data. */ + if (class == BPF_LD && (mode == BPF_IND || mode == BPF_ABS)) + return false; + + /* Conservatively return true at default. */ + return true; +} + +/* + * Return the 32-bit subregister defined by INSN, or -1 if INSN does not + * explicitly define a 32-bit value. + */ +int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn) +{ + int dst_reg = insn_def_regno(insn); + + if (dst_reg < 0 || bpf_is_reg64(prog, insn)) + return -1; + + return dst_reg; } static int kfunc_desc_cmp_by_imm_off(const void *a, const void *b) @@ -169,11 +225,12 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the * original insn at old prog. */ - data[off].zext_dst = insn_has_def32(insn + off + cnt - 1); + data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; if (cnt == 1) return; prog_len = new_prog->len; + env->insn_aux_data_len = prog_len; memmove(data + off + cnt - 1, data + off, sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); @@ -181,7 +238,7 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, for (i = off; i < off + cnt - 1; i++) { /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; - data[i].zext_dst = insn_has_def32(insn + i); + data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; } /* @@ -345,13 +402,17 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, sizeof(*env->subprog_info) * move); env->subprog_cnt -= j - i; - /* remove func_info */ + /* remove func_info and its aux */ if (aux->func_info) { move = aux->func_info_cnt - j; memmove(aux->func_info + i, aux->func_info + j, sizeof(*aux->func_info) * move); + if (aux->func_info_aux) + memmove(aux->func_info_aux + i, + aux->func_info_aux + j, + sizeof(*aux->func_info_aux) * move); aux->func_info_cnt -= j - i; /* func_info->insn_off is set after all code rewrites, * in adjust_btf_func() - no need to adjust @@ -440,7 +501,6 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; - struct bpf_insn *insns = env->prog->insnsi; int end = start + len; int i; @@ -449,9 +509,6 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) kvfree(aux_data[i].jt); aux_data[i].jt = NULL; } - - if (bpf_is_ldimm64(&insns[i])) - i++; } } @@ -464,7 +521,6 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); - /* Should be called before bpf_remove_insns, as it uses prog->insnsi */ bpf_clear_insn_aux_data(env, off, cnt); err = bpf_remove_insns(env->prog, off, cnt); @@ -483,6 +539,7 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) memmove(aux_data + off, aux_data + off + cnt, sizeof(*aux_data) * (orig_prog_len - off - cnt)); + env->insn_aux_data_len -= cnt; return 0; } @@ -616,11 +673,7 @@ int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, if (load_reg == -1) continue; - /* NOTE: arg "reg" (the fourth one) is only used for - * BPF_STX + SRC_OP, so it is safe to pass NULL - * here. - */ - if (bpf_is_reg64(&insn, load_reg, NULL, DST_OP)) { + if (bpf_is_reg64(env->prog, &insn)) { if (class == BPF_LD && BPF_MODE(code) == BPF_IMM) i++; @@ -759,6 +812,7 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) for (i = 0; i < insn_cnt; i++, insn++) { bpf_convert_ctx_access_t convert_ctx_access; + enum bpf_reg_type ptr_type; u8 mode; if (env->insn_aux_data[i + delta].nospec) { @@ -851,7 +905,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) continue; } - switch ((int)env->insn_aux_data[i + delta].ptr_type) { + ptr_type = env->insn_aux_data[i + delta].ptr_type; + switch ((int)ptr_type) { case PTR_TO_CTX: if (!ops->convert_ctx_access) continue; @@ -867,26 +922,6 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) case PTR_TO_XDP_SOCK: convert_ctx_access = bpf_xdp_sock_convert_ctx_access; break; - case PTR_TO_BTF_ID: - case PTR_TO_BTF_ID | PTR_UNTRUSTED: - /* PTR_TO_BTF_ID | MEM_ALLOC always has a valid lifetime, unlike - * PTR_TO_BTF_ID, and an active referenced id, but the same cannot - * be said once it is marked PTR_UNTRUSTED, hence we must handle - * any faults for loads into such types. BPF_WRITE is disallowed - * for this case. - */ - case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED: - case PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED: - if (type == BPF_READ) { - if (BPF_MODE(insn->code) == BPF_MEM) - insn->code = BPF_LDX | BPF_PROBE_MEM | - BPF_SIZE((insn)->code); - else - insn->code = BPF_LDX | BPF_PROBE_MEMSX | - BPF_SIZE((insn)->code); - env->prog->aux->num_exentries++; - } - continue; case PTR_TO_ARENA: if (BPF_MODE(insn->code) == BPF_MEMSX) { if (!bpf_jit_supports_insn(insn, true)) { @@ -900,6 +935,29 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) env->prog->aux->num_exentries++; continue; default: + /* + * A pointer which may fault on a dereference must not + * be loaded from without fault protection, hence turn + * the BPF_LDX into a BPF_PROBE_MEM one so that a bad + * address is handled rather than panicking the kernel. + * A store through one is rejected earlier, there is no + * probed counterpart to rewrite it into. + */ + if (bpf_is_ptr_to_mem_or_btf_id(ptr_type) && + bpf_may_fault_on_deref(ptr_type) && + type == BPF_READ) { + if (BPF_MODE(insn->code) == BPF_MEM) + insn->code = BPF_LDX | BPF_PROBE_MEM | + BPF_SIZE(insn->code); + else + insn->code = BPF_LDX | BPF_PROBE_MEMSX | + BPF_SIZE(insn->code); + env->prog->aux->num_exentries++; + continue; + } + if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, + "access to a fault prone pointer is not rewritten as a probed one")) + return -EFAULT; continue; } @@ -1005,26 +1063,6 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s env->subprog_info[env->subprog_cnt].start = env->prog->len; } -struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) -{ - size_t size; - void *new_aux; - - size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len); - new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT); - if (new_aux) - memcpy(new_aux, env->insn_aux_data, size); - return new_aux; -} - -void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux) -{ - /* the expanded elements are zero-filled, so no special handling is required */ - vfree(env->insn_aux_data); - env->insn_aux_data = orig_insn_aux; -} - static int jit_subprogs(struct bpf_verifier_env *env) { struct bpf_prog *prog = env->prog, **func, *tmp; @@ -1299,7 +1337,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) bool blinded = false; struct bpf_insn *insn; struct bpf_prog *prog, *orig_prog; - struct bpf_insn_aux_data *orig_insn_aux; u32 *orig_subprog_starts; if (env->subprog_cnt <= 1) @@ -1307,14 +1344,8 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) prog = orig_prog = env->prog; if (bpf_prog_need_blind(prog)) { - orig_insn_aux = bpf_dup_insn_aux_data(env); - if (!orig_insn_aux) { - err = -ENOMEM; - goto out_cleanup; - } orig_subprog_starts = bpf_dup_subprog_starts(env); if (!orig_subprog_starts) { - vfree(orig_insn_aux); err = -ENOMEM; goto out_cleanup; } @@ -1334,7 +1365,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) if (blinded) { bpf_jit_prog_release_other(prog, orig_prog); kvfree(orig_subprog_starts); - vfree(orig_insn_aux); } return 0; @@ -1364,7 +1394,6 @@ out_jit_err: out_restore: bpf_restore_subprog_starts(env, orig_subprog_starts); - bpf_restore_insn_aux_data(env, orig_insn_aux); kvfree(orig_subprog_starts); out_cleanup: /* cleanup main prog to be interpreted */ @@ -1378,7 +1407,6 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) #ifndef CONFIG_BPF_JIT_ALWAYS_ON struct bpf_prog *prog = env->prog; struct bpf_insn *insn = prog->insnsi; - bool has_kfunc_call = bpf_prog_has_kfunc_call(prog); int depth; #endif int i, err = 0; @@ -1404,8 +1432,8 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) return err; } #ifndef CONFIG_BPF_JIT_ALWAYS_ON - if (has_kfunc_call) { - verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); + if (prog->jit_required) { + verbose(env, "program requires BPF JIT compiler but it is not available\n"); return -EINVAL; } for (i = 0; i < env->subprog_cnt; i++) { @@ -1447,7 +1475,6 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env) return err; } - /* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */ static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) { @@ -1514,15 +1541,12 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } for (i = 0; i < insn_cnt;) { - if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) { - if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) || - (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) { - /* convert to 32-bit mov that clears upper 32-bit */ - insn->code = BPF_ALU | BPF_MOV | BPF_X; - /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ - insn->off = 0; - insn->imm = 0; - } /* cast from as(0) to as(1) should be handled by JIT */ + if (is_addr_space_cast32(env->prog, insn)) { + /* convert to 32-bit mov that clears upper 32-bit */ + insn->code = BPF_ALU | BPF_MOV | BPF_X; + /* clear off and imm, so it's a normal 'wX = wY' from JIT pov */ + insn->off = 0; + insn->imm = 0; goto next_insn; } @@ -1819,6 +1843,43 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } + if (bpf_jit_supports_percpu_insn() && + insn->code == (BPF_LD | BPF_IMM | BPF_DW) && + (insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) { + struct bpf_map *map; + + aux = &env->insn_aux_data[i + delta]; + map = env->used_maps[aux->map_index]; + if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) + goto next_insn; + + prog->jit_required = true; + + /* + * We are *skipping* first half of ld_imm64 insn + * with 'i++;', patching over second half of it + * with that same half + mov64_percpu_reg insn. + * All because bpf_patch_insn_data() can only + * replace one 8-byte insn, which does not work + * well for ld_imm64 insn. + */ + + insn_buf[0] = insn[1]; + insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); + cnt = 2; + + i++; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = prog = new_prog; + insn = new_prog->insnsi + i + delta; + goto next_insn; + } + if (insn->code != (BPF_JMP | BPF_CALL)) goto next_insn; if (insn->src_reg == BPF_PSEUDO_CALL) @@ -1841,8 +1902,10 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } /* Skip inlining the helper call if the JIT does it. */ - if (bpf_jit_inlines_helper_call(insn->imm)) + if (bpf_jit_inlines_helper_call(insn->imm)) { + prog->jit_required = 1; goto next_insn; + } if (insn->imm == BPF_FUNC_get_route_realm) prog->dst_needed = 1; @@ -2007,6 +2070,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) return -EFAULT; } + if (bpf_map_is_percpu_map(map_ptr->map_type)) + prog->jit_required = true; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -2111,6 +2177,7 @@ patch_map_ops_generic: * way, it's fine to back out this inlining logic */ #ifdef CONFIG_SMP + prog->jit_required = true; insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)&cpu_number); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, 0); @@ -2132,6 +2199,7 @@ patch_map_ops_generic: /* Implement bpf_get_current_task() and bpf_get_current_task_btf() inline. */ if ((insn->imm == BPF_FUNC_get_current_task || insn->imm == BPF_FUNC_get_current_task_btf) && bpf_verifier_inlines_helper_call(env, insn->imm)) { + prog->jit_required = true; insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, (u32)(unsigned long)¤t_task); insn_buf[1] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); @@ -2338,7 +2406,7 @@ patch_call_imm: func_id_name(insn->imm), insn->imm); return -EFAULT; } - insn->imm = fn->func - __bpf_call_base; + insn->imm = BPF_CALL_IMM(fn->func); next_insn: if (subprogs[cur_subprog + 1].start == i + delta + 1) { subprogs[cur_subprog].stack_depth += stack_depth_extra; diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 9f394e1aa2e8..d40cb5dd446c 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -998,7 +998,6 @@ static void dec_elem_count(struct bpf_htab *htab) atomic_dec(&htab->count); } - static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) { htab_put_fd_value(htab, l); @@ -2970,7 +2969,6 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v return 0; } - static long rhtab_map_delete_elem(struct bpf_map *map, void *key) { struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map); diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index c18f1e16edee..b3cc5c8fc875 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -278,7 +278,7 @@ const struct bpf_func_proto bpf_get_current_comm_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, }; #if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK) @@ -539,7 +539,7 @@ const struct bpf_func_proto bpf_strtol_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(s64), @@ -567,7 +567,7 @@ const struct bpf_func_proto bpf_strtoul_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(u64), @@ -583,7 +583,7 @@ static const struct bpf_func_proto bpf_strncmp_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_CONST_STR, }; @@ -627,7 +627,7 @@ const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto = { .arg1_type = ARG_ANYTHING, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto bpf_get_raw_smp_processor_id_proto = { @@ -653,7 +653,7 @@ const struct bpf_func_proto bpf_event_output_data_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_copy_from_user, void *, dst, u32, size, @@ -675,7 +675,7 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -706,7 +706,7 @@ const struct bpf_func_proto bpf_copy_from_user_task_proto = { .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_BTF_ID, .arg4_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], @@ -1093,10 +1093,10 @@ const struct bpf_func_proto bpf_snprintf_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx) @@ -1888,7 +1888,7 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL | MEM_UNINIT | MEM_WRITE, }; @@ -1943,7 +1943,7 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_DYNPTR, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, @@ -2004,7 +2004,7 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = { .arg1_type = ARG_PTR_TO_DYNPTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, .arg5_type = ARG_ANYTHING, }; @@ -3395,19 +3395,21 @@ __bpf_kfunc void bpf_throw(u64 cookie) WARN_ON_ONCE(!ctx.aux->exception_boundary); WARN_ON_ONCE(!ctx.bp); WARN_ON_ONCE(!ctx.cnt); - /* Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning + /* + * Prevent KASAN false positives for CONFIG_KASAN_STACK by unpoisoning * deeper stack depths than ctx.sp as we do not return from bpf_throw, - * which skips compiler generated instrumentation to do the same. + * which skips compiler generated instrumentation to do the same. Some + * architectures cannot recover sp while unwinding, so fall back to bp. */ - kasan_unpoison_task_stack_below((void *)(long)ctx.sp); + kasan_unpoison_task_stack_below((void *)(long)(ctx.sp ?: ctx.bp)); ctx.aux->bpf_exception_cb(cookie, ctx.sp + ctx.aux->stack_arg_sp_adjust, ctx.bp, 0, 0); WARN(1, "A call to BPF exception callback should never return\n"); } -__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__map, unsigned int flags) +__bpf_kfunc int bpf_wq_init(struct bpf_wq *wq, void *p__const_map, unsigned int flags) { struct bpf_async_kern *async = (struct bpf_async_kern *)wq; - struct bpf_map *map = p__map; + struct bpf_map *map = p__const_map; BUILD_BUG_ON(sizeof(struct bpf_async_kern) > sizeof(struct bpf_wq)); BUILD_BUG_ON(__alignof__(struct bpf_async_kern) != __alignof__(struct bpf_wq)); @@ -4388,7 +4390,7 @@ struct bpf_task_work_ctx { struct bpf_map *map; void *map_val; enum task_work_notify_mode mode; - bpf_task_work_callback_t callback_fn; + bpf_callback_t callback_fn; struct rcu_head rcu; } __aligned(8); @@ -4471,7 +4473,8 @@ static void bpf_task_work_callback(struct callback_head *cb) key = (void *)map_key_from_value(ctx->map, ctx->map_val, &idx); migrate_disable(); - ctx->callback_fn(ctx->map, key, ctx->map_val); + ctx->callback_fn((u64)(long)ctx->map, (u64)(long)key, + (u64)(long)ctx->map_val, 0, 0); migrate_enable(); bpf_task_work_ctx_reset(ctx); @@ -4594,7 +4597,7 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work } static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work *tw, - struct bpf_map *map, bpf_task_work_callback_t callback_fn, + struct bpf_map *map, void *callback_fn, struct bpf_prog_aux *aux, enum task_work_notify_mode mode) { struct bpf_prog *prog; @@ -4619,7 +4622,7 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work } ctx->task = task; - ctx->callback_fn = callback_fn; + ctx->callback_fn = (bpf_callback_t)callback_fn; ctx->prog = prog; ctx->mode = mode; ctx->map = map; @@ -4642,17 +4645,17 @@ release_prog: * mode * @task: Task struct for which callback should be scheduled * @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping - * @map__map: bpf_map that embeds struct bpf_task_work in the values + * @map__const_map: bpf_map that embeds struct bpf_task_work in the values * @callback: pointer to BPF subprogram to call * @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier * * Return: 0 if task work has been scheduled successfully, negative error code otherwise */ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { - return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_SIGNAL); + return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_SIGNAL); } /** @@ -4660,17 +4663,17 @@ __bpf_kfunc int bpf_task_work_schedule_signal(struct task_struct *task, struct b * mode * @task: Task struct for which callback should be scheduled * @tw: Pointer to struct bpf_task_work in BPF map value for internal bookkeeping - * @map__map: bpf_map that embeds struct bpf_task_work in the values + * @map__const_map: bpf_map that embeds struct bpf_task_work in the values * @callback: pointer to BPF subprogram to call * @aux: pointer to bpf_prog_aux of the caller BPF program, implicitly set by the verifier * * Return: 0 if task work has been scheduled successfully, negative error code otherwise */ __bpf_kfunc int bpf_task_work_schedule_resume(struct task_struct *task, struct bpf_task_work *tw, - void *map__map, bpf_task_work_callback_t callback, + void *map__const_map, bpf_task_work_callback_t callback, struct bpf_prog_aux *aux) { - return bpf_task_work_schedule(task, tw, map__map, callback, aux, TWA_RESUME); + return bpf_task_work_schedule(task, tw, map__const_map, callback, aux, TWA_RESUME); } static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep, @@ -4811,30 +4814,32 @@ BTF_ID_FLAGS(func, bpf_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_obj_drop_impl, KF_RELEASE) BTF_ID_FLAGS(func, bpf_percpu_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_percpu_obj_drop_impl, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_refcount_acquire, KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, KF_ACQUIRE | KF_RET_NULL | KF_RCU) -BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_push_front_impl) -BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_push_back_impl) -BTF_ID_FLAGS(func, bpf_list_add, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_is_first) -BTF_ID_FLAGS(func, bpf_list_is_last) -BTF_ID_FLAGS(func, bpf_list_empty) +BTF_ID_FLAGS(func, bpf_refcount_acquire, + KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, + KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_front_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_back_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_add, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_is_first, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_is_last, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_empty, KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_task_release, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_add, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_rbtree_add_impl) -BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_left, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_right, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_add, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_add_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_left, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_right, KF_RET_NULL | KF_SPINLOCK_SAFE) #ifdef CONFIG_CGROUPS BTF_ID_FLAGS(func, bpf_cgroup_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) @@ -4868,7 +4873,6 @@ static const struct btf_kfunc_id_set generic_kfunc_set = { .set = &generic_btf_ids, }; - BTF_ID_LIST(generic_dtor_ids) BTF_ID(struct, task_struct) BTF_ID(func, bpf_task_release_dtor) @@ -4884,9 +4888,9 @@ BTF_ID_FLAGS(func, bpf_rcu_read_lock) BTF_ID_FLAGS(func, bpf_rcu_read_unlock) BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL) BTF_ID_FLAGS(func, bpf_dynptr_slice_rdwr, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_iter_num_new, KF_ITER_NEW) -BTF_ID_FLAGS(func, bpf_iter_num_next, KF_ITER_NEXT | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_iter_num_destroy, KF_ITER_DESTROY) +BTF_ID_FLAGS(func, bpf_iter_num_new, KF_ITER_NEW | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_iter_num_next, KF_ITER_NEXT | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_iter_num_destroy, KF_ITER_DESTROY | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_iter_task_vma_new, KF_ITER_NEW | KF_RCU) BTF_ID_FLAGS(func, bpf_iter_task_vma_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_task_vma_destroy, KF_ITER_DESTROY) @@ -4961,8 +4965,8 @@ BTF_ID_FLAGS(func, bpf_strncasestr); #if defined(CONFIG_BPF_LSM) && defined(CONFIG_CGROUPS) BTF_ID_FLAGS(func, bpf_cgroup_read_xattr, KF_RCU) #endif -BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 0aadfbae0acc..74fc4b3f80d6 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -269,7 +269,6 @@ bpf_insn_successors(struct bpf_verifier_env *env, u32 idx) __diag_pop(); - static inline bool update_insn(struct bpf_verifier_env *env, struct func_instance *instance, u32 frame, u32 insn_idx) { @@ -497,7 +496,6 @@ static void print_instance(struct bpf_verifier_env *env, struct func_instance *i pos = env->log.end_pos; verbose(env, "%3d: ", insn_idx); bpf_verbose_insn(env, &insns[insn_idx]); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); /* remove \n */ insn_pos = env->log.end_pos; verbose(env, "%*c;", bpf_vlog_alignment(insn_pos - pos), ' '); pos = env->log.end_pos; @@ -1043,7 +1041,6 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i if (!printed) { verbose(env, "%3d: ", idx); bpf_verbose_insn(env, insn); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); printed = true; } verbose(env, "\tr%d: ", i); verbose_arg_track(env, &at_in[i]); @@ -1058,7 +1055,6 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i if (!printed) { verbose(env, "%3d: ", idx); bpf_verbose_insn(env, insn); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); printed = true; } verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]); @@ -1070,7 +1066,6 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i if (!printed) { verbose(env, "%3d: ", idx); bpf_verbose_insn(env, insn); - bpf_vlog_reset(&env->log, env->log.end_pos - 1); printed = true; } verbose(env, "\tfp%+d: ", -(i + 1) * 8); verbose_arg_track(env, &at_stack_in[i]); @@ -1213,12 +1208,9 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, clear_stack_for_all_offs(insn, at_out, insn->dst_reg, at_stack_out, sz); - if (insn->imm == BPF_CMPXCHG) - at_out[BPF_REG_0] = none; - else if (insn->imm == BPF_LOAD_ACQ) - *dst = none; - else if (insn->imm & BPF_FETCH) - *src = none; + r = bpf_atomic_load_reg(insn); + if (r >= 0) + at_out[r] = none; } } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) { u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code)); @@ -1545,6 +1537,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env, verbose(env, "%3d: ", idx); bpf_verbose_insn(env, &insns[idx]); + verbose(env, "\n"); /* Collect what needs printing */ if (is_ldx_stx_call && @@ -1868,7 +1861,6 @@ static int analyze_subprog(struct bpf_verifier_env *env, if (need_resched()) cond_resched(); - /* * When an instance is reused (must_write_initialized == true), * record into a fresh instance and merge afterward. This avoids @@ -2050,29 +2042,38 @@ out: /* Each field is a register bitmask */ struct insn_live_regs { - u16 use; /* registers read by instruction */ - u16 def; /* registers written by instruction */ - u16 in; /* registers that may be alive before instruction */ - u16 out; /* registers that may be alive after instruction */ + u32 use; /* registers read by instruction */ + u32 def; /* registers written by instruction */ + u32 in; /* registers that may be alive before instruction */ + u32 out; /* registers that may be alive after instruction */ }; /* Bitmask with 1s for all caller saved registers */ #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) +static inline u32 reg32_mask(u32 n) { return BIT(n); } +static inline u32 reg64_mask(u32 n) { return BIT(n) | BIT(n + 16); } +static inline u32 mask_widen(u32 m) { return m | (m << 16); } +static inline u16 mask_lo(u32 m) { return (u16)m; } +static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } + /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, struct insn_live_regs *info) { struct bpf_call_summary cs; - u8 class = BPF_CLASS(insn->code); - u8 code = BPF_OP(insn->code); - u8 mode = BPF_MODE(insn->code); - u16 src = BIT(insn->src_reg); - u16 dst = BIT(insn->dst_reg); - u16 r0 = BIT(0); - u16 def = 0; - u16 use = 0xffff; + const u8 class = BPF_CLASS(insn->code); + const u8 code = BPF_OP(insn->code); + const u8 mode = BPF_MODE(insn->code); + const u8 size = BPF_SIZE(insn->code); + const u32 src = reg64_mask(insn->src_reg); + const u32 dst = reg64_mask(insn->dst_reg); + const u32 src32 = mask_lo(src); + const u32 dst32 = mask_lo(dst); + const u32 r0 = reg64_mask(0); + u32 def = 0; + u32 use = U32_MAX; switch (class) { case BPF_LD: @@ -2083,8 +2084,8 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, use = 0; } break; - case BPF_LD | BPF_ABS: - case BPF_LD | BPF_IND: + case BPF_ABS: + case BPF_IND: /* stick with defaults */ break; } @@ -2092,7 +2093,15 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, case BPF_LDX: switch (mode) { case BPF_MEM: + /* a narrow load still redefines the whole register */ + def = dst; + use = src; + break; case BPF_MEMSX: + /* + * sign extension defines the whole register; + * src holds a pointer, hence is used as 64-bit. + */ def = dst; use = src; break; @@ -2110,12 +2119,19 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, switch (mode) { case BPF_MEM: def = 0; - use = dst | src; + use = dst | (size == BPF_DW ? src : src32); break; - case BPF_ATOMIC: + case BPF_ATOMIC: { + /* + * dst holds a pointer and is always used as 64-bit; + * the value operand and r0 are read as 32-bit for BPF_W atomics. + */ + u32 srcv = size == BPF_DW ? src : src32; + u32 r0v = size == BPF_DW ? r0 : mask_lo(r0); + switch (insn->imm) { case BPF_CMPXCHG: - use = r0 | dst | src; + use = r0v | dst | srcv; def = r0; break; case BPF_LOAD_ACQ: @@ -2124,10 +2140,10 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, break; case BPF_STORE_REL: def = 0; - use = dst | src; + use = dst | srcv; break; default: - use = dst | src; + use = dst | srcv; if (insn->imm & BPF_FETCH) def = src; else @@ -2135,6 +2151,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, } break; } + } break; case BPF_ALU: case BPF_ALU64: @@ -2148,14 +2165,14 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, if (BPF_SRC(insn->code) == BPF_K) use = 0; else - use = src; + use = class == BPF_ALU64 ? src : src32; break; default: def = dst; if (BPF_SRC(insn->code) == BPF_K) - use = dst; + use = class == BPF_ALU64 ? dst : dst32; else - use = dst | src; + use = class == BPF_ALU64 ? (dst | src) : (dst32 | src32); } break; case BPF_JMP: @@ -2181,13 +2198,14 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, use = def & ~BIT(BPF_REG_0); if (bpf_get_call_summary(env, insn, &cs)) use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1); + def = mask_widen(def); + use = mask_widen(use); break; default: def = 0; - if (BPF_SRC(insn->code) == BPF_K) - use = dst; - else - use = dst | src; + use = class == BPF_JMP ? dst : dst32; + if (BPF_SRC(insn->code) == BPF_X) + use |= class == BPF_JMP ? src : src32; } break; } @@ -2209,6 +2227,7 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct bpf_insn *insns = env->prog->insnsi; struct insn_live_regs *state; int insn_cnt = env->prog->len; + u64 pos, insn_pos; int err = 0, i, j; bool changed; @@ -2252,8 +2271,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) int insn_idx = env->cfg.insn_postorder[i]; struct insn_live_regs *live = &state[insn_idx]; struct bpf_iarray *succ; - u16 new_out = 0; - u16 new_in = 0; + u32 new_out = 0; + u32 new_in = 0; succ = bpf_insn_successors(env, insn_idx); for (int s = 0; s < succ->cnt; ++s) @@ -2267,8 +2286,20 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) } } - for (i = 0; i < insn_cnt; ++i) - insn_aux[i].live_regs_before = state[i].in; + for (i = 0; i < insn_cnt; ++i) { + int def32 = bpf_insn_def32(env->prog, &insns[i]); + u32 out = state[i].out; + u32 in = state[i].in; + + insn_aux[i].live_regs_before = mask_lo(in) | mask_hi(in); + /* + * On architectures where 32-bit operations do not reset upper halves + * of the registers, the verifier needs to zero extend a destination + * register if an instruction defines a 32-bit subregister and the + * upper half of that register is alive after the instruction. + */ + insn_aux[i].zext_dst = def32 >= 0 && (mask_hi(out) & BIT(def32)); + } if (env->log.level & BPF_LOG_LEVEL2) { verbose(env, "Live regs before insn:\n"); @@ -2284,7 +2315,12 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) else verbose(env, "."); verbose(env, " "); + pos = env->log.end_pos; bpf_verbose_insn(env, &insns[i]); + insn_pos = env->log.end_pos; + if (insn_aux[i].zext_dst) + verbose(env, "%*c; zext", bpf_vlog_alignment(insn_pos - pos), ' '); + verbose(env, "\n"); if (bpf_is_ldimm64(&insns[i])) i++; } diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index b740fa73ee26..589770ca3d3a 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -615,17 +615,6 @@ static void print_scalar_ranges(struct bpf_verifier_env *env, } } -static bool type_is_map_ptr(enum bpf_reg_type t) { - switch (base_type(t)) { - case CONST_PTR_TO_MAP: - case PTR_TO_MAP_KEY: - case PTR_TO_MAP_VALUE: - return true; - default: - return false; - } -} - /* * _a stands for append, was shortened to avoid multiline statements below. * This macro is used to output a comma separated list of attributes. diff --git a/kernel/bpf/mmap_unlock_work.h b/kernel/bpf/mmap_unlock_work.h index 5d18d7d85bef..1834db20b861 100644 --- a/kernel/bpf/mmap_unlock_work.h +++ b/kernel/bpf/mmap_unlock_work.h @@ -4,12 +4,15 @@ #ifndef __MMAP_UNLOCK_WORK_H__ #define __MMAP_UNLOCK_WORK_H__ +#include <linux/atomic.h> +#include <linux/err.h> #include <linux/irq_work.h> /* irq_work to run mmap_read_unlock() in irq_work */ struct mmap_unlock_irq_work { struct irq_work irq_work; struct mm_struct *mm; + atomic_t active; }; DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); @@ -18,32 +21,36 @@ DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); * We cannot do mmap_read_unlock() when the irq is disabled, because of * risk to deadlock with rq_lock. To look up vma when the irqs are * disabled, we need to run mmap_read_unlock() in irq_work. We use a - * percpu variable to do the irq_work. If the irq_work is already used - * by another lookup, we fall over. + * percpu variable to do the irq_work. The active flag reserves the slot + * before mmap_read_trylock() and until the irq_work callback consumes mm. */ -static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr) +static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = false; + struct mmap_unlock_irq_work *work; - if (irqs_disabled()) { - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { - work = this_cpu_ptr(&mmap_unlock_work); - if (irq_work_is_busy(&work->irq_work)) { - /* cannot queue more up_read, fallback */ - irq_work_busy = true; - } - } else { - /* - * PREEMPT_RT does not allow to trylock mmap sem in - * interrupt disabled context. Force the fallback code. - */ - irq_work_busy = true; - } - } + if (!irqs_disabled()) + return NULL; + + /* + * PREEMPT_RT does not allow to trylock mmap sem in interrupt + * disabled context. Force the fallback code. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + return ERR_PTR(-EBUSY); + + work = this_cpu_ptr(&mmap_unlock_work); + if (irq_work_is_busy(&work->irq_work) || + atomic_cmpxchg_acquire(&work->active, 0, 1)) + return ERR_PTR(-EBUSY); - *work_ptr = work; - return irq_work_busy; + return work; +} + +static inline void +bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work) +{ + if (work) + atomic_set_release(&work->active, 0); } static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm) diff --git a/kernel/bpf/net_namespace.c b/kernel/bpf/net_namespace.c index 25f30f9edaef..81006a242618 100644 --- a/kernel/bpf/net_namespace.c +++ b/kernel/bpf/net_namespace.c @@ -171,33 +171,28 @@ static int bpf_netns_link_update_prog(struct bpf_link *link, struct net *net; int idx, ret; + guard(mutex)(&netns_bpf_mutex); + if (old_prog && old_prog != link->prog) return -EPERM; if (new_prog->type != link->prog->type) return -EINVAL; - mutex_lock(&netns_bpf_mutex); - net = net_link->net; - if (!net || !check_net(net)) { + if (!net || !check_net(net)) /* Link auto-detached or netns dying */ - ret = -ENOLINK; - goto out_unlock; - } + return -ENOLINK; run_array = rcu_dereference_protected(net->bpf.run_array[type], lockdep_is_held(&netns_bpf_mutex)); idx = link_index(net, type, net_link); ret = bpf_prog_array_update_at(run_array, idx, new_prog); if (ret) - goto out_unlock; + return ret; old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); - -out_unlock: - mutex_unlock(&netns_bpf_mutex); - return ret; + return 0; } static int bpf_netns_link_fill_info(const struct bpf_link *link, diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 9a5f94371e50..6e8b18c32a10 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -99,8 +99,10 @@ static long __queue_map_get(struct bpf_map *map, void *value, bool delete) int err = 0; void *ptr; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); @@ -121,7 +123,6 @@ out: return err; } - static long __stack_map_get(struct bpf_map *map, void *value, bool delete) { struct bpf_queue_stack *qs = bpf_queue_stack(map); @@ -130,8 +131,10 @@ static long __stack_map_get(struct bpf_map *map, void *value, bool delete) void *ptr; u32 index; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index 35ae64ade36b..3f1013d80544 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -321,7 +321,7 @@ static unsigned long ringbuf_avail_data_sz(struct bpf_ringbuf *rb) if (unlikely(rb->overwrite_mode)) { over_pos = smp_load_acquire(&rb->overwrite_pos); prod_pos = smp_load_acquire(&rb->producer_pos); - return prod_pos - max(cons_pos, over_pos); + return min(prod_pos - cons_pos, prod_pos - over_pos); } else { prod_pos = smp_load_acquire(&rb->producer_pos); return prod_pos - cons_pos; @@ -482,7 +482,7 @@ static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) prod_pos = rb->producer_pos; new_prod_pos = prod_pos + len; - while (pend_pos < prod_pos) { + while (prod_pos - pend_pos > 0) { hdr = (void *)rb->data + (pend_pos & rb->mask); hdr_len = READ_ONCE(hdr->len); if (hdr_len & BPF_RINGBUF_BUSY_BIT) @@ -634,7 +634,7 @@ const struct bpf_func_proto bpf_ringbuf_output_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index 2129defc4a9a..111ec80ea958 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -745,10 +745,10 @@ __bpf_kfunc void bpf_res_spin_unlock_irqrestore(struct bpf_res_spin_lock *lock, __bpf_kfunc_end_defs(); BTF_KFUNCS_START(rqspinlock_kfunc_ids) -BTF_ID_FLAGS(func, bpf_res_spin_lock, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_res_spin_unlock) -BTF_ID_FLAGS(func, bpf_res_spin_lock_irqsave, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_res_spin_unlock_irqrestore) +BTF_ID_FLAGS(func, bpf_res_spin_lock, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_unlock, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_lock_irqsave, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_unlock_irqrestore, KF_SPINLOCK_SAFE) BTF_KFUNCS_END(rqspinlock_kfunc_ids) static const struct btf_kfunc_id_set rqspinlock_kfunc_set = { diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 41fe87d7302f..a839041e0d00 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -414,8 +414,7 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, u32 trace_nr, bool user, bool may_fault) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + struct mmap_unlock_irq_work *work; bool has_user_ctx = user && current && current->mm; struct stack_map_build_id_cache cache = {}; struct vm_area_struct *vma; @@ -426,15 +425,16 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, return; } - /* If the irq_work is in use, fall back to report ips. Same - * fallback is used for kernel stack (!user) on a stackmap with - * build_id. - */ - if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) { - /* cannot access current->mm, fall back to ips */ - for (i = 0; i < trace_nr; i++) - stack_map_build_id_set_ip(&id_offs[i]); - return; + if (!has_user_ctx) + goto fallback; + + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) + goto fallback; + + if (!mmap_read_trylock(current->mm)) { + bpf_mmap_unlock_guard_put(work); + goto fallback; } for (i = 0; i < trace_nr; i++) { @@ -465,6 +465,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, vma->vm_pgoff); } bpf_mmap_unlock_mm(work, current->mm); + return; + +fallback: + /* cannot access current->mm, fall back to ips */ + for (i = 0; i < trace_nr; i++) + stack_map_build_id_set_ip(&id_offs[i]); } static struct perf_callchain_entry * @@ -504,78 +510,116 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth) #endif } -static long __bpf_get_stackid(struct bpf_map *map, - struct perf_callchain_entry *trace, u64 flags) +struct stackid { + struct stack_map_bucket *bucket; + const u64 *ips; + u32 nr; + u32 len; + u32 hash; + u32 id; + bool hash_matches; +}; + +static int stackid_init(struct stackid *stackid, struct bpf_map *map, + const struct perf_callchain_entry *trace, u32 trace_nr, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); - struct stack_map_bucket *bucket, *new_bucket, *old_bucket; - u32 hash, id, trace_nr, trace_len, i, max_depth; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; - bool user = flags & BPF_F_USER_STACK; - u64 *ips; - bool hash_matches; + u32 max_depth; - if (trace->nr <= skip) + if (trace_nr <= skip) /* skipping more than usable stack trace */ return -EFAULT; max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags); - trace_nr = min_t(u32, trace->nr - skip, max_depth - skip); - trace_len = trace_nr * sizeof(u64); - ips = trace->ip + skip; - hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0); - id = hash & (smap->n_buckets - 1); - bucket = READ_ONCE(smap->buckets[id]); + stackid->nr = min_t(u32, trace_nr - skip, max_depth - skip); + stackid->len = stackid->nr * sizeof(u64); + stackid->ips = trace->ip + skip; + stackid->hash = jhash2((const u32 *)stackid->ips, stackid->len / sizeof(u32), 0); + stackid->id = stackid->hash & (smap->n_buckets - 1); + stackid->bucket = READ_ONCE(smap->buckets[stackid->id]); + stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; + return 0; +} + +static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map, + const struct perf_callchain_entry *trace, u32 trace_nr, + u64 flags) +{ + int err; + + err = stackid_init(stackid, map, trace, trace_nr, flags); + if (err) + return err; - hash_matches = bucket && bucket->hash == hash; /* fast cmp */ - if (hash_matches && flags & BPF_F_FAST_STACK_CMP) - return id; + if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP) + return stackid->id; + + if (stack_map_use_build_id(map)) + return -ENOENT; + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) + return stackid->id; + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) + return -EEXIST; + return -ENOENT; +} + +static struct stack_map_bucket * +stackid_new_bucket(struct stackid *stackid, struct bpf_map *map) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + struct bpf_stack_build_id *id_offs; + struct stack_map_bucket *bucket; + u32 i; + + bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); + if (unlikely(!bucket)) + return NULL; + + if (stack_map_use_build_id(map)) { + id_offs = (struct bpf_stack_build_id *)bucket->data; + for (i = 0; i < stackid->nr; i++) + id_offs[i].ip = stackid->ips[i]; + } else { + memcpy(bucket->data, stackid->ips, stackid->len); + } + + bucket->hash = stackid->hash; + bucket->nr = stackid->nr; + return bucket; +} + +static long stackid_install(struct stackid *stackid, struct bpf_map *map, + struct stack_map_bucket *new_bucket, u64 flags) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *old_bucket; + u32 trace_len; if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; - /* for build_id+offset, pop a bucket before slow cmp */ - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - new_bucket->nr = trace_nr; id_offs = (struct bpf_stack_build_id *)new_bucket->data; - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - stack_map_get_build_id_offset(id_offs, trace_nr, user, false /* !may_fault */); - trace_len = trace_nr * sizeof(struct bpf_stack_build_id); - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, new_bucket->data, trace_len) == 0) { + stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */); + trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); - return id; + return stackid->id; } - if (bucket && !(flags & BPF_F_REUSE_STACKID)) { + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); return -EEXIST; } - } else { - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, ips, trace_len) == 0) - return id; - if (bucket && !(flags & BPF_F_REUSE_STACKID)) - return -EEXIST; - - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - memcpy(new_bucket->data, ips, trace_len); } - new_bucket->hash = hash; - new_bucket->nr = trace_nr; - - old_bucket = xchg(&smap->buckets[id], new_bucket); + old_bucket = xchg(&smap->buckets[stackid->id], new_bucket); if (old_bucket) pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); - return id; + return stackid->id; } BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, @@ -583,23 +627,36 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, { u32 elem_size = stack_map_data_size(map); bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *new_bucket; struct perf_callchain_entry *trace; + struct stackid stackid; bool kernel = !user; u32 max_depth; + int err; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) return -EINVAL; max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags); - trace = get_perf_callchain(regs, kernel, user, max_depth, - false, false, 0); - if (unlikely(!trace)) - /* couldn't fetch the stack trace */ - return -EFAULT; + scoped_guard(preempt) { + trace = get_perf_callchain(regs, kernel, user, max_depth, + false, false, 0); + if (unlikely(!trace)) + /* couldn't fetch the stack trace */ + return -EFAULT; + + err = stackid_fastpath(&stackid, map, trace, trace->nr, flags); + if (err != -ENOENT) + return err; + + new_bucket = stackid_new_bucket(&stackid, map); + if (!new_bucket) + return -ENOMEM; + } - return __bpf_get_stackid(map, trace, flags); + return stackid_install(&stackid, map, new_bucket, flags); } const struct bpf_func_proto bpf_get_stackid_proto = { @@ -611,7 +668,7 @@ const struct bpf_func_proto bpf_get_stackid_proto = { .arg3_type = ARG_ANYTHING, }; -static __u64 count_kernel_ip(struct perf_callchain_entry *trace) +static __u64 count_kernel_ip(const struct perf_callchain_entry *trace) { __u64 nr_kernel = 0; @@ -626,10 +683,13 @@ static __u64 count_kernel_ip(struct perf_callchain_entry *trace) BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, struct bpf_map *, map, u64, flags) { + const struct perf_callchain_entry *trace; struct perf_event *event = ctx->event; - struct perf_callchain_entry *trace; + struct stack_map_bucket *new_bucket; + struct stackid stackid; bool kernel, user; __u64 nr_kernel; + u32 trace_nr; int ret; /* perf_sample_data doesn't have callchain, use bpf_get_stackid */ @@ -649,26 +709,28 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, return -EFAULT; nr_kernel = count_kernel_ip(trace); - __u64 nr = trace->nr; /* save original */ if (kernel) { - trace->nr = nr_kernel; - ret = __bpf_get_stackid(map, trace, flags); + trace_nr = nr_kernel; } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; + trace_nr = trace->nr; skip += nr_kernel; if (skip > BPF_F_SKIP_FIELD_MASK) return -EFAULT; flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - ret = __bpf_get_stackid(map, trace, flags); } - /* restore nr */ - trace->nr = nr; + ret = stackid_fastpath(&stackid, map, trace, trace_nr, flags); + if (ret != -ENOENT) + return ret; - return ret; + new_bucket = stackid_new_bucket(&stackid, map); + if (new_bucket) + return stackid_install(&stackid, map, new_bucket, flags); + return -ENOMEM; } const struct bpf_func_proto bpf_get_stackid_proto_pe = { @@ -680,19 +742,55 @@ const struct bpf_func_proto bpf_get_stackid_proto_pe = { .arg3_type = ARG_ANYTHING, }; +static u32 callchain_store(const struct perf_callchain_entry *trace, u32 trace_nr, + void *buf, u32 elem_size, u64 flags) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + u32 skip = flags & BPF_F_SKIP_FIELD_MASK; + const u64 *ips; + u32 copy_len; + + trace_nr = trace_nr - skip; + copy_len = trace_nr * elem_size; + + ips = trace->ip + skip; + if (user_build_id) { + struct bpf_stack_build_id *id_offs = buf; + + for (u32 i = 0; i < trace_nr; i++) + id_offs[i].ip = ips[i]; + } else { + memcpy(buf, ips, copy_len); + } + return trace_nr; +} + +static long callchain_finalize(void *buf, u32 size, u32 trace_nr, u32 elem_size, + u64 flags, bool may_fault) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + bool user = flags & BPF_F_USER_STACK; + u32 copy_len = trace_nr * elem_size; + + if (user_build_id) + stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); + + if (size > copy_len) + memset(buf + copy_len, 0, size - copy_len); + return copy_len; +} + static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, - struct perf_callchain_entry *trace_in, void *buf, u32 size, u64 flags, bool may_fault) { - u32 trace_nr, copy_len, elem_size, max_depth; bool user_build_id = flags & BPF_F_USER_BUILD_ID; bool crosstask = task && task != current; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; bool user = flags & BPF_F_USER_STACK; struct perf_callchain_entry *trace; + u32 trace_nr, elem_size, max_depth; bool kernel = !user; int err = -EINVAL; - u64 *ips; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_USER_BUILD_ID))) @@ -718,13 +816,11 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, max_depth = stack_map_calculate_max_depth(size, elem_size, flags); + preempt_disable(); if (may_fault) rcu_read_lock(); /* need RCU for perf's callchain below */ - if (trace_in) { - trace = trace_in; - trace->nr = min_t(u32, trace->nr, max_depth); - } else if (kernel && task) { + if (kernel && task) { trace = get_callchain_entry_for_task(task, max_depth); } else { trace = get_perf_callchain(regs, kernel, user, max_depth, @@ -734,33 +830,18 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, if (unlikely(!trace) || trace->nr < skip) { if (may_fault) rcu_read_unlock(); + preempt_enable(); goto err_fault; } - trace_nr = trace->nr - skip; - copy_len = trace_nr * elem_size; - - ips = trace->ip + skip; - if (user_build_id) { - struct bpf_stack_build_id *id_offs = buf; - u32 i; - - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - } else { - memcpy(buf, ips, copy_len); - } + trace_nr = callchain_store(trace, trace->nr, buf, elem_size, flags); - /* trace/ips should not be dereferenced after this point */ + /* trace should not be dereferenced after this point */ if (may_fault) rcu_read_unlock(); + preempt_enable(); - if (user_build_id) - stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); - - if (size > copy_len) - memset(buf + copy_len, 0, size - copy_len); - return copy_len; + return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault); err_fault: err = -EFAULT; @@ -772,7 +853,7 @@ clear: BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size, u64, flags) { - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */); } const struct bpf_func_proto bpf_get_stack_proto = { @@ -781,14 +862,14 @@ const struct bpf_func_proto bpf_get_stack_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, size, u64, flags) { - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, true /* may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, true /* may_fault */); } const struct bpf_func_proto bpf_get_stack_sleepable_proto = { @@ -797,7 +878,7 @@ const struct bpf_func_proto bpf_get_stack_sleepable_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -807,14 +888,17 @@ static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size, struct pt_regs *regs; long res = -EINVAL; - if (!try_get_task_stack(task)) + if (!try_get_task_stack(task)) { + memset(buf, 0, size); return -EFAULT; + } regs = task_pt_regs(task); if (regs) - res = __bpf_get_stack(regs, task, NULL, buf, size, flags, may_fault); + res = __bpf_get_stack(regs, task, buf, size, flags, may_fault); + else + memset(buf, 0, size); put_task_stack(task); - return res; } @@ -831,7 +915,7 @@ const struct bpf_func_proto bpf_get_task_stack_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -848,22 +932,48 @@ const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; +static int __bpf_get_stack_pe(const struct perf_callchain_entry *trace, u32 trace_nr, + void *buf, u32 size, u64 flags) +{ + bool user_build_id = flags & BPF_F_USER_BUILD_ID; + u64 skip = flags & BPF_F_SKIP_FIELD_MASK; + bool user = flags & BPF_F_USER_STACK; + u32 elem_size, max_depth, nr_trace; + bool kernel = !user; + + if (kernel && user_build_id) + return -EINVAL; + + elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64); + if (unlikely(size % elem_size)) + return -EINVAL; + + max_depth = stack_map_calculate_max_depth(size, elem_size, flags); + trace_nr = min_t(u32, trace_nr, max_depth); + + if (trace_nr < skip) + return -EFAULT; + + nr_trace = callchain_store(trace, trace_nr, buf, elem_size, flags); + return callchain_finalize(buf, size, nr_trace, elem_size, flags, false /* !may_fault */); +} + BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, void *, buf, u32, size, u64, flags) { struct pt_regs *regs = (struct pt_regs *)(ctx->regs); + const struct perf_callchain_entry *trace; struct perf_event *event = ctx->event; - struct perf_callchain_entry *trace; bool kernel, user; int err = -EINVAL; __u64 nr_kernel; if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)) - return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */); + return __bpf_get_stack(regs, NULL, buf, size, flags, false /* !may_fault */); if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_USER_BUILD_ID))) @@ -880,27 +990,20 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, nr_kernel = count_kernel_ip(trace); if (kernel) { - __u64 nr = trace->nr; - - trace->nr = nr_kernel; - err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */); - - /* restore nr */ - trace->nr = nr; + err = __bpf_get_stack_pe(trace, nr_kernel, buf, size, flags); } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; skip += nr_kernel; if (skip > BPF_F_SKIP_FIELD_MASK) goto clear; - flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */); + err = __bpf_get_stack_pe(trace, trace->nr, buf, size, flags); } - return err; clear: - memset(buf, 0, size); + if (err < 0) + memset(buf, 0, size); return err; } @@ -911,7 +1014,7 @@ const struct bpf_func_proto bpf_get_stack_proto_pe = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index ea2153cf28d0..4e6aafad33bd 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -812,7 +812,8 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, * infinite loop check triggering, see * iter_active_depths_differ() */ - if (old_reg->iter.btf != cur_reg->iter.btf || + if (old_reg->type != cur_reg->type || + old_reg->iter.btf != cur_reg->iter.btf || old_reg->iter.btf_id != cur_reg->iter.btf_id || old_reg->iter.state != cur_reg->iter.state || /* ignore {old_reg,cur_reg}->iter.depth, see above */ diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 6db306d23b47..6874ba1424af 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -40,7 +40,6 @@ #include <linux/tracepoint.h> #include <linux/overflow.h> #include <linux/cookie.h> -#include <linux/verification.h> #include <linux/btf_ids.h> #include <net/netfilter/nf_bpf_link.h> @@ -637,7 +636,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid, return ret; } - static int btf_field_cmp(const void *a, const void *b) { const struct btf_field *f1 = a, *f2 = b; @@ -1078,11 +1076,24 @@ static void bpf_map_mmap_close(struct vm_area_struct *vma) bpf_map_write_active_dec(map); } +static vm_fault_t bpf_map_mmap_fault(struct vm_fault *vmf) +{ + struct bpf_map *map = vmf->vma->vm_private_data; + + return map->ops->map_mmap_fault(map, vmf); +} + static const struct vm_operations_struct bpf_map_default_vmops = { .open = bpf_map_mmap_open, .close = bpf_map_mmap_close, }; +static const struct vm_operations_struct bpf_map_lazy_vmops = { + .open = bpf_map_mmap_open, + .close = bpf_map_mmap_close, + .fault = bpf_map_mmap_fault, +}; + static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma) { struct bpf_map *map = filp->private_data; @@ -1118,7 +1129,7 @@ out: return err; /* set default open/close callbacks */ - vma->vm_ops = &bpf_map_default_vmops; + vma->vm_ops = map->ops->map_mmap_fault ? &bpf_map_lazy_vmops : &bpf_map_default_vmops; vma->vm_private_data = map; vm_flags_clear(vma, VM_MAYEXEC); /* If mapping is read-only, then disallow potentially re-mapping with @@ -1599,13 +1610,6 @@ static int map_create_alloc(union bpf_attr *attr, bpfptr_t uattr, struct bpf_ver err = -EFAULT; goto free_map; } - - /* See libbpf: emit_signature_match() */ - BUILD_BUG_ON(offsetof(struct bpf_map, excl) != SHA256_DIGEST_SIZE); - BUILD_BUG_ON(!__same_type(map->excl, u32)); - BUILD_BUG_ON(offsetof(struct bpf_map, sha) != 0); - BUILD_BUG_ON(!__same_type(map->sha, u8[SHA256_DIGEST_SIZE])); - map->excl = 1; } else if (attr->excl_prog_hash_size) { bpf_log(log, "Invalid excl_prog_hash_size.\n"); err = -EINVAL; @@ -1657,7 +1661,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_at err = security_bpf_map_create(map, attr, token, uattr.is_kernel); if (err) - goto free_map_sec; + goto free_map; err = bpf_map_alloc_id(map); if (err) @@ -1838,7 +1842,6 @@ free_key: return err; } - #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) @@ -2886,64 +2889,6 @@ static bool is_perfmon_prog_type(enum bpf_prog_type prog_type) } } -static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) -{ - switch (keyring_id) { - case 0: - return BPF_SIG_KEYRING_BUILTIN; - case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING: - return BPF_SIG_KEYRING_SECONDARY; - case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: - return BPF_SIG_KEYRING_PLATFORM; - default: - return BPF_SIG_KEYRING_USER; - } -} - -static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr, - bool is_kernel, s32 *keyring_serial) -{ - bpfptr_t usig = make_bpfptr(attr->signature, is_kernel); - struct bpf_dynptr_kern sig_ptr, insns_ptr; - struct bpf_key *key = NULL; - void *sig; - int err = 0; - - /* - * Don't attempt to use kmalloc_large or vmalloc for signatures. - * Practical signature for BPF program should be below this limit. - */ - if (attr->signature_size > KMALLOC_MAX_CACHE_SIZE) - return -EINVAL; - - if (system_keyring_id_check(attr->keyring_id) == 0) - key = bpf_lookup_system_key(attr->keyring_id); - else - key = bpf_lookup_user_key(attr->keyring_id, 0); - - if (!key) - return -EINVAL; - - sig = kvmemdup_bpfptr(usig, attr->signature_size); - if (IS_ERR(sig)) { - bpf_key_put(key); - return PTR_ERR(sig); - } - - bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0, - attr->signature_size); - bpf_dynptr_init(&insns_ptr, prog->insnsi, BPF_DYNPTR_TYPE_LOCAL, 0, - prog->len * sizeof(struct bpf_insn)); - - err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&insns_ptr, - (struct bpf_dynptr *)&sig_ptr, key); - if (!err) - *keyring_serial = bpf_key_serial(key); - bpf_key_put(key); - kvfree(sig); - return err; -} - static int bpf_prog_mark_insn_arrays_ready(struct bpf_prog *prog) { int err; @@ -3109,6 +3054,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at prog->aux->attach_btf = attach_btf; prog->aux->attach_btf_id = multi_func ? bpf_multi_func_btf_id[0] : attr->attach_btf_id; prog->aux->dst_prog = dst_prog; + if (dst_prog) { + prog->aux->saved_dst_prog_type = dst_prog->type; + prog->aux->saved_dst_attach_type = dst_prog->expected_attach_type; + } prog->aux->dev_bound = !!attr->prog_ifindex; prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; @@ -3133,17 +3082,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at /* eBPF programs must be GPL compatible to use GPL-ed functions */ prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0; - if (attr->signature) { - err = bpf_prog_verify_signature(prog, attr, uattr.is_kernel, - &prog->aux->sig.keyring_serial); - if (err) - goto free_prog; - prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id); - prog->aux->sig.verdict = BPF_SIG_VERIFIED; - } else { - prog->aux->sig.keyring_type = BPF_SIG_KEYRING_NONE; - prog->aux->sig.verdict = BPF_SIG_UNSIGNED; - } + prog->aux->sig.keyring_type = BPF_SIG_KEYRING_NONE; + prog->aux->sig.verdict = BPF_SIG_UNSIGNED; prog->orig_prog = NULL; prog->jited = 0; @@ -3189,10 +3129,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at if (err < 0) goto free_prog; - err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel); - if (err) - goto free_prog; - /* run eBPF verifier */ err = bpf_check(&prog, attr, uattr, attr_log); if (err < 0) @@ -3471,9 +3407,10 @@ static const char *bpf_link_type_strs[] = { static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) { const struct bpf_link *link = filp->private_data; - const struct bpf_prog *prog = link->prog; + const struct bpf_prog *prog; enum bpf_link_type type = link->type; char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; + u32 prog_id = 0; if (type < ARRAY_SIZE(bpf_link_type_strs) && bpf_link_type_strs[type]) { if (link->type == BPF_LINK_TYPE_KPROBE_MULTI) @@ -3490,13 +3427,20 @@ static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) } seq_printf(m, "link_id:\t%u\n", link->id); + rcu_read_lock(); + prog = READ_ONCE(link->prog); if (prog) { bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); + prog_id = prog->aux->id; + } + rcu_read_unlock(); + + if (prog) { seq_printf(m, "prog_tag:\t%s\n" "prog_id:\t%u\n", prog_tag, - prog->aux->id); + prog_id); } if (link->ops->show_fdinfo) link->ops->show_fdinfo(link, m); @@ -3564,7 +3508,6 @@ int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) if (fd < 0) return fd; - id = bpf_link_alloc_id(link); if (id < 0) { put_unused_fd(fd); @@ -3626,10 +3569,12 @@ static void bpf_tracing_link_release(struct bpf_link *link) { struct bpf_tracing_link *tr_link = container_of(link, struct bpf_tracing_link, link.link); + int err; - WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link.node, - tr_link->trampoline, - tr_link->tgt_prog)); + err = bpf_trampoline_unlink_prog(&tr_link->link.node, + tr_link->trampoline, + tr_link->tgt_prog); + WARN_ONCE(err, "bpf_trampoline_unlink_prog failed: %d\n", err); bpf_trampoline_put(tr_link->trampoline); @@ -5535,6 +5480,7 @@ static int bpf_link_get_info_by_fd(struct file *file, { struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); struct bpf_link_info info; + const struct bpf_prog *prog; u32 info_len = attr->info.info_len; int err; @@ -5549,8 +5495,12 @@ static int bpf_link_get_info_by_fd(struct file *file, info.type = link->type; info.id = link->id; - if (link->prog) - info.prog_id = link->prog->aux->id; + + rcu_read_lock(); + prog = READ_ONCE(link->prog); + if (prog) + info.prog_id = prog->aux->id; + rcu_read_unlock(); if (link->ops->fill_link_info) { err = link->ops->fill_link_info(link, &info); @@ -5565,7 +5515,6 @@ static int bpf_link_get_info_by_fd(struct file *file, return 0; } - static int token_get_info_by_fd(struct file *file, struct bpf_token *token, const union bpf_attr *attr, @@ -6567,7 +6516,6 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size, KERNEL_BPFPTR(NULL), 0); } - /* To shut up -Wmissing-prototypes. * This function is used by the kernel light skeleton * to load bpf programs when modules are loaded or during kernel boot. @@ -6623,7 +6571,7 @@ static const struct bpf_func_proto bpf_sys_bpf_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; const struct bpf_func_proto * __weak @@ -6670,7 +6618,7 @@ static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, .arg4_size = sizeof(u64), diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index e791ae065c39..13e1aabe6f88 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -753,9 +753,9 @@ static struct bpf_iter_reg task_vma_reg_info = { BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags) { - struct mmap_unlock_irq_work *work = NULL; + struct mmap_unlock_irq_work *work; struct vm_area_struct *vma; - bool irq_work_busy = false; + bool __maybe_unused mmput_needed = false; struct mm_struct *mm; int ret = -ENOENT; @@ -765,14 +765,43 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, if (!task) return -ENOENT; - mm = task->mm; + if (task == current) { + mm = task->mm; + } else { + /* + * Foreign task: pin task->mm against a concurrent exit_mm(). + * Use trylock on alloc_lock instead of get_task_mm()'s + * blocking task_lock() to avoid deadlocking the target task. + */ + if (!IS_ENABLED(CONFIG_MMU)) + return -EOPNOTSUPP; + if (irqs_disabled()) + return -EBUSY; + if (!spin_trylock(&task->alloc_lock)) + return -EBUSY; + mm = task->mm; + if (mm && !(task->flags & PF_KTHREAD)) { + mmget(mm); + mmput_needed = true; + } else { + mm = NULL; + } + spin_unlock(&task->alloc_lock); + } if (!mm) return -ENOENT; - irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) { + ret = PTR_ERR(work); + goto out; + } - if (irq_work_busy || !mmap_read_trylock(mm)) - return -EBUSY; + if (!mmap_read_trylock(mm)) { + bpf_mmap_unlock_guard_put(work); + ret = -EBUSY; + goto out; + } vma = find_vma(mm, start); @@ -782,6 +811,11 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, ret = 0; } bpf_mmap_unlock_mm(work, mm); +out: +#ifdef CONFIG_MMU + if (mmput_needed) + mmput_async(mm); +#endif return ret; } @@ -1161,6 +1195,8 @@ static void do_mmap_read_unlock(struct irq_work *entry) work = container_of(entry, struct mmap_unlock_irq_work, irq_work); mmap_read_unlock_non_owner(work->mm); + work->mm = NULL; + bpf_mmap_unlock_guard_put(work); } static int __init task_iter_init(void) diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 1a721fc4bef5..90b70ea0d370 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -529,6 +529,36 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_a return tnodes; } +/* + * The arena base against which save_args() converts the arguments marked + * with BTF_FMODEL_ARENA_ARG. Only the struct_ops indirect trampoline + * converts: it dispatches to a single prog whose arena is known at + * generation time. Return 0 when there is nothing to convert. + */ +u64 bpf_tramp_arena_base(const struct btf_func_model *m, + struct bpf_tramp_nodes *tnodes, u32 flags) +{ + const struct bpf_prog *prog; + int i; + + if (!(flags & BPF_TRAMP_F_INDIRECT) || + tnodes[BPF_TRAMP_FENTRY].nr_nodes != 1) + return 0; + + for (i = 0; i < m->nr_args; i++) + if (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG) + break; + if (i == m->nr_args) + return 0; + + /* Verification rejects an arena argument without an arena. */ + prog = tnodes[BPF_TRAMP_FENTRY].nodes[0]->link->prog; + if (WARN_ON_ONCE(!prog->aux->arena)) + return 0; + + return bpf_arena_get_kern_vm_start(prog->aux->arena); +} + static void bpf_tramp_image_free(struct bpf_tramp_image *im) { bpf_image_ksym_del(&im->ksym); @@ -670,6 +700,13 @@ out: return ERR_PTR(err); } +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) +{ + trampoline_lock(tr); + tr->flags |= flags; + trampoline_unlock(tr); +} + static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex, const struct bpf_trampoline_ops *ops, void *data) { @@ -913,6 +950,13 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, int cnt = 0, i; kind = bpf_attach_type_to_tramp(node->link->prog); + /* + * Arena ctx args are converted only by struct_ops indirect + * trampolines. They must never be attached to a generic trampoline. + */ + if (WARN_ON_ONCE(bpf_prog_has_arena_ctx_arg(node->link->prog))) + return -ENOTSUPP; + if (tr->extension_prog) /* cannot attach fentry/fexit if extension prog is attached. * cannot overwrite extension prog either. @@ -997,12 +1041,15 @@ static void bpf_shim_tramp_link_release(struct bpf_link *link) { struct bpf_shim_tramp_link *shim_link = container_of(link, struct bpf_shim_tramp_link, link.link); + int err; /* paired with 'shim_link->trampoline = tr' in bpf_trampoline_link_cgroup_shim */ if (!shim_link->trampoline) return; - WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link.node, shim_link->trampoline, NULL)); + err = bpf_trampoline_unlink_prog(&shim_link->link.node, shim_link->trampoline, NULL); + WARN_ONCE(err, "bpf_trampoline_unlink_prog failed: %d\n", err); + bpf_trampoline_put(shim_link->trampoline); } @@ -1536,6 +1583,7 @@ static int register_fentry_multi(struct bpf_trampoline *tr, struct bpf_tramp_ima if (bpf_trampoline_use_jmp(tr->flags)) addr = ftrace_jmp_set(addr); + tr->func.ftrace_managed = true; ftrace_hash_add(data->reg, data->entry, ip, addr); tr->cur_image = im; return 0; @@ -1584,7 +1632,17 @@ static void bpf_trampoline_multi_attach_init(struct bpf_trampoline *tr) static void bpf_trampoline_multi_attach_free(struct bpf_trampoline *tr) { - if (tr->multi_attach.old_image) + /* + * Only free old_image if it is no longer the active image. + * When bpf_trampoline_update() fails before modify_fentry_multi()/ + * unregister_fentry_multi() is called, cur_image is unchanged + * (cur_image == old_image) and ftrace still points to it. Freeing + * it would cause a UAF when ftrace calls into the freed memory. + * On success, cur_image is either a new image or NULL, so + * old_image != cur_image means the image is stale. + */ + if (tr->multi_attach.old_image && + tr->multi_attach.old_image != tr->cur_image) bpf_tramp_image_put(tr->multi_attach.old_image); tr->multi_attach.old_image = NULL; @@ -1708,19 +1766,21 @@ rollback_put: return err; } -int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_link *link) +void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link) { struct bpf_tracing_multi_data *data = &link->data; struct bpf_tracing_multi_node *mnode; - int i; + int i, err; trampoline_lock_all(); for_each_mnode(mnode, link) { data->entry = &mnode->entry; bpf_trampoline_multi_attach_init(mnode->trampoline); - WARN_ON_ONCE(__bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline, - NULL, &trampoline_multi_ops, data)); + err = __bpf_trampoline_unlink_prog(&mnode->node, mnode->trampoline, NULL, + &trampoline_multi_ops, data); + WARN_ONCE(err, "__bpf_trampoline_unlink_prog failed: %d\n", err); } if (ftrace_hash_count(data->unreg)) @@ -1737,7 +1797,6 @@ int bpf_trampoline_multi_detach(struct bpf_prog *prog, struct bpf_tracing_multi_ bpf_trampoline_put(mnode->trampoline); clear_tracing_multi_data(data); - return 0; } #undef for_each_mnode_cnt diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fdc5fbb1f78c..e421ea2b80c3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22,6 +22,8 @@ #include <linux/ctype.h> #include <linux/error-injection.h> #include <linux/bpf_lsm.h> +#include <linux/security.h> +#include <linux/verification.h> #include <linux/btf_ids.h> #include <linux/poison.h> #include <linux/module.h> @@ -32,6 +34,7 @@ #include <linux/trace_events.h> #include <linux/kallsyms.h> +#include "diagnostics.h" #include "disasm.h" static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { @@ -191,6 +194,7 @@ struct bpf_verifier_stack_elem { struct bpf_verifier_stack_elem *next; /* length of verifier log at the time this state was pushed on stack */ u32 log_pos; + u64 diag_log_pos; }; #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 @@ -201,9 +205,11 @@ struct bpf_verifier_stack_elem { #define BPF_PRIV_STACK_MIN_SIZE 64 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id); -static int release_reference_nomark(struct bpf_verifier_state *state, int id); +static int __release_reference_nomark(struct bpf_verifier_state *state, int id); +static int release_reference_nomark(struct bpf_verifier_env *env, int id); static int release_reference(struct bpf_verifier_env *env, int id); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); +static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); static bool is_tracing_prog_type(enum bpf_prog_type type); static int ref_set_non_owning(struct bpf_verifier_env *env, @@ -250,27 +256,6 @@ static int validate_ref_obj(struct bpf_verifier_env *env, struct ref_obj_desc *r return 0; } -struct bpf_call_arg_meta { - struct bpf_map_desc map; - struct bpf_dynptr_desc dynptr; - struct ref_obj_desc ref_obj; - bool raw_mode; - bool pkt_access; - u8 release_regno; - int regno; - int access_size; - int mem_size; - u64 msize_max_value; - int func_id; - struct btf *btf; - u32 btf_id; - struct btf *ret_btf; - u32 ret_btf_id; - u32 subprogno; - struct btf_field *kptr_field; - s64 const_map_key; -}; - struct bpf_kfunc_meta { struct btf *btf; const struct btf_type *proto; @@ -322,6 +307,7 @@ static const char *btf_type_name(const struct btf *btf, u32 id) } static DEFINE_MUTEX(bpf_verifier_lock); +static DEFINE_MUTEX(btf_vmlinux_lock); static DEFINE_MUTEX(bpf_percpu_ma_lock); __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) @@ -422,7 +408,7 @@ static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) return btf_type_is_void(type); } -static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) +const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) { struct bpf_func_info *info; @@ -652,7 +638,6 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, bool first_slot, int id, int parent_id); - static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, struct bpf_reg_state *sreg1, struct bpf_reg_state *sreg2, @@ -831,6 +816,10 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) && dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) { verbose(env, "cannot overwrite referenced dynptr\n"); + bpf_diag_res( + env, env->insn_idx, "referenced dynptr overwrite", + "This stack slot contains a dynptr that owns or protects a referenced resource. Overwriting the last dynptr for that resource would lose the verifier-tracked release path.", + "Release or clone the dynptr so another live dynptr still tracks the referenced resource before overwriting this stack slot."); return -EINVAL; } @@ -901,36 +890,39 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re return true; } -static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - enum bpf_arg_type arg_type) +static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { - struct bpf_func_state *state = bpf_func(env, reg); - enum bpf_dynptr_type dynptr_type; + struct bpf_func_state *state; int spi; + if (reg->type == CONST_PTR_TO_DYNPTR) + return reg->dynptr.type; + + spi = dynptr_get_spi(env, reg); + if (spi < 0) + return BPF_DYNPTR_TYPE_INVALID; + state = bpf_func(env, reg); + return state->stack[spi].spilled_ptr.dynptr.type; +} + +static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + enum bpf_arg_type arg_type) +{ /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ if (arg_type == ARG_PTR_TO_DYNPTR) return true; - dynptr_type = arg_to_dynptr_type(arg_type); - if (reg->type == CONST_PTR_TO_DYNPTR) { - return reg->dynptr.type == dynptr_type; - } else { - spi = dynptr_get_spi(env, reg); - if (spi < 0) - return false; - return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; - } + return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type); } static void __mark_reg_known_zero(struct bpf_reg_state *reg); static bool in_rcu_cs(struct bpf_verifier_env *env); -static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta); +static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta); static int mark_stack_slots_iter(struct bpf_verifier_env *env, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, struct bpf_reg_state *reg, int insn_idx, struct btf *btf, u32 btf_id, int nr_slots) { @@ -1060,10 +1052,10 @@ static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_s } static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx); -static int release_irq_state(struct bpf_verifier_state *state, int id); +static int release_irq_state(struct bpf_verifier_env *env, int id); static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, struct bpf_reg_state *reg, int insn_idx, int kfunc_class) { @@ -1113,13 +1105,23 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r if (st->irq.kfunc_class != kfunc_class) { const char *flag_kfunc = st->irq.kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; const char *used_kfunc = kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; + const char *reason; verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n", flag_kfunc, used_kfunc); + reason = bpf_diag_fmt(env, + "This IRQ flag was saved by %s IRQ kfuncs, but the restore call " + "belongs to the %s IRQ kfunc family. Save and restore operations " + "must use the same family.", + flag_kfunc, used_kfunc); + bpf_diag_irq(env, env->insn_idx, "IRQ flag restore mismatch", reason, + "Restore the flag with the matching IRQ restore kfunc for the save " + "operation that created it.", + bpf_diag_irq_depth(env->cur_state)); return -EINVAL; } - err = release_irq_state(env->cur_state, st->id); + err = release_irq_state(env, st->id); WARN_ON_ONCE(err && err != -EACCES); if (err) { int insn_idx = 0; @@ -1133,6 +1135,11 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n", env->cur_state->active_irq_id, insn_idx); + bpf_diag_irq(env, env->insn_idx, "IRQ flag restore out of order", + "IRQ-disabled regions must be restored in last-in, first-out order, " + "but this restore does not match the currently active IRQ flag.", + "Restore nested IRQ flags in the reverse order they were saved.", + bpf_diag_irq_depth(env->cur_state)); return err; } @@ -1434,6 +1441,7 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par s->type = REF_TYPE_PTR; s->id = ++env->id_gen; s->parent_id = parent_id; + bpf_diag_record_ref_acquire(env, insn_idx, s->id); return s->id; } @@ -1453,6 +1461,8 @@ static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum r state->active_locks++; state->active_lock_id = id; state->active_lock_ptr = ptr; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_LOCK, true, + state->active_locks); return 0; } @@ -1468,6 +1478,8 @@ static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx) s->id = ++env->id_gen; state->active_irq_id = s->id; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_IRQ, true, + bpf_diag_irq_depth(state)); return s->id; } @@ -1509,8 +1521,9 @@ static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg return find_reference_state(env->cur_state, reg->id); } -static int release_lock_state(struct bpf_verifier_state *state, int type, int id, void *ptr) +static int release_lock_state(struct bpf_verifier_env *env, int type, int id, void *ptr) { + struct bpf_verifier_state *state = env->cur_state; void *prev_ptr = NULL; u32 prev_id = 0; int i; @@ -1523,6 +1536,8 @@ static int release_lock_state(struct bpf_verifier_state *state, int type, int id /* Reassign active lock (id, ptr). */ state->active_lock_id = prev_id; state->active_lock_ptr = prev_ptr; + bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_LOCK, + false, state->active_locks); return 0; } if (state->refs[i].type & REF_TYPE_LOCK_MASK) { @@ -1533,8 +1548,9 @@ static int release_lock_state(struct bpf_verifier_state *state, int type, int id return -EINVAL; } -static int release_irq_state(struct bpf_verifier_state *state, int id) +static int release_irq_state(struct bpf_verifier_env *env, int id) { + struct bpf_verifier_state *state = env->cur_state; u32 prev_id = 0; int i; @@ -1547,6 +1563,8 @@ static int release_irq_state(struct bpf_verifier_state *state, int id) if (state->refs[i].id == id) { release_reference_state(state, i); state->active_irq_id = prev_id; + bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_IRQ, + false, bpf_diag_irq_depth(state)); return 0; } else { prev_id = state->refs[i].id; @@ -1610,6 +1628,8 @@ static int copy_func_state(struct bpf_func_state *dst, const struct bpf_func_state *src) { memcpy(dst, src, offsetof(struct bpf_func_state, stack)); + /* Instruction accounting is path-local, not part of verifier state. */ + dst->insns_subtotal = 0; return copy_stack_state(dst, src); } @@ -1689,7 +1709,6 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta return true; } - void bpf_free_backedges(struct bpf_scc_visit *visit) { struct bpf_scc_backedge *backedge, *next; @@ -1716,6 +1735,7 @@ static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, err = bpf_copy_verifier_state(cur, &head->st); if (err) return err; + bpf_diag_event_log_restore(env, head->diag_log_pos); } if (pop_log) bpf_vlog_reset(&env->log, head->log_pos); @@ -1759,6 +1779,7 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, elem->prev_insn_idx = prev_insn_idx; elem->next = env->head; elem->log_pos = env->log.end_pos; + elem->diag_log_pos = bpf_diag_event_log_save(env); env->head = elem; env->stack_size++; err = bpf_copy_verifier_state(&elem->st, cur); @@ -1805,6 +1826,17 @@ static const int caller_saved[CALLER_SAVED_REGS] = { BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 }; +static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env, + struct bpf_reg_state *regs) +{ + int i; + + for (i = 1; i < CALLER_SAVED_REGS; i++) { + bpf_diag_record_scrub(env, ®s[caller_saved[i]], + BPF_DIAG_MOD_CALLER_SAVED); + } +} + /* This helper doesn't clear reg->id */ static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) { @@ -1872,32 +1904,34 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type ty reg->dynptr.first_slot = first_slot; } -static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) +/* + * Refine the return type of the bpf_map_lookup_elem() for special map types: + * map-in-map, xskmap, sockmap and sockhash. + */ +static void refine_map_lookup_value(struct bpf_reg_state *reg) { - if (base_type(reg->type) == PTR_TO_MAP_VALUE) { - const struct bpf_map *map = reg->map_ptr; + enum bpf_type_flag maybe_null = reg->type & PTR_MAYBE_NULL; + const struct bpf_map *map = reg->map_ptr; - if (map->inner_map_meta) { - reg->type = CONST_PTR_TO_MAP; - reg->map_ptr = map->inner_map_meta; - /* transfer reg's id which is unique for every map_lookup_elem - * as UID of the inner map. - */ - if (btf_record_has_field(map->inner_map_meta->record, - BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) { - reg->map_uid = reg->id; - } - } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { - reg->type = PTR_TO_XDP_SOCK; - } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || - map->map_type == BPF_MAP_TYPE_SOCKHASH) { - reg->type = PTR_TO_SOCKET; - } else { - reg->type = PTR_TO_MAP_VALUE; - } - return; + if (map->inner_map_meta) { + reg->type = CONST_PTR_TO_MAP | maybe_null; + reg->map_ptr = map->inner_map_meta; + /* transfer reg's id which is unique for every map_lookup_elem + * as UID of the inner map. + */ + if (btf_record_has_field(map->inner_map_meta->record, + BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) + reg->map_uid = reg->id; + } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { + reg->type = PTR_TO_XDP_SOCK | maybe_null; + } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || + map->map_type == BPF_MAP_TYPE_SOCKHASH) { + reg->type = PTR_TO_SOCKET | maybe_null; } +} +static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) +{ reg->type &= ~PTR_MAYBE_NULL; } @@ -2147,12 +2181,9 @@ out: /* Mark a register as having a completely unknown (scalar) value. */ void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg) { - s32 subreg_def = reg->subreg_def; - memset(reg, 0, sizeof(*reg)); reg->type = SCALAR_VALUE; reg->var_off = tnum_unknown; - reg->subreg_def = subreg_def; __mark_reg_unbounded(reg); } @@ -2228,7 +2259,6 @@ static int mark_btf_ld_reg(struct bpf_verifier_env *env, } } -#define DEF_NOT_SUBREG (0) static void init_reg_state(struct bpf_verifier_env *env, struct bpf_func_state *state) { @@ -2237,7 +2267,6 @@ static void init_reg_state(struct bpf_verifier_env *env, for (i = 0; i < MAX_BPF_REG; i++) { bpf_mark_reg_not_init(env, ®s[i]); - regs[i].subreg_def = DEF_NOT_SUBREG; } /* frame pointer */ @@ -2261,6 +2290,7 @@ static void init_func_state(struct bpf_verifier_env *env, { state->callsite = callsite; state->frameno = frameno; + bpf_diag_init_frame(env, state); state->subprogno = subprogno; state->callback_ret_range = retval_range(0, 0); init_reg_state(env, state); @@ -2283,6 +2313,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, elem->prev_insn_idx = prev_insn_idx; elem->next = env->head; elem->log_pos = env->log.end_pos; + elem->diag_log_pos = bpf_diag_event_log_save(env); env->head = elem; env->stack_size++; if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { @@ -2309,7 +2340,6 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, return &elem->st; } - static int cmp_subprogs(const void *a, const void *b) { return ((struct bpf_subprog_info *)a)->start - @@ -2490,6 +2520,83 @@ int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, return 0; } +#define BPF_FD_SLOT_BTF 1UL + +static void fd_slot_set_map(struct bpf_fd_array *slot, struct bpf_map *map) +{ + slot->val = (unsigned long)map; +} + +static void fd_slot_set_btf(struct bpf_fd_array *slot, struct btf *btf) +{ + slot->val = (unsigned long)btf | BPF_FD_SLOT_BTF; +} + +static struct bpf_map *fd_slot_map(struct bpf_fd_array slot) +{ + if (slot.val & BPF_FD_SLOT_BTF) + return NULL; + return (struct bpf_map *)slot.val; +} + +static struct btf *fd_slot_btf(struct bpf_fd_array slot) +{ + if (!(slot.val & BPF_FD_SLOT_BTF)) + return NULL; + return (struct btf *)(slot.val & ~BPF_FD_SLOT_BTF); +} + +static struct btf * +fd_array_get_btf_continuous(struct bpf_verifier_env *env, u32 idx) +{ + struct btf *btf; + + if (idx >= env->fd_array_cnt) { + verbose(env, "kfunc fd_idx %u out of bounds, fd_array_cnt %u\n", + idx, env->fd_array_cnt); + return ERR_PTR(-EINVAL); + } + btf = fd_slot_btf(env->fd_array[idx]); + if (!btf) { + verbose(env, "kfunc fd_idx %u is not a module BTF\n", idx); + return ERR_PTR(-EINVAL); + } + btf_get(btf); + return btf; +} + +static struct btf * +fd_array_get_btf_sparse(struct bpf_verifier_env *env, u32 idx) +{ + struct btf *btf; + int btf_fd; + + if (copy_from_bpfptr_offset(&btf_fd, env->fd_array_raw, + (size_t)idx * sizeof(btf_fd), sizeof(btf_fd))) + return ERR_PTR(-EFAULT); + btf = btf_get_by_fd(btf_fd); + if (IS_ERR(btf)) { + verbose(env, "invalid module BTF fd specified\n"); + return btf; + } + return btf; +} + +static struct btf *fd_array_get_btf(struct bpf_verifier_env *env, u32 idx) +{ + if (env->signature) { + verbose(env, "signed program cannot bind any BTF\n"); + return ERR_PTR(-EACCES); + } + if (env->fd_array) + return fd_array_get_btf_continuous(env, idx); + if (!bpfptr_is_null(env->fd_array_raw)) + return fd_array_get_btf_sparse(env, idx); + + verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); + return ERR_PTR(-EPROTO); +} + static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) { @@ -2498,7 +2605,6 @@ static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, struct bpf_kfunc_btf *b; struct module *mod; struct btf *btf; - int btf_fd; tab = env->prog->aux->kfunc_btf_tab; b = bsearch(&kf_btf, tab->descs, tab->nr_descs, @@ -2509,22 +2615,9 @@ static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, return ERR_PTR(-E2BIG); } - if (bpfptr_is_null(env->fd_array)) { - verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); - return ERR_PTR(-EPROTO); - } - - if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, - offset * sizeof(btf_fd), - sizeof(btf_fd))) - return ERR_PTR(-EFAULT); - - btf = btf_get_by_fd(btf_fd); - if (IS_ERR(btf)) { - verbose(env, "invalid module BTF fd specified\n"); + btf = fd_array_get_btf(env, offset); + if (IS_ERR(btf)) return btf; - } - if (!btf_is_module(btf)) { verbose(env, "BTF fd for kfunc is not a module BTF\n"); btf_put(btf); @@ -2582,6 +2675,26 @@ static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) return btf_vmlinux ?: ERR_PTR(-ENOENT); } +static struct btf *find_kfunc_desc_btf_cached(struct bpf_verifier_env *env, s16 offset) +{ + struct bpf_kfunc_btf kf_btf = { .offset = offset }; + struct bpf_kfunc_btf_tab *tab; + struct bpf_kfunc_btf *b; + + if (!offset) + return btf_vmlinux ?: ERR_PTR(-ENOENT); + if (offset < 0) + return ERR_PTR(-EINVAL); + + tab = env->prog->aux->kfunc_btf_tab; + if (!tab) + return ERR_PTR(-ENOENT); + + b = bsearch(&kf_btf, tab->descs, tab->nr_descs, + sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); + return b ? b->btf : ERR_PTR(-ENOENT); +} + #define KF_IMPL_SUFFIX "_impl" static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_log *log, @@ -2674,8 +2787,12 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, return 0; } +static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + struct bpf_func_proto *proto); + int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) { + struct bpf_call_arg_meta meta; struct bpf_kfunc_btf_tab *btf_tab; struct btf_func_model func_model; struct bpf_kfunc_desc_tab *tab; @@ -2715,6 +2832,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) prog_aux->kfunc_tab = tab; } + env->prog->jit_required = 1; + /* func_id == 0 is always invalid, but instead of returning an error, be * conservative and wait until the code elimination pass before returning * error, so that invalid calls that get pruned out can be in BPF programs @@ -2759,26 +2878,41 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) if (err) return err; - desc = &tab->descs[tab->nr_descs++]; + memset(&meta, 0, sizeof(meta)); + meta.btf = kfunc.btf; + meta.func_id = kfunc.id; + meta.func_proto = kfunc.proto; + meta.func_name = kfunc.name; + meta.kfunc_flags = kfunc.flags ? *kfunc.flags : 0; + + tab = krealloc(tab, struct_size(tab, descs, tab->nr_descs + 1), GFP_KERNEL_ACCOUNT); + if (!tab) + return -ENOMEM; + prog_aux->kfunc_tab = tab; + + desc = &tab->descs[tab->nr_descs]; + memset(desc, 0, sizeof(*desc)); + + err = gen_kfunc_arg_proto(env, &meta, &desc->proto); + if (err) + return err; + desc->func_id = func_id; desc->offset = offset; desc->addr = addr; desc->func_model = func_model; + tab->nr_descs++; sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off, NULL); return 0; } -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return !!prog->aux->kfunc_tab; -} - -static int add_subprog_and_kfunc(struct bpf_verifier_env *env) +static int add_subprogs(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprog = env->subprog_info; int i, ret, insn_cnt = env->prog->len, ex_cb_insn; struct bpf_insn *insn = env->prog->insnsi; + const char *operation, *suggestion; /* Add entry function. */ ret = add_subprog(env, 0); @@ -2786,20 +2920,26 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) return ret; for (i = 0; i < insn_cnt; i++, insn++) { - if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) && - !bpf_pseudo_kfunc_call(insn)) + if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn)) continue; if (!env->bpf_capable) { + if (bpf_pseudo_func(insn)) { + operation = "BPF function reference"; + suggestion = "Load this program with the required capability, or avoid BPF function references in unprivileged programs."; + } else { + operation = "BPF-to-BPF function call"; + suggestion = "Load this program with the required capability, or avoid BPF-to-BPF function calls in unprivileged programs."; + } verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); + bpf_diag_policy( + env, i, operation, + "loading or calling other BPF functions requires CAP_BPF or CAP_SYS_ADMIN", + suggestion); return -EPERM; } - if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn)) - ret = add_subprog(env, i + insn->imm + 1); - else - ret = bpf_add_kfunc_call(env, insn->imm, insn->off); - + ret = add_subprog(env, i + insn->imm + 1); if (ret < 0) return ret; } @@ -2837,6 +2977,32 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) return 0; } +static int add_kfuncs(struct bpf_verifier_env *env) +{ + struct bpf_insn *insn = env->prog->insnsi; + int i, ret, insn_cnt = env->prog->len; + + for (i = 0; i < insn_cnt; i++, insn++) { + if (!bpf_pseudo_kfunc_call(insn)) + continue; + + if (!env->bpf_capable) { + verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); + bpf_diag_policy( + env, i, "kernel function call", + "calling kernel functions requires CAP_BPF or CAP_SYS_ADMIN", + "Load this program with the required capability, or avoid kernel function calls in unprivileged programs."); + return -EPERM; + } + + ret = bpf_add_kfunc_call(env, insn->imm, insn->off); + if (ret < 0) + return ret; + } + + return 0; +} + static int check_subprogs(struct bpf_verifier_env *env) { int i, subprog_start, subprog_end, off, cur_subprog = 0; @@ -2870,6 +3036,12 @@ static int check_subprogs(struct bpf_verifier_env *env) off = i + bpf_jmp_offset(&insn[i]) + 1; if (off < subprog_start || off >= subprog_end) { verbose(env, "jump out of range from insn %d to %d\n", i, off); + bpf_diag_program_structure( + env, i, "jump out of range", + "Keep branch targets within the same subprogram, or use an explicit subprogram call.", + "Instruction %d jumps to instruction %d, but subprogram %d only contains instructions %d through %d. " + "A branch target must stay inside the same subprogram.", + i, off, cur_subprog, subprog_start, subprog_end - 1); return -EINVAL; } next: @@ -2882,6 +3054,11 @@ next: code != (BPF_JMP32 | BPF_JA) && code != (BPF_JMP | BPF_JA)) { verbose(env, "last insn is not an exit or jmp\n"); + bpf_diag_program_structure( + env, i, "subprogram can fall through", + "End each subprogram with an exit or an explicit jump that keeps control flow inside the subprogram.", + "Subprogram %d reaches its last instruction %d without an exit or jump, so control could continue into the next subprogram.", + cur_subprog, i); return -EINVAL; } subprog_start = subprog_end; @@ -2952,8 +3129,13 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) if (bpf_pseudo_func(&insn[idx])) continue; verbose(env, "recursive call from %s() to %s()\n", - subprog_name(env, cur), - subprog_name(env, callee)); + bpf_subprog_name(env, cur), + bpf_subprog_name(env, callee)); + bpf_diag_program_structure( + env, idx, "recursive subprogram call", + "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", + "This bpf2bpf call would make the subprogram call graph recursive. " + "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); ret = -EINVAL; goto out; } @@ -2974,7 +3156,7 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) if (env->log.level & BPF_LOG_LEVEL2) for (i = 0; i < cnt; i++) verbose(env, "topo_order[%d] = %s\n", - i, subprog_name(env, env->subprog_topo_order[i])); + i, bpf_subprog_name(env, env->subprog_topo_order[i])); out: kvfree(dfs_stack); kvfree(color); @@ -2990,133 +3172,25 @@ static void mark_stack_slots_scratched(struct bpf_verifier_env *env, mark_stack_slot_scratched(env, spi - i); } -/* This function is supposed to be used by the following 32-bit optimization - * code only. It returns TRUE if the source or destination register operates - * on 64-bit, otherwise return FALSE. - */ -bool bpf_is_reg64(struct bpf_insn *insn, - u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t) -{ - u8 code, class, op; - - code = insn->code; - class = BPF_CLASS(code); - op = BPF_OP(code); - if (class == BPF_JMP) { - /* BPF_EXIT for "main" will reach here. Return TRUE - * conservatively. - */ - if (op == BPF_EXIT) - return true; - if (op == BPF_CALL) { - /* BPF to BPF call will reach here because of marking - * caller saved clobber with DST_OP_NO_MARK for which we - * don't care the register def because they are anyway - * marked as NOT_INIT already. - */ - if (insn->src_reg == BPF_PSEUDO_CALL) - return false; - /* Helper call will reach here because of arg type - * check, conservatively return TRUE. - */ - if (t == SRC_OP) - return true; - - return false; - } - } - - if (class == BPF_ALU64 && op == BPF_END && (insn->imm == 16 || insn->imm == 32)) - return false; - - if (class == BPF_ALU64 || class == BPF_JMP || - (class == BPF_ALU && op == BPF_END && insn->imm == 64)) - return true; - - if (class == BPF_ALU || class == BPF_JMP32) - return false; - - if (class == BPF_LDX) { - if (t != SRC_OP) - return BPF_SIZE(code) == BPF_DW || BPF_MODE(code) == BPF_MEMSX; - /* LDX source must be ptr. */ - return true; - } - - if (class == BPF_STX) { - /* BPF_STX (including atomic variants) has one or more source - * operands, one of which is a ptr. Check whether the caller is - * asking about it. - */ - if (t == SRC_OP && reg->type != SCALAR_VALUE) - return true; - return BPF_SIZE(code) == BPF_DW; - } - - if (class == BPF_LD) { - u8 mode = BPF_MODE(code); - - /* LD_IMM64 */ - if (mode == BPF_IMM) - return true; - - /* Both LD_IND and LD_ABS return 32-bit data. */ - if (t != SRC_OP) - return false; - - /* Implicit ctx ptr. */ - if (regno == BPF_REG_6) - return true; - - /* Explicit source could be any width. */ - return true; - } - - if (class == BPF_ST) - /* The only source register for BPF_ST is a ptr. */ - return true; - - /* Conservatively return true at default. */ - return true; -} - -static void mark_insn_zext(struct bpf_verifier_env *env, - struct bpf_reg_state *reg) -{ - s32 def_idx = reg->subreg_def; - - if (def_idx == DEF_NOT_SUBREG) - return; - - env->insn_aux_data[def_idx - 1].zext_dst = true; - /* The dst will be zero extended, so won't be sub-register anymore. */ - reg->subreg_def = DEF_NOT_SUBREG; -} - static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, enum bpf_reg_arg_type t) { - struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; struct bpf_reg_state *reg; - bool rw64; mark_reg_scratched(env, regno); reg = ®s[regno]; - rw64 = bpf_is_reg64(insn, regno, reg, t); if (t == SRC_OP) { /* check whether register used as source operand can be read */ if (reg->type == NOT_INIT) { verbose(env, "R%d !read_ok\n", regno); + bpf_diag_unreadable_reg(env, env->insn_idx, regno); return -EACCES; } /* We don't need to worry about FP liveness because it's read-only */ if (regno == BPF_REG_FP) return 0; - if (rw64) - mark_insn_zext(env, reg); - return 0; } else { /* check whether register used as dest operand can be written to */ @@ -3124,7 +3198,6 @@ static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *r verbose(env, "frame pointer is read only\n"); return -EACCES; } - reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; if (t == DST_OP) mark_reg_unknown(env, regs, regno); } @@ -3228,7 +3301,7 @@ static void linked_regs_unpack(u64 val, struct linked_regs *s) } } -static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) +const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn) { const struct btf_type *func; struct btf *desc_btf; @@ -3236,18 +3309,20 @@ static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) return NULL; - desc_btf = find_kfunc_desc_btf(data, insn->off); + desc_btf = find_kfunc_desc_btf_cached(data, insn->off); if (IS_ERR(desc_btf)) return "<error>"; func = btf_type_by_id(desc_btf, insn->imm); + if (!func || !btf_type_is_func(func)) + return "<error>"; return btf_name_by_offset(desc_btf, func->name_off); } void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn) { const struct bpf_insn_cbs cbs = { - .cb_call = disasm_kfunc_name, + .cb_call = bpf_disasm_kfunc_name, .cb_print = verbose, .private_data = env, }; @@ -3305,34 +3380,6 @@ static int mark_chain_precision_batch(struct bpf_verifier_env *env, return bpf_mark_chain_precision(env, starting_state, -1, NULL); } -static bool is_spillable_regtype(enum bpf_reg_type type) -{ - switch (base_type(type)) { - case PTR_TO_MAP_VALUE: - case PTR_TO_STACK: - case PTR_TO_CTX: - case PTR_TO_PACKET: - case PTR_TO_PACKET_META: - case PTR_TO_PACKET_END: - case PTR_TO_FLOW_KEYS: - case CONST_PTR_TO_MAP: - case PTR_TO_SOCKET: - case PTR_TO_SOCK_COMMON: - case PTR_TO_TCP_SOCK: - case PTR_TO_XDP_SOCK: - case PTR_TO_BTF_ID: - case PTR_TO_BUF: - case PTR_TO_MEM: - case PTR_TO_FUNC: - case PTR_TO_MAP_KEY: - case PTR_TO_ARENA: - return true; - default: - return false; - } -} - - /* check if register is a constant scalar value */ static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32) { @@ -3346,13 +3393,18 @@ static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32) return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value; } +static bool is_pointer_regtype(enum bpf_reg_type type) +{ + return type != SCALAR_VALUE && type != NOT_INIT; +} + static bool __is_pointer_value(bool allow_ptr_leaks, const struct bpf_reg_state *reg) { if (allow_ptr_leaks) return false; - return reg->type != SCALAR_VALUE; + return is_pointer_regtype(reg->type); } static void clear_scalar_id(struct bpf_reg_state *reg) @@ -3389,6 +3441,7 @@ static void save_register_state(struct bpf_verifier_env *env, { int i; + bpf_diag_mod_begin(env, &state->stack[spi].spilled_ptr, reg, BPF_DIAG_MOD_SPILL); state->stack[spi].spilled_ptr = *reg; for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) @@ -3397,6 +3450,8 @@ static void save_register_state(struct bpf_verifier_env *env, /* size < 8 bytes spill */ for (; i; i--) mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); + + bpf_diag_mod_end(env); } static bool is_bpf_st_mem(struct bpf_insn *insn) @@ -3469,7 +3524,16 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bpf_is_spilled_reg(&state->stack[spi]) && !bpf_is_spilled_scalar_reg(&state->stack[spi]) && size != BPF_REG_SIZE) { + const char *reason; + verbose(env, "attempt to corrupt spilled pointer on stack\n"); + reason = bpf_diag_fmt(env, + "This store writes %d bytes at stack offset %d into a stack slot that currently holds a spilled pointer. " + "Partial writes to spilled pointers are rejected because they can corrupt pointer metadata and leak kernel pointers.", + size, off); + bpf_diag_memory( + env, insn_idx, "stack spill corruption", reason, + "Write the full 8-byte spilled pointer slot, or use a separate stack slot for scalar data before overwriting only part of it."); return -EACCES; } @@ -3477,7 +3541,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, if (value_regno >= 0) reg = &cur->regs[value_regno]; if (!env->bypass_spec_v4) { - bool sanitize = reg && is_spillable_regtype(reg->type); + bool sanitize = reg && is_pointer_regtype(reg->type); for (i = 0; i < size; i++) { u8 type = state->stack[spi].slot_type[(slot - i) % @@ -3518,7 +3582,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, __mark_reg_known(tmp_reg, insn->imm); tmp_reg->type = SCALAR_VALUE; save_register_state(env, state, spi, tmp_reg, size); - } else if (reg && is_spillable_regtype(reg->type)) { + } else if (reg && is_pointer_regtype(reg->type)) { /* register containing pointer is being spilled into stack */ if (size != BPF_REG_SIZE) { verbose_linfo(env, insn_idx, "; "); @@ -3533,6 +3597,9 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, } else { u8 type = STACK_MISC; + if (bpf_is_spilled_reg(&state->stack[spi])) + bpf_diag_record_scrub(env, &state->stack[spi].spilled_ptr, + BPF_DIAG_MOD_WRITE); scrub_special_slot(state, spi); /* when we zero initialize stack slots mark them as such */ @@ -3693,6 +3760,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, if (err) return err; } + bpf_diag_record_scrub_stack(env, state, min_off, max_off, + BPF_DIAG_MOD_VAR_WRITE); return 0; } @@ -3703,14 +3772,21 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, * SCALAR. This function does not deal with register filling; the caller must * ensure that all spilled registers in the stack range have been marked as * read. + * + * STACK_SPILL bytes backed by spilled scalar const zeroes are also considered + * zero bytes. In that case, mark the contributing stack slots precise so + * pruning cannot reuse a zero-spill state for a later non-zero spill state. + * + * Returns an error if precision backtracking fails. */ -static void mark_reg_stack_read(struct bpf_verifier_env *env, - /* func where src register points to */ - struct bpf_func_state *ptr_state, - int min_off, int max_off, int dst_regno) +static int mark_reg_stack_read(struct bpf_verifier_env *env, + /* func where src register points to */ + struct bpf_func_state *ptr_state, + int min_off, int max_off, int dst_regno) { struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; + u64 zero_spill_mask = 0; int i, slot, spi; u8 *stype; int zeros = 0; @@ -3720,19 +3796,48 @@ static void mark_reg_stack_read(struct bpf_verifier_env *env, spi = slot / BPF_REG_SIZE; mark_stack_slot_scratched(env, spi); stype = ptr_state->stack[spi].slot_type; - if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) - break; - zeros++; + if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) { + zeros++; + continue; + } + if (stype[slot % BPF_REG_SIZE] == STACK_SPILL && + bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) { + zero_spill_mask |= 1ull << spi; + zeros++; + continue; + } + break; } if (zeros == max_off - min_off) { /* Any access_size read into register is zero extended, * so the whole register == const_zero. */ __mark_reg_const_zero(env, &state->regs[dst_regno]); + if (zero_spill_mask) { + bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask); + return mark_chain_precision_batch(env, env->cur_state); + } } else { /* have read misc data from the stack */ mark_reg_unknown(env, state->regs, dst_regno); } + + return 0; +} + +static void bpf_diag_stack_read_uninit(struct bpf_verifier_env *env, int off, int i, + int size) +{ + const char *reason; + + reason = bpf_diag_fmt(env, + "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " + "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", + size, off, i); + bpf_diag_memory( + env, env->insn_idx, "uninitialized stack read", reason, + "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " + "or load with CAP_PERFMON if uninitialized stack reads are intended."); } /* Read the stack at 'off' and put the results into the register indicated by @@ -3754,6 +3859,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; struct bpf_reg_state *reg; u8 *stype, type; + int err; int insn_flags = INSN_F_STACK_ACCESS; int hist_spi = spi, hist_frame = reg_state->frameno; @@ -3763,6 +3869,12 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, mark_stack_slot_scratched(env, spi); check_fastcall_stack_contract(env, state, env->insn_idx, off); + /* + * Refine the in-progress load record's origin to the source stack slot. + */ + if (dst_regno >= 0) + bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE); + if (bpf_is_spilled_reg(®_state->stack[spi])) { u8 spill_size = 1; @@ -3781,11 +3893,6 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, if (size <= spill_size && bpf_stack_narrow_access_ok(off, size, spill_size)) { - /* The earlier check_reg_arg() has decided the - * subreg_def for this insn. Save it first. - */ - s32 subreg_def = state->regs[dst_regno].subreg_def; - if (env->bpf_capable && size == 4 && spill_size == 4 && get_reg_width(reg) <= 32) /* Ensure stack slot has an ID to build a relation @@ -3793,7 +3900,6 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, */ assign_scalar_id_before_mov(env, reg); state->regs[dst_regno] = *reg; - state->regs[dst_regno].subreg_def = subreg_def; /* Break the relation on a narrowing fill. * coerce_reg_to_size will adjust the boundaries. @@ -3823,6 +3929,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); + bpf_diag_stack_read_uninit(env, off, i, size); } return -EACCES; } @@ -3836,7 +3943,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, __mark_reg_const_zero(env, &state->regs[dst_regno]); insn_flags = 0; /* not restoring original register state */ } else { - mark_reg_unknown(env, state->regs, dst_regno); + err = mark_reg_stack_read(env, reg_state, off, off + size, + dst_regno); + if (err) + return err; insn_flags = 0; /* not restoring original register state */ } } @@ -3878,11 +3988,15 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); + bpf_diag_stack_read_uninit(env, off, i, size); } return -EACCES; } - if (dst_regno >= 0) - mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); + if (dst_regno >= 0) { + err = mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); + if (err) + return err; + } insn_flags = 0; /* we are not restoring spilled register */ } if (insn_flags) @@ -3936,7 +4050,10 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg min_off = reg_smin(reg) + off; max_off = reg_smax(reg) + off; - mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); + err = mark_reg_stack_read(env, ptr_state, min_off, max_off + size, + dst_regno); + if (err) + return err; check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); return 0; } @@ -3964,11 +4081,19 @@ static int check_stack_read(struct bpf_verifier_env *env, * check_stack_read_fixed_off). */ if (dst_regno < 0 && var_off) { + const char *reason; char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", tn_buf, off, size); + reason = bpf_diag_fmt(env, + "The helper would access the stack through variable offset %s plus fixed offset %d and size %d. " + "Helper stack memory arguments require a constant stack offset and a precise initialized range.", + tn_buf, off, size); + bpf_diag_memory( + env, env->insn_idx, "variable stack access", reason, + "Use a fixed stack offset for helper memory arguments, or copy the needed bytes into a fixed stack slot first."); return -EACCES; } /* Variable offset is prohibited for unprivileged mode for simplicity @@ -3995,7 +4120,6 @@ static int check_stack_read(struct bpf_verifier_env *env, return err; } - /* check_stack_write dispatches to check_stack_write_fixed_off or * check_stack_write_var_off. * @@ -4054,14 +4178,17 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s if (spi + 1 > subprog->max_out_stack_arg_cnt) subprog->max_out_stack_arg_cnt = spi + 1; + arg = &state->stack_arg_regs[spi]; + bpf_diag_mod_begin(env, arg, value_reg, BPF_DIAG_MOD_WRITE); + if (value_reg) { state->stack_arg_regs[spi] = *value_reg; } else { /* BPF_ST: store immediate, treat as scalar */ - arg = &state->stack_arg_regs[spi]; arg->type = SCALAR_VALUE; __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm); } + bpf_diag_mod_end(env); state->no_stack_arg_load = true; return bpf_push_jmp_history(env, env->cur_state, INSN_F_STACK_ARG_ACCESS, spi, 0, 0); @@ -4094,7 +4221,9 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st caller = vstate->frame[vstate->curframe - 1]; arg = &caller->stack_arg_regs[spi]; cur = vstate->frame[vstate->curframe]; + bpf_diag_mod_begin(env, &cur->regs[dst_regno], arg, BPF_DIAG_MOD_WRITE); cur->regs[dst_regno] = *arg; + bpf_diag_mod_end(env); return bpf_push_jmp_history(env, env->cur_state, INSN_F_STACK_ARG_ACCESS, spi, 0, 0); } @@ -4109,7 +4238,8 @@ static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) } static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, - int nargs) + int nargs, const char *callee_name, const struct btf *btf, + const struct btf_param *args) { int i, spi; @@ -4117,8 +4247,14 @@ static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_fu spi = i - MAX_BPF_FUNC_REG_ARGS; if (spi >= caller->out_stack_arg_cnt || caller->stack_arg_regs[spi].type == NOT_INIT) { + const char *arg_name = NULL; + + if (args && args[i].name_off) + arg_name = btf_name_by_offset(btf, args[i].name_off); verbose(env, "callee expects %d args, stack arg%d is not initialized\n", nargs, spi + 1); + bpf_diag_stack_arg_uninit(env, env->insn_idx, nargs, spi, + callee_name, arg_name); return -EFAULT; } } @@ -4199,6 +4335,9 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ int off, int size, u32 mem_size, bool zero_size_allowed) { + const char *proof = ""; + const char *start; + s64 max_start, max_end; int err; /* We may have adjusted the register pointing to memory region, so we @@ -4217,14 +4356,28 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ reg_smin(reg) + off < 0)) { verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", reg_arg_name(env, argno)); - return -EACCES; + err = -EACCES; + if (bpf_diag_enabled(env)) { + start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off); + proof = bpf_diag_fmt( + env, "the minimal bound for a memory access is a negative value: %s", + start); + } + goto report_error; } + err = __check_mem_access(env, reg, argno, reg_smin(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s min value is outside of the allowed memory range\n", reg_arg_name(env, argno)); - return err; + if (bpf_diag_enabled(env)) { + start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off); + proof = bpf_diag_fmt( + env, "the minimal bound for a memory access is %s and is outside of the object of size %u", + start, mem_size); + } + goto report_error; } /* If we haven't set a max value then we need to bail since we can't be @@ -4234,17 +4387,36 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ if (reg_umax(reg) >= BPF_MAX_VAR_OFF) { verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n", reg_arg_name(env, argno)); - return -EACCES; + err = -EACCES; + if (bpf_diag_enabled(env)) + proof = bpf_diag_fmt( + env, "the maximal bound for a memory access is %llu and exceeds maximum allowed offset of %u", + reg_umax(reg), BPF_MAX_VAR_OFF); + goto report_error; } + err = __check_mem_access(env, reg, argno, reg_umax(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s max value is outside of the allowed memory range\n", reg_arg_name(env, argno)); - return err; + if (bpf_diag_enabled(env)) { + max_start = (s64)reg_umax(reg) + off; + max_end = max_start + size; + proof = bpf_diag_fmt( + env, "the maximal bound for a memory access is %lld: start %lld + access_size %d, beyond object_size %u", + max_end, max_start, size, mem_size); + } + goto report_error; } return 0; + +report_error: + bpf_diag_mem_bounds(env, env->insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg_type_str(env, reg->type), proof, + off, size, mem_size, reg); + return err; } static int __check_ptr_off_reg(struct bpf_verifier_env *env, @@ -4273,6 +4445,9 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env, if (!fixed_off_ok && reg->var_off.value != 0) { verbose(env, "dereference of modified %s ptr %s off=%lld disallowed\n", reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); + bpf_diag_invalid_deref(env, env->insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, + BPF_DIAG_DEREF_MODIFIED_PTR, reg->var_off.value); return -EACCES; } @@ -4349,7 +4524,8 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, */ if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, kptr_field->kptr.btf, kptr_field->kptr.btf_id, - kptr_field->type != BPF_KPTR_UNREF)) + kptr_field->type != BPF_KPTR_UNREF, + !type_is_alloc(reg->type))) goto bad_type; return 0; bad_type: @@ -4375,7 +4551,9 @@ static bool in_sleepable(struct bpf_verifier_env *env) static bool in_rcu_cs(struct bpf_verifier_env *env) { return env->cur_state->active_rcu_locks || + env->cur_state->active_preempt_locks || env->cur_state->active_locks || + env->cur_state->active_irq_id || !in_sleepable(env); } @@ -4394,6 +4572,9 @@ BTF_ID(struct, task_struct) #ifdef CONFIG_CRYPTO BTF_ID(struct, bpf_crypto_ctx) #endif +#ifdef CONFIG_INET +BTF_ID(struct, bpf_ksock) +#endif BTF_SET_END(rcu_protected_types) static bool rcu_protected_object(const struct btf *btf, u32 btf_id) @@ -4604,7 +4785,7 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state * } static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, - const struct bpf_call_arg_meta *meta, + const struct bpf_func_proto *fn, enum bpf_access_type t) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -4628,8 +4809,8 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, case BPF_PROG_TYPE_LWT_XMIT: case BPF_PROG_TYPE_SK_SKB: case BPF_PROG_TYPE_SK_MSG: - if (meta) - return meta->pkt_access; + if (fn) + return fn->pkt_access; env->seen_direct_write = true; return true; @@ -4790,7 +4971,6 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, valid = false; } - if (valid) { env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; @@ -4844,6 +5024,22 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, + struct bpf_insn *insn) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + /* + * A BPF_LOAD_ACQ is not rewritten to a BPF_PROBE_MEM load by the + * verifier, unlike a regular BPF_LDX. The JIT would emit a plain load + * with no exception table entry, so a fault (e.g. NULL deref) crashes + * the kernel instead of being handled. Reject the source pointer types + * that would have needed that protection, the remaining ones stay + * allowed. + */ + return insn->imm == BPF_LOAD_ACQ && bpf_may_fault_on_deref(reg->type); +} + /* Return false if @regno contains a pointer whose type isn't supported for * atomic instruction @insn. */ @@ -4860,7 +5056,8 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_arena_reg(env, regno)) return bpf_jit_supports_insn(insn, true); - + if (is_load_acq_unsafe(env, regno, insn)) + return false; return true; } @@ -4873,6 +5070,18 @@ static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = { [CONST_PTR_TO_MAP] = btf_bpf_map_id, }; +static enum bpf_reg_type lookup_reg2btf_ids(u32 ref_id) +{ + enum bpf_reg_type type; + + for (type = 0; type < __BPF_REG_TYPE_MAX; type++) { + if (reg2btf_ids[type] && *reg2btf_ids[type] == ref_id) + return type; + } + + return NOT_INIT; +} + static bool is_trusted_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) { /* A referenced register is always trusted. */ @@ -5187,10 +5396,6 @@ continue_func: if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn)) return -EFAULT; if (subprog[sidx].is_async_cb) { - if (subprog[sidx].has_tail_call) { - verifier_bug(env, "subprog has tail_call and async cb"); - return -EFAULT; - } /* async callbacks don't increase bpf prog stack size unless called directly */ if (!bpf_pseudo_call(insn + i)) continue; @@ -5213,8 +5418,8 @@ continue_func: if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; - if (subprog[idx].has_tail_call) - tail_call_reachable = true; + /* sync tail_call_reachable with callee state on entry */ + tail_call_reachable = subprog[idx].has_tail_call; frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1; if (frame >= MAX_CALL_FRAMES) { @@ -5231,8 +5436,8 @@ continue_func: */ if (tail_call_reachable) { for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { - if (subprog[tmp].is_exception_cb) { - verbose(env, "cannot tail call within exception cb\n"); + if (subprog[tmp].is_cb) { + verbose(env, "cannot tail call within callback\n"); return -EINVAL; } if (subprog[tmp].stack_arg_cnt) { @@ -5577,6 +5782,8 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, u64 addr; int err; + if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY || map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) + return -EINVAL; err = map->ops->map_direct_value_addr(map, &addr, off); if (err) return err; @@ -5789,6 +5996,11 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, return -EACCES; } + if (atype != BPF_READ && bpf_may_fault_on_deref(reg->type)) { + verbose(env, "only read is supported\n"); + return -EACCES; + } + if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { if (!btf_is_kernel(reg->btf)) { verifier_bug(env, "reg->btf must be kernel btf"); @@ -5801,8 +6013,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, reg_arg_name(env, argno), tname, off, size); } else { /* Writes are permitted with default btf_struct_access for - * program allocated objects (which always have id > 0), - * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC. + * program allocated objects (which always have id > 0). */ if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) { verbose(env, "only read is supported\n"); @@ -6069,6 +6280,51 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val) reg_bounds_sync(dst_reg); } +static int check_map_mem_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int off, + int bpf_size, int value_regno, bool is_ldsx) +{ + struct bpf_reg_state *regs = cur_regs(env); + int size = bpf_size_to_bytes(bpf_size); + struct bpf_map *map = reg->map_ptr; + + switch (map->map_type) { + case BPF_MAP_TYPE_INSN_ARRAY: + if (bpf_size != BPF_DW) { + verbose(env, "Invalid read of %d bytes from insn_array\n", size); + return -EACCES; + } + regs[value_regno] = *reg; + add_scalar_to_reg(®s[value_regno], off); + regs[value_regno].type = PTR_TO_INSN; + return 0; + case BPF_MAP_TYPE_PERCPU_ARRAY: + goto reg_unknown; + default: + break; + } + + /* If map is read-only, track its contents as scalars. */ + if (tnum_is_const(reg->var_off) && + bpf_map_is_rdonly(map) && + map->ops->map_direct_value_addr) { + int map_off = off + reg->var_off.value; + u64 val = 0; + int err; + + err = bpf_map_direct_read(map, map_off, size, &val, is_ldsx); + if (err) + return err; + + regs[value_regno].type = SCALAR_VALUE; + __mark_reg_known(®s[value_regno], val); + return 0; + } + +reg_unknown: + mark_reg_unknown(env, regs, value_regno); + return 0; +} + /* check whether memory at (regno + off) is accessible for t = (read | write) * if t==write, value_regno is a register which value is stored into memory * if t==read, value_regno is a register which will receive the value from memory @@ -6123,38 +6379,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (kptr_field) { err = check_map_kptr_access(env, value_regno, insn_idx, kptr_field); } else if (t == BPF_READ && value_regno >= 0) { - struct bpf_map *map = reg->map_ptr; - - /* - * If map is read-only, track its contents as scalars, - * unless it is an insn array (see the special case below) - */ - if (tnum_is_const(reg->var_off) && - bpf_map_is_rdonly(map) && - map->ops->map_direct_value_addr && - map->map_type != BPF_MAP_TYPE_INSN_ARRAY) { - int map_off = off + reg->var_off.value; - u64 val = 0; - - err = bpf_map_direct_read(map, map_off, size, - &val, is_ldsx); - if (err) - return err; - - regs[value_regno].type = SCALAR_VALUE; - __mark_reg_known(®s[value_regno], val); - } else if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { - if (bpf_size != BPF_DW) { - verbose(env, "Invalid read of %d bytes from insn_array\n", - size); - return -EACCES; - } - regs[value_regno] = *reg; - add_scalar_to_reg(®s[value_regno], off); - regs[value_regno].type = PTR_TO_INSN; - } else { - mark_reg_unknown(env, regs, value_regno); - } + err = check_map_mem_read(env, reg, off, bpf_size, value_regno, is_ldsx); } } else if (base_type(reg->type) == PTR_TO_MEM) { bool rdonly_mem = type_is_rdonly_mem(reg->type); @@ -6163,6 +6388,9 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (type_may_be_null(reg->type)) { verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, + BPF_DIAG_DEREF_NULLABLE_PTR, 0); return -EACCES; } @@ -6220,12 +6448,6 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b } else { mark_reg_known_zero(env, regs, value_regno); - /* A load of ctx field could have different - * actual load size with the one encoded in the - * insn. When the dst is PTR, it is for sure not - * a sub-register. - */ - regs[value_regno].subreg_def = DEF_NOT_SUBREG; if (base_type(info.reg_type) == PTR_TO_BTF_ID) { regs[value_regno].btf = info.btf; regs[value_regno].btf_id = info.btf_id; @@ -6319,18 +6541,38 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else { + enum bpf_diag_invalid_deref_kind kind = BPF_DIAG_DEREF_INVALID_PTR; + verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + if (reg->type == SCALAR_VALUE) + kind = BPF_DIAG_DEREF_SCALAR; + else if (type_may_be_null(reg->type)) + kind = BPF_DIAG_DEREF_NULLABLE_PTR; + bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, kind, 0); return -EACCES; } if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && regs[value_regno].type == SCALAR_VALUE) { - if (!is_ldsx) + if (!is_ldsx) { /* b/h/w load zero-extends, mark upper bits as known 0 */ coerce_reg_to_size(®s[value_regno], size); - else + } else { + /* + * Sign-extension can change the register value relative + * to a scalar it is linked with by id (e.g. a zero- + * extending fill of the same spilled stack slot), thus + * drop the shared id in that case. + */ + bool no_sext = reg_umax(®s[value_regno]) < + (1ULL << (size * BITS_PER_BYTE - 1)); + coerce_reg_to_size_sx(®s[value_regno], size); + if (!no_sext) + clear_scalar_id(®s[value_regno]); + } } return err; } @@ -6368,15 +6610,19 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, src_reg_type = regs[insn->src_reg].type; - /* Check if (src_reg + off) is readable. The state of dst_reg will be - * updated by this call. + /* + * check_stack_read_fixed_off() may refine the modification's origin to + * the source stack slot. */ + bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); + if (!err) + bpf_diag_mod_end(env); return err; } @@ -6467,21 +6713,12 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } - if (insn->imm & BPF_FETCH) { - if (insn->imm == BPF_CMPXCHG) - load_reg = BPF_REG_0; - else - load_reg = insn->src_reg; - + load_reg = bpf_atomic_load_reg(insn); + if (load_reg >= 0) { /* check and record load of old value */ err = check_reg_arg(env, load_reg, DST_OP); if (err) return err; - } else { - /* This instruction accesses a memory location but doesn't - * actually load it into a register. - */ - load_reg = -1; } dst_reg = cur_regs(env) + insn->dst_reg; @@ -6491,18 +6728,20 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, -1, true, false); - if (!err && load_reg >= 0) + if (!err && load_reg >= 0) { + bpf_diag_mod_begin(env, cur_regs(env) + load_reg, NULL, BPF_DIAG_MOD_WRITE); err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, load_reg, true, false); + if (!err) + bpf_diag_mod_end(env); + } if (err) return err; - if (is_arena_reg(env, insn->dst_reg)) { - err = save_aux_ptr_type(env, PTR_TO_ARENA, false); - if (err) - return err; - } + err = save_aux_ptr_type(env, dst_reg->type, false); + if (err) + return err; /* Check whether we can write into the same memory. */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, true, false); @@ -6516,7 +6755,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, { int err; - err = check_load_mem(env, insn, true, false, false, "atomic_load"); + err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err; @@ -6527,7 +6766,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, return -EACCES; } - return 0; + return check_load_mem(env, insn, true, false, false, "atomic_load"); } static int check_atomic_store(struct bpf_verifier_env *env, @@ -6604,12 +6843,14 @@ static int check_stack_range_initialized( */ bool clobber = type == BPF_WRITE; /* - * Negative access_size signals global subprog/kfunc arg check where + * Negative access_size signals global subprog arg check where * STACK_POISON slots are acceptable. static stack liveness * might have determined that subprog doesn't read them, * but BTF based global subprog validation isn't accurate enough. */ bool allow_poison = access_size < 0 || clobber; + /* The call will initialize the memory; uninitialized stack allowed */ + bool raw_mode = meta && meta->arg_raw_mem.regno == reg_from_argno(argno); access_size = abs(access_size); @@ -6622,7 +6863,6 @@ static int check_stack_range_initialized( if (err) return err; - if (tnum_is_const(reg->var_off)) { min_off = max_off = reg->var_off.value + off; } else { @@ -6645,41 +6885,14 @@ static int check_stack_range_initialized( * helper return since specific bounds are unknown what may * cause uninitialized stack leaking. */ - if (meta && meta->raw_mode) - meta = NULL; + raw_mode = false; min_off = reg_smin(reg) + off; max_off = reg_smax(reg) + off; } - if (meta && meta->raw_mode) { - /* Ensure we won't be overwriting dynptrs when simulating byte - * by byte access in check_helper_call using meta.access_size. - * This would be a problem if we have a helper in the future - * which takes: - * - * helper(uninit_mem, len, dynptr) - * - * Now, uninint_mem may overlap with dynptr pointer. Hence, it - * may end up writing to dynptr itself when touching memory from - * arg 1. This can be relaxed on a case by case basis for known - * safe cases, but reject due to the possibilitiy of aliasing by - * default. - */ - for (i = min_off; i < max_off + access_size; i++) { - int stack_off = -i - 1; - - spi = bpf_get_spi(i); - /* raw_mode may write past allocated_stack */ - if (state->allocated_stack <= stack_off) - continue; - if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) { - verbose(env, "potential write to dynptr at off=%d disallowed\n", i); - return -EACCES; - } - } - meta->access_size = access_size; - meta->regno = reg_from_argno(argno); + if (raw_mode) { + meta->arg_raw_mem.size = access_size; return 0; } @@ -6738,14 +6951,17 @@ mark: return 0; } -static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - int access_size, enum bpf_access_type access_type, - bool zero_size_allowed, - struct bpf_call_arg_meta *meta) +static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, int access_size, + enum bpf_access_type access_type, bool zero_size_allowed, + struct bpf_call_arg_meta *meta, bool *known_memory) { struct bpf_reg_state *regs = cur_regs(env); u32 *max_access; + if (known_memory) + *known_memory = true; + switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: @@ -6815,6 +7031,8 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ if (zero_size_allowed && access_size == 0 && bpf_register_is_null(reg)) return 0; + if (known_memory && base_type(reg->type) != PTR_TO_CTX) + *known_memory = false; verbose(env, "%s type=%s ", reg_arg_name(env, argno), reg_type_str(env, reg->type)); @@ -6823,6 +7041,12 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ } } +enum bpf_mem_size_failure { + BPF_MEM_SIZE_FAIL_NONE, + BPF_MEM_SIZE_FAIL_MEMORY, + BPF_MEM_SIZE_FAIL_SIZE, +}; + /* verify arguments to helpers or kfuncs consisting of a pointer and an access * size. * @@ -6831,11 +7055,15 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ static int check_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, struct bpf_reg_state *size_reg, argno_t mem_argno, - argno_t size_argno, enum bpf_access_type access_type, + argno_t size_argno, u32 access_type, bool zero_size_allowed, - struct bpf_call_arg_meta *meta) + struct bpf_call_arg_meta *meta, + enum bpf_mem_size_failure *failure) { - int err; + int err = 0; + + if (failure) + *failure = BPF_MEM_SIZE_FAIL_NONE; /* This is used to refine r0 return value bounds for helpers * that enforce this value as an upper bound on return values. @@ -6858,22 +7086,33 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", reg_arg_name(env, size_argno)); - return -EACCES; + err = -EACCES; + goto size_error; } if (reg_umin(size_reg) == 0 && !zero_size_allowed) { verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n", reg_arg_name(env, size_argno), reg_umin(size_reg), reg_umax(size_reg)); - return -EACCES; + err = -EACCES; + goto size_error; } if (reg_umax(size_reg) >= BPF_MAX_VAR_SIZ) { verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n", reg_arg_name(env, size_argno)); - return -EACCES; + err = -EACCES; + goto size_error; } - err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), - access_type, zero_size_allowed, meta); + + if (access_type & BPF_READ) + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), + BPF_READ, zero_size_allowed, meta, NULL); + if (!err && access_type & BPF_WRITE) + err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), + BPF_WRITE, zero_size_allowed, meta, NULL); + if (err && failure) + *failure = BPF_MEM_SIZE_FAIL_MEMORY; + if (!err) { int regno = reg_from_argno(size_argno); @@ -6882,18 +7121,25 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, else err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno)); } + + return err; + +size_error: + if (failure) + *failure = BPF_MEM_SIZE_FAIL_SIZE; return err; } static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, u32 mem_size) + argno_t argno, u32 mem_size, enum bpf_access_type access_type, + struct bpf_call_arg_meta *meta, bool *known_memory) { - bool may_be_null = type_may_be_null(reg->type); - struct bpf_reg_state saved_reg; - int err; + int size, err = 0; if (bpf_register_is_null(reg)) return 0; + if (known_memory) + *known_memory = true; if (mem_size > S32_MAX) { verbose(env, "%s memory size %u is too large\n", @@ -6901,48 +7147,83 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg return -EACCES; } - /* Assuming that the register contains a value check if the memory - * access is safe. Temporarily save and restore the register's state as - * the conversion shouldn't be visible to a caller. + /* + * Only a global subprog (meta == NULL) may read poisoned stack slots: + * its static stack liveness proved the callee body skips them. */ - if (may_be_null) { - saved_reg = *reg; - mark_ptr_not_null_reg(reg); + size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; + + if (access_type & BPF_READ) + err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta, + known_memory); + if (!err && (access_type & BPF_WRITE)) + err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta, + known_memory); + + return err; +} + +static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct ret_mem_desc *ret_mem) +{ + int regno = reg_from_argno(argno); + int err; + + if (ret_mem->found) { + verifier_bug(env, "only one allocation size argument permitted"); + return -EFAULT; } - int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size; + if (!tnum_is_const(reg->var_off)) { + verbose(env, "%s is not a const\n", reg_arg_name(env, argno)); + return -EINVAL; + } - err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, NULL); - err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, NULL); + if (reg->var_off.value > U32_MAX) { + verbose(env, "%s allocation size exceeds u32 max\n", reg_arg_name(env, argno)); + return -EINVAL; + } - if (may_be_null) - *reg = saved_reg; + if (regno >= 0) + err = mark_chain_precision(env, regno); + else + err = mark_stack_arg_precision(env, arg_idx_from_argno(argno)); + if (err) + return err; - return err; + ret_mem->size = reg->var_off.value; + ret_mem->found = true; + + return 0; } -static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg, - struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno) +static int process_const_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct bpf_call_arg_meta *meta) { - bool may_be_null = type_may_be_null(mem_reg->type); - struct bpf_reg_state saved_reg; - struct bpf_call_arg_meta meta; + int regno = reg_from_argno(argno); int err; - memset(&meta, 0, sizeof(meta)); + if (meta->arg_constant.found) { + verifier_bug(env, "only one constant argument permitted"); + return -EFAULT; + } - if (may_be_null) { - saved_reg = *mem_reg; - mark_ptr_not_null_reg(mem_reg); + if (!tnum_is_const(reg->var_off)) { + verbose(env, "%s must be a known constant\n", reg_arg_name(env, argno)); + return -EINVAL; } - err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, &meta); - err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, &meta); + if (regno >= 0) + err = mark_chain_precision(env, regno); + else + err = mark_stack_arg_precision(env, arg_idx_from_argno(argno)); + if (err < 0) + return err; - if (may_be_null) - *mem_reg = saved_reg; + meta->arg_constant.found = true; + meta->arg_constant.value = reg->var_off.value; - return err; + return 0; } enum { @@ -6978,6 +7259,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK; const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin"; struct bpf_verifier_state *cur = env->cur_state; + struct bpf_reference_state *lock; bool is_const = tnum_is_const(reg->var_off); bool is_irq = flags & PROCESS_LOCK_IRQ; u64 val = reg->var_off.value; @@ -7027,14 +7309,25 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state ptr = btf; if (!is_res_lock && cur->active_locks) { - if (find_lock_state(env->cur_state, REF_TYPE_LOCK, 0, NULL)) { + lock = find_lock_state(cur, REF_TYPE_LOCK, 0, NULL); + if (lock) { verbose(env, "Locking two bpf_spin_locks are not allowed\n"); + bpf_diag_lock( + env, env->insn_idx, "nested spin lock", + "This path already holds a bpf_spin_lock. The verifier allows only one regular BPF spin lock at a time.", + "Unlock the current bpf_spin_lock before taking another one.", lock); return -EINVAL; } } else if (is_res_lock && cur->active_locks) { - if (find_lock_state(env->cur_state, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, reg->id, ptr)) { + lock = find_lock_state(cur, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, + reg->id, ptr); + if (lock) { verbose(env, "Acquiring the same lock again, AA deadlock detected\n"); + bpf_diag_lock( + env, env->insn_idx, "recursive resource spin lock", + "This path already holds the same resource spin lock. Taking it again would deadlock.", + "Avoid reacquiring the same resource spin lock before it is unlocked.", lock); return -EINVAL; } } @@ -7061,6 +7354,10 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state if (!cur->active_locks) { verbose(env, "%s_unlock without taking a lock\n", lock_str); + bpf_diag_res( + env, env->insn_idx, "unlock without lock", + "This unlock operation has no matching active lock on the current path.", + "Take the matching lock before this unlock, or remove the unmatched unlock path."); return -EINVAL; } @@ -7070,18 +7367,39 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state type = REF_TYPE_RES_LOCK; else type = REF_TYPE_LOCK; - if (!find_lock_state(cur, type, reg->id, ptr)) { + + lock = find_lock_state(cur, type, reg->id, ptr); + if (!lock) { verbose(env, "%s_unlock of different lock\n", lock_str); + lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id, + cur->active_lock_ptr); + bpf_diag_lock( + env, env->insn_idx, "unlock of a different lock", + "This unlock does not match any active lock with the same tracked identity on the current path.", + "Unlock the same lock object that was most recently acquired.", lock); return -EINVAL; } if (reg->id != cur->active_lock_id || ptr != cur->active_lock_ptr) { verbose(env, "%s_unlock cannot be out of order\n", lock_str); + lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id, + cur->active_lock_ptr); + bpf_diag_lock( + env, env->insn_idx, "unlock out of order", + "Locks must be released in last-in, first-out order, but this unlock does not match the currently active lock.", + "Release nested locks in the reverse order they were acquired.", lock); return -EINVAL; } - if (release_lock_state(cur, type, reg->id, ptr)) { + if (release_lock_state(env, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); + bpf_diag_lock( + env, env->insn_idx, "unlock of a different lock", + "The verifier could not release a lock state matching this unlock operation.", + "Pass the same lock object and lock kind that were used for the matching lock operation.", + lock); return -EINVAL; } + if (!in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); invalidate_non_owning_refs(env); } @@ -7152,18 +7470,6 @@ static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state return check_map_field_pointer(env, reg, argno, BPF_TIMER, map); } -static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_call_arg_meta *meta) -{ - return process_timer_func(env, reg, argno, &meta->map); -} - -static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) -{ - return process_timer_func(env, reg, argno, &meta->map); -} - static int process_kptr_func(struct bpf_verifier_env *env, int regno, struct bpf_call_arg_meta *meta) { @@ -7212,6 +7518,12 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, return 0; } +static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, + const char *call_name, const char *reason, const char *suggestion); +__printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx, + argno_t argno, const char *call_name, + const char *suggestion, const char *fmt, ...); + /* * Validate dynptr arguments for helper, kfunc and subprog. * @@ -7236,7 +7548,8 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, * and checked dynamically during runtime. */ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, int insn_idx, enum bpf_arg_type arg_type, + argno_t argno, int insn_idx, const char *call_name, + enum bpf_arg_type arg_type, struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) { int spi, err = 0; @@ -7245,6 +7558,11 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "%s expected pointer to stack or const struct bpf_dynptr\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, call_name, + "Pass the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path.", + "a dynptr argument must be a pointer to a dynptr stack slot or a verifier-provided const struct bpf_dynptr, but %s is %s", + reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -7267,6 +7585,10 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat if (!is_dynptr_reg_valid_uninit(env, reg)) { verbose(env, "Dynptr has to be an uninitialized dynptr\n"); + bpf_diag_res( + env, insn_idx, "dynptr is already initialized", + "This kfunc constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", + "Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot."); return -EINVAL; } @@ -7283,21 +7605,35 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { verbose(env, "CONST_PTR_TO_DYNPTR cannot be released\n"); + bpf_diag_res( + env, insn_idx, "const dynptr release", + "This release operation was given a const dynptr. Const dynptr values are verifier-provided views and cannot be released by the program.", + "Release only mutable dynptrs that the program initialized or reserved."); return -EINVAL; } if (!is_dynptr_reg_valid_init(env, reg)) { verbose(env, "Expected an initialized dynptr as %s\n", reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "uninitialized dynptr use", + "This operation requires an initialized dynptr, but the stack slot does not currently hold a valid dynptr on this path.", + "Initialize the dynptr on every path before this call, and avoid overwriting or releasing it before this use."); return -EINVAL; } /* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */ if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) { - verbose(env, - "Expected a dynptr of type %s as %s\n", - dynptr_type_str(arg_to_dynptr_type(arg_type)), - reg_arg_name(env, argno)); + enum bpf_dynptr_type expected_type = arg_to_dynptr_type(arg_type); + enum bpf_dynptr_type actual_type = dynptr_reg_type(env, reg); + + verbose(env, "Expected a dynptr of type %s as %s\n", + dynptr_type_str(expected_type), reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, call_name, + "Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.", + "the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s", + dynptr_type_str(actual_type), dynptr_type_str(expected_type)); return -EINVAL; } @@ -7308,10 +7644,6 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat if (spi < 0) return spi; - /* - * For CONST_PTR_TO_DYNPTR, reg is already scratched by check_reg_arg - * in check_helper_call and mark_btf_func_reg_size in check_kfunc_call. - */ mark_stack_slots_scratched(env, spi, BPF_DYNPTR_NR_SLOTS); reg = &state->stack[spi].spilled_ptr; @@ -7326,23 +7658,22 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat return err; } -static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static bool is_iter_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY); } -static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static bool is_iter_new_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ITER_NEW; } - -static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta) +static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ITER_DESTROY; } -static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx, +static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, const struct btf_param *arg) { /* btf_check_iter_kfuncs() guarantees that first argument of any iter @@ -7356,7 +7687,7 @@ static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx, } static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { struct bpf_func_state *state = bpf_func(env, reg); const struct btf_type *t; @@ -7366,6 +7697,11 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (reg->type != PTR_TO_STACK) { verbose(env, "%s expected pointer to an iterator on stack\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, meta->func_name, + "Pass the address of a stack iterator object for iterator new, next, and destroy calls.", + "iterator state must live in verifier-tracked stack memory, but %s is %s", + reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -7379,6 +7715,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (btf_id < 0) { verbose(env, "expected valid iter pointer as %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, insn_idx, argno, meta->func_name, + "the kfunc expects a recognized iterator state pointer, but this argument does not match a valid iterator type", + "Pass the exact iterator state type expected by this kfunc."); return -EINVAL; } t = btf_type_by_id(meta->btf, btf_id); @@ -7389,6 +7729,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { verbose(env, "expected uninitialized iter_%s as %s\n", iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "iterator is already initialized", + "Iterator creation requires an uninitialized iterator stack object, but this stack range already contains iterator state.", + "Use a fresh iterator stack slot, or destroy the existing iterator before reusing the slot."); return -EINVAL; } @@ -7413,9 +7757,16 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * case -EINVAL: verbose(env, "expected an initialized iter_%s as %s\n", iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "uninitialized iterator use", + "This iterator operation requires an initialized iterator state object, but the stack range does not contain a live iterator on this path.", + "Call the matching iterator new kfunc on every path before calling next or destroy, and do not destroy the iterator before this use."); return err; case -EPROTO: verbose(env, "expected an RCU CS when using %s\n", meta->func_name); + bpf_diag_ctx_required( + env, insn_idx, meta->func_name, BPF_DIAG_CONTEXT_RCU, + "Wrap iterator use in bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced."); return err; default: return err; @@ -7523,7 +7874,7 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env, } static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_st, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { int iter_frameno = meta->iter.frameno; int iter_spi = meta->iter.spi; @@ -7610,7 +7961,7 @@ static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_ * bpf_iter_num_destroy(&it); */ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr; @@ -7666,13 +8017,18 @@ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, static bool arg_type_is_mem_size(enum bpf_arg_type type) { - return type == ARG_CONST_SIZE || - type == ARG_CONST_SIZE_OR_ZERO; + return type == ARG_MEM_SIZE || type == ARG_MEM_SIZE_OR_ZERO; } static bool arg_type_is_raw_mem(enum bpf_arg_type type) { - return base_type(type) == ARG_PTR_TO_MEM && + /* + * A map value output buffer (e.g. bpf_map_pop_elem) is also a raw + * (uninitialized) memory argument, and like ARG_PTR_TO_MEM it may be + * passed as a PTR_TO_STACK that reaches check_stack_range_initialized(). + */ + return (base_type(type) == ARG_PTR_TO_MEM || + base_type(type) == ARG_PTR_TO_MAP_VALUE) && type & MEM_UNINIT; } @@ -7807,8 +8163,8 @@ static const struct bpf_reg_types dynptr_types = { static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_MAP_KEY] = &mem_types, [ARG_PTR_TO_MAP_VALUE] = &mem_types, - [ARG_CONST_SIZE] = &scalar_types, - [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, + [ARG_MEM_SIZE] = &scalar_types, + [ARG_MEM_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, @@ -7830,13 +8186,70 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_DYNPTR] = &dynptr_types, }; +static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, + const char *call_name, const char *reason, + const char *suggestion) +{ + int arg = arg_from_argno(argno); + int regno = reg_from_argno(argno); + int stack_slot = -1; + + if (arg < 0 && regno >= BPF_REG_1 && regno <= BPF_REG_5) + arg = regno; + if (arg > MAX_BPF_FUNC_REG_ARGS) + stack_slot = arg - MAX_BPF_FUNC_REG_ARGS - 1; + + bpf_diag_call_type(env, insn_idx, arg, regno, stack_slot, + call_name && *call_name ? call_name : "call", + reg_arg_name(env, argno), reason, suggestion); +} + +static const char *bpf_diag_arg_name(struct bpf_verifier_env *env, argno_t argno) +{ + return bpf_diag_fmt(env, "%s", reg_arg_name(env, argno)); +} + +__printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx, + argno_t argno, const char *call_name, + const char *suggestion, const char *fmt, ...) +{ + const char *reason; + va_list args; + + va_start(args, fmt); + reason = bpf_diag_vfmt(env, fmt, args); + va_end(args); + + bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion); +} + +static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, + const enum bpf_reg_type *types, int count) +{ + size_t len = 0, size = 1; + char *buf; + int i; + + for (i = 0; i < count; i++) + size += strlen(reg_type_str(env, types[i])) + (i ? 2 : 0); + + buf = bpf_diag_fmt_buf(env, size); + if (!buf) + return ""; + + for (i = 0; i < count; i++) + len += scnprintf(buf + len, size - len, "%s%s", i ? ", " : "", + reg_type_str(env, types[i])); + return buf; +} + static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, - const u32 *arg_btf_id, - struct bpf_call_arg_meta *meta) + enum bpf_arg_type arg_type, const u32 *arg_btf_id, + struct bpf_call_arg_meta *meta, const char *call_name) { enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; + const char *actual, *accepted; int i, j, err; compatible = compatible_reg_types[base_type(arg_type)]; @@ -7883,6 +8296,12 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re for (j = 0; j + 1 < i; j++) verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); + actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type)); + accepted = bpf_diag_expected_reg_types(env, compatible->types, i); + bpf_diag_call_arg_fmt(env, env->insn_idx, argno, call_name, + "Pass a value with one of the accepted pointer or scalar types for this call.", + "it has type %s, but this argument accepts %s", + actual, accepted); return -EACCES; found: @@ -7919,6 +8338,10 @@ found: (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { verbose(env, "Possibly NULL pointer passed to helper %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, env->insn_idx, argno, call_name, + "the pointer may be NULL, but this call requires a non-NULL pointer", + "Add a NULL check and make the call only on the non-NULL path."); return -EACCES; } @@ -7947,7 +8370,7 @@ found: if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value, btf_vmlinux, *arg_btf_id, - strict_type_match)) { + strict_type_match, !type_is_alloc(reg->type))) { verbose(env, "%s is of type %s but %s is expected\n", reg_arg_name(env, argno), btf_type_name(reg->btf, reg->btf_id), @@ -8109,6 +8532,12 @@ static int check_arg_const_str(struct bpf_verifier_env *env, return -EACCES; } + if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { + verbose(env, "%s points to percpu_array map which cannot be used as const string\n", + reg_arg_name(env, argno)); + return -EACCES; + } + if (!bpf_map_is_rdonly(map)) { verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno)); return -EACCES; @@ -8208,15 +8637,53 @@ static int get_constant_map_key(struct bpf_verifier_env *env, static bool can_elide_value_nullness(const struct bpf_map *map); +static int process_map_ptr_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, struct bpf_call_arg_meta *meta) +{ + /* Use map_uid (which is unique id of inner map) to reject: + * inner_map1 = bpf_map_lookup_elem(outer_map, key1) + * inner_map2 = bpf_map_lookup_elem(outer_map, key2) + * if (inner_map1 && inner_map2) { + * timer = bpf_map_lookup_elem(inner_map1); + * if (timer) + * // mismatch would have been allowed + * bpf_timer_init(timer, inner_map2); + * } + * + * Comparing map_ptr is enough to distinguish normal and outer maps. + */ + if (meta->map.ptr && + (meta->map.ptr != reg->map_ptr || meta->map.uid != reg->map_uid)) { + argno_t obj_argno = argno_from_reg(reg_from_argno(argno) - 1); + struct btf_record *rec = meta->map.ptr->record; + const char *obj_name = "workqueue"; + + if (rec->timer_off >= 0) + obj_name = "timer"; + else if (rec->task_work_off >= 0) + obj_name = "bpf_task_work"; + + verbose(env, "%s pointer in %s map_uid=%d ", + obj_name, reg_arg_name(env, obj_argno), meta->map.uid); + verbose(env, "doesn't match map pointer in %s map_uid=%d\n", + reg_arg_name(env, argno), reg->map_uid); + return -EINVAL; + } + + meta->map.ptr = reg->map_ptr; + meta->map.uid = reg->map_uid; + return 0; +} + static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta, - const struct bpf_func_proto *fn, int insn_idx) { + const struct bpf_func_proto *fn = meta->fn; u32 regno = BPF_REG_1 + arg; struct bpf_reg_state *reg = reg_state(env, regno); enum bpf_arg_type arg_type = fn->arg_type[arg]; - argno_t argno = argno_from_arg(arg + 1); + argno_t argno = argno_from_reg(regno); enum bpf_reg_type type = reg->type; u32 *arg_btf_id = NULL; u32 key_size; @@ -8239,7 +8706,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, } if (type_is_pkt_pointer(type) && - !may_access_direct_pkt_data(env, meta, BPF_READ)) { + !may_access_direct_pkt_data(env, fn, BPF_READ)) { verbose(env, "helper access to the packet is not allowed\n"); return -EACCES; } @@ -8261,11 +8728,12 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) arg_btf_id = fn->arg_btf_id[arg]; - err = check_reg_type(env, reg, argno_from_reg(regno), arg_type, arg_btf_id, meta); + err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta, + func_id_name(meta->func_id)); if (err) return err; - err = check_func_arg_reg_off(env, reg, argno_from_reg(regno), arg_type); + err = check_func_arg_reg_off(env, reg, argno, arg_type); if (err) return err; @@ -8274,6 +8742,10 @@ skip_type_check: !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", func_id_name(meta->func_id), reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, insn_idx, argno, func_id_name(meta->func_id), + "release helpers require a value that owns a live resource returned by a matching acquire helper", + "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); return -EINVAL; } @@ -8283,29 +8755,9 @@ skip_type_check: switch (base_type(arg_type)) { case ARG_CONST_MAP_PTR: /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ - if (meta->map.ptr) { - /* Use map_uid (which is unique id of inner map) to reject: - * inner_map1 = bpf_map_lookup_elem(outer_map, key1) - * inner_map2 = bpf_map_lookup_elem(outer_map, key2) - * if (inner_map1 && inner_map2) { - * timer = bpf_map_lookup_elem(inner_map1); - * if (timer) - * // mismatch would have been allowed - * bpf_timer_init(timer, inner_map2); - * } - * - * Comparing map_ptr is enough to distinguish normal and outer maps. - */ - if (meta->map.ptr != reg->map_ptr || - meta->map.uid != reg->map_uid) { - verbose(env, - "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - } - meta->map.ptr = reg->map_ptr; - meta->map.uid = reg->map_uid; + err = process_map_ptr_arg(env, reg, argno, meta); + if (err) + return err; break; case ARG_PTR_TO_MAP_KEY: /* bpf_map_xxx(..., map_ptr, ..., key) call: @@ -8322,7 +8774,8 @@ skip_type_check: return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, reg, argno_from_reg(regno), key_size, BPF_READ, false, NULL); + err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL, + NULL); if (err) return err; if (can_elide_value_nullness(meta->map.ptr)) { @@ -8348,10 +8801,18 @@ skip_type_check: verifier_bug(env, "invalid map_ptr to access map->value"); return -EFAULT; } - meta->raw_mode = arg_type & MEM_UNINIT; - err = check_helper_mem_access(env, reg, argno_from_reg(regno), meta->map.ptr->value_size, + + /* + * Disable raw mode for bpf_map_peek_elem() on a bloom filter. The helper reads + * the value buffer as an input rather than filling it. + */ + if (meta->func_id == BPF_FUNC_map_peek_elem && + meta->map.ptr->map_type == BPF_MAP_TYPE_BLOOM_FILTER) + meta->arg_raw_mem.regno = 0; + + err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + false, meta, NULL); break; case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { @@ -8367,11 +8828,11 @@ skip_type_check: return -EACCES; } if (meta->func_id == BPF_FUNC_spin_lock) { - err = process_spin_lock(env, reg, argno_from_reg(regno), PROCESS_SPIN_LOCK); + err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK); if (err) return err; } else if (meta->func_id == BPF_FUNC_spin_unlock) { - err = process_spin_lock(env, reg, argno_from_reg(regno), 0); + err = process_spin_lock(env, reg, argno, 0); if (err) return err; } else { @@ -8380,7 +8841,7 @@ skip_type_check: } break; case ARG_PTR_TO_TIMER: - err = process_timer_helper(env, reg, argno_from_reg(regno), meta); + err = process_timer_func(env, reg, argno, &meta->map); if (err) return err; break; @@ -8391,51 +8852,41 @@ skip_type_check: /* The access to this pointer is only checked when we hit the * next is_mem_size argument below. */ - meta->raw_mode = arg_type & MEM_UNINIT; if (arg_type & MEM_FIXED_SIZE) { - err = check_helper_mem_access(env, reg, argno_from_reg(regno), fn->arg_size[arg], - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], + arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL); if (err) return err; if (arg_type & MEM_ALIGNED) err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); } break; - case ARG_CONST_SIZE: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), - argno_from_reg(regno), - fn->arg_type[arg - 1] & MEM_WRITE ? - BPF_WRITE : BPF_READ, - false, meta); + case ARG_MEM_SIZE: + err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, + argno_from_reg(regno - 1), argno, + fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, + false, meta, NULL); break; - case ARG_CONST_SIZE_OR_ZERO: - err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), - argno_from_reg(regno), - fn->arg_type[arg - 1] & MEM_WRITE ? - BPF_WRITE : BPF_READ, - true, meta); + case ARG_MEM_SIZE_OR_ZERO: + err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, + argno_from_reg(regno - 1), argno, + fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, + true, meta, NULL); break; case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, &meta->ref_obj, - &meta->dynptr); + err = process_dynptr_func(env, reg, argno, insn_idx, func_id_name(meta->func_id), + arg_type, &meta->ref_obj, &meta->dynptr); if (err) return err; break; case ARG_CONST_ALLOC_SIZE_OR_ZERO: - if (!tnum_is_const(reg->var_off)) { - verbose(env, "R%d is not a known constant'\n", - regno); - return -EACCES; - } - meta->mem_size = reg->var_off.value; - err = mark_chain_precision(env, regno); + err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); if (err) return err; break; case ARG_PTR_TO_CONST_STR: { - err = check_arg_const_str(env, reg, argno_from_reg(regno)); + err = check_arg_const_str(env, reg, argno); if (err) return err; break; @@ -8472,12 +8923,7 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) if (func_id == BPF_FUNC_map_delete_elem) return true; break; - case BPF_PROG_TYPE_SOCKET_FILTER: - case BPF_PROG_TYPE_SCHED_CLS: - case BPF_PROG_TYPE_SCHED_ACT: - case BPF_PROG_TYPE_XDP: case BPF_PROG_TYPE_SK_REUSEPORT: - case BPF_PROG_TYPE_FLOW_DISSECTOR: case BPF_PROG_TYPE_SK_LOOKUP: return true; default: @@ -8747,26 +9193,21 @@ error: return -EINVAL; } -static bool check_raw_mode_ok(const struct bpf_func_proto *fn) +static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) { - int count = 0; + int i; - if (arg_type_is_raw_mem(fn->arg1_type)) - count++; - if (arg_type_is_raw_mem(fn->arg2_type)) - count++; - if (arg_type_is_raw_mem(fn->arg3_type)) - count++; - if (arg_type_is_raw_mem(fn->arg4_type)) - count++; - if (arg_type_is_raw_mem(fn->arg5_type)) - count++; + for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (fn->arg_type[i] == ARG_DONTCARE) + break; + if (!arg_type_is_raw_mem(fn->arg_type[i])) + continue; + if (meta->arg_raw_mem.regno) + return false; + meta->arg_raw_mem.regno = i + 1; + } - /* We only support one arg being in raw mode at the moment, - * which is sufficient for the helper functions we have - * right now. - */ - return count <= 1; + return true; } static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg) @@ -8807,6 +9248,8 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) int i; for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { + if (fn->arg_type[i] == ARG_DONTCARE) + break; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID) return !!fn->arg_btf_id[i]; if (base_type(fn->arg_type[i]) == ARG_PTR_TO_SPIN_LOCK) @@ -8828,6 +9271,8 @@ static bool check_mem_arg_rw_flag_ok(const struct bpf_func_proto *fn) for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; + if (arg_type == ARG_DONTCARE) + break; if (base_type(arg_type) != ARG_PTR_TO_MEM) continue; if (!(arg_type & (MEM_WRITE | MEM_RDONLY))) @@ -8844,6 +9289,8 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { enum bpf_arg_type arg_type = fn->arg_type[i]; + if (arg_type == ARG_DONTCARE) + break; if (arg_type_is_release(arg_type)) { if (meta->release_regno) return false; @@ -8856,7 +9303,7 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_ static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta) { - return check_raw_mode_ok(fn) && + return check_raw_mode_ok(fn, meta) && check_arg_pair_ok(fn) && check_mem_arg_rw_flag_ok(fn) && check_proto_release_reg(fn, meta) && @@ -8875,8 +9322,10 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env) struct bpf_reg_state *reg; bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ - if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) + if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) { + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_PKT_DATA_CHANGE); mark_reg_invalid(env, reg); + } })); } @@ -8906,7 +9355,7 @@ static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range reg->range = AT_PKT_END; } -static int release_reference_nomark(struct bpf_verifier_state *state, int id) +static int __release_reference_nomark(struct bpf_verifier_state *state, int id) { int i; @@ -8921,6 +9370,16 @@ static int release_reference_nomark(struct bpf_verifier_state *state, int id) return -EINVAL; } +static int release_reference_nomark(struct bpf_verifier_env *env, int id) +{ + int err; + + err = __release_reference_nomark(env->cur_state, id); + if (!err) + bpf_diag_record_ref_release(env, env->insn_idx, id); + return err; +} + static int idstack_push(struct bpf_idmap *idmap, u32 id) { int i; @@ -8963,8 +9422,10 @@ static int release_reference(struct bpf_verifier_env *env, int id) if (err) return err; - if (find_reference_state(vstate, id)) - WARN_ON_ONCE(release_reference_nomark(vstate, id)); + if (find_reference_state(vstate, id)) { + err = release_reference_nomark(env, id); + WARN_ON_ONCE(err); + } while ((id = idstack_pop(idstack))) { /* @@ -8992,10 +9453,25 @@ static int release_reference(struct bpf_verifier_env *env, int id) return err; } + /* + * A dynptr occupies two stack slots that invalidate_dynptr() + * clears together. Record both scrubs before invalidating it. + */ + if (stack && stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) { + struct bpf_stack_state *dyn_stack = stack; + + if (reg->dynptr.first_slot) + dyn_stack--; + bpf_diag_record_scrub(env, &dyn_stack[0].spilled_ptr, + BPF_DIAG_MOD_REF_RELEASE); + bpf_diag_record_scrub(env, &dyn_stack[1].spilled_ptr, + BPF_DIAG_MOD_REF_RELEASE); + invalidate_dynptr(env, dyn_stack); + continue; + } + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_REF_RELEASE); if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL) mark_reg_invalid(env, reg); - else if (stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) - invalidate_dynptr(env, stack); })); } @@ -9008,8 +9484,10 @@ static void invalidate_non_owning_refs(struct bpf_verifier_env *env) struct bpf_reg_state *reg; bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ - if (type_is_non_owning_ref(reg->type)) + if (type_is_non_owning_ref(reg->type)) { + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_NON_OWN_REF); mark_reg_invalid(env, reg); + } })); } @@ -9022,8 +9500,10 @@ static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env) bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ if (reg->type & MEM_RCU) { + bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); reg->type |= PTR_UNTRUSTED; + bpf_diag_mod_end(env); } })); } @@ -9034,15 +9514,19 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) struct bpf_reg_state *reg; int err; - err = release_reference_nomark(env->cur_state, id); + err = release_reference_nomark(env, id); + if (err) + return err; bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ if (reg->id != id) continue; if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { + bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); reg->id = 0; reg->type &= ~MEM_ALLOC; reg->type |= MEM_RCU; + bpf_diag_mod_end(env); } })); @@ -9054,6 +9538,8 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, { int i; + bpf_diag_record_caller_saved(env, regs); + /* after the call registers r0 - r5 were scratched */ for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); @@ -9061,13 +9547,15 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, } } -static void invalidate_outgoing_stack_args(const struct bpf_verifier_env *env, +static void invalidate_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *state) { int i, nslots = state->out_stack_arg_cnt; - for (i = 0; i < nslots; i++) + for (i = 0; i < nslots; i++) { + bpf_diag_record_scrub(env, &state->stack_arg_regs[i], BPF_DIAG_MOD_CALLER_SAVED); bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]); + } } typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, @@ -9135,20 +9623,28 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_func_state *caller = cur_func(env); struct bpf_verifier_log *log = &env->log; struct ref_obj_desc ref_obj = {}; + const struct btf_param *args; + const struct btf_type *func, *func_proto; u32 i; int ret, err; ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { - err = check_outgoing_stack_args(env, caller, sub->arg_cnt); + err = check_outgoing_stack_args(env, caller, sub->arg_cnt, + bpf_subprog_name(env, subprog), + NULL, NULL); if (err) return err; } return ret; } - ret = check_outgoing_stack_args(env, caller, sub->arg_cnt); + func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); + func_proto = btf_type_by_id(btf, func->type); + args = btf_params(func_proto); + ret = check_outgoing_stack_args(env, caller, sub->arg_cnt, + bpf_subprog_name(env, subprog), btf, args); if (ret) return ret; @@ -9187,7 +9683,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); if (ret < 0) return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size)) + if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, + NULL)) return -EINVAL; if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { @@ -9213,7 +9710,9 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (ret) return ret; - ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, &ref_obj, NULL); + ret = process_dynptr_func(env, reg, argno, env->insn_idx, + bpf_subprog_name(env, subprog), arg->arg_type, + &ref_obj, NULL); if (ret) return ret; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { @@ -9224,7 +9723,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, continue; memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ - err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta); + err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta, + bpf_subprog_name(env, subprog)); err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); if (err) return err; @@ -9365,17 +9865,25 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { - const char *sub_name = subprog_name(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); + const char *operation; + bool returns_void; if (env->cur_state->active_locks) { verbose(env, "global function calls are not allowed while holding a lock,\n" "use static function instead\n"); + operation = bpf_diag_fmt(env, "global function %s()", sub_name); + bpf_diag_ctx_active(env, *insn_idx, operation, BPF_DIAG_CONTEXT_LOCK, + "Release the lock before calling the global function, or use a static function instead."); return -EINVAL; } if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { verbose(env, "sleepable global function %s() called in %s\n", sub_name, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); + bpf_diag_ctx_forbidden(env, *insn_idx, operation, + "Move the call outside the critical section, or use a non-sleepable function."); return -EINVAL; } @@ -9388,17 +9896,22 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); + returns_void = subprog_returns_void(env, subprog); if (env->subprog_info[subprog].changes_pkt_data) clear_all_pkt_pointers(env); /* mark global subprog for verifying after main prog */ subprog_aux(env, subprog)->called = true; + if (returns_void) + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); + else + bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env)); /* All non-void global functions return a 64-bit SCALAR_VALUE. */ - if (!subprog_returns_void(env, subprog)) { + if (!returns_void) { mark_reg_unknown(env, caller->regs, BPF_REG_0); - caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; + bpf_diag_mod_end(env); } if (env->subprog_info[subprog].might_throw) { @@ -9433,6 +9946,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); clear_caller_saved_regs(env, caller->regs); /* and go analyze first insn of the callee */ @@ -9683,6 +10197,42 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, static bool is_rbtree_lock_required_kfunc(u32 btf_id); +static void account_processed_insn(struct bpf_verifier_env *env) +{ + struct bpf_func_state *frame = cur_func(env); + + env->insn_processed++; + frame->insns_subtotal++; + env->subprog_info[frame->subprogno].insns_self++; +} + +static void account_processed_insns(struct bpf_verifier_env *env, + struct bpf_func_state *callee, + struct bpf_func_state *caller) +{ + u32 insns; + + if (!callee) + return; + + insns = callee->insns_subtotal; + + env->subprog_info[callee->subprogno].insns_total += insns; + if (caller) + caller->insns_subtotal += insns; + callee->insns_subtotal = 0; +} + +static void account_current_path(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + int frame; + + for (frame = state->curframe; frame >= 0; frame--) + account_processed_insns(env, state->frame[frame], + frame ? state->frame[frame - 1] : NULL); +} + /* Are we currently verifying the callback for a rbtree helper that must * be called with lock held? If so, no need to complain about unreleased * lock @@ -9760,7 +10310,9 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) } } else { /* return to the caller whatever r0 had in the callee */ + bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); caller->regs[BPF_REG_0] = *r0; + bpf_diag_mod_end(env); } /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, @@ -9779,6 +10331,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) verbose(env, "to caller at %d:\n", *insn_idx); print_verifier_state(env, state, caller->frameno, true); } + account_processed_insns(env, callee, caller); /* clear everything in the callee. In case of exceptional exits using * bpf_throw, this will be done by copy_verifier_state for extra frames. */ free_func_state(callee); @@ -9965,6 +10518,7 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi continue; verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", state->refs[i].id, state->refs[i].insn_idx); + bpf_diag_leak(env, state->refs[i].id, state->refs[i].insn_idx, env->insn_idx); refs_lingering = true; } return refs_lingering ? -EINVAL : 0; @@ -9976,6 +10530,8 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit if (check_lock && env->cur_state->active_locks) { verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_LOCK, + "Release the BPF spin lock before this operation on every path."); return -EINVAL; } @@ -9987,16 +10543,23 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit if (check_lock && env->cur_state->active_irq_id) { verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_IRQ, + "Restore the saved IRQ state before this operation on every path."); return -EINVAL; } if (check_lock && env->cur_state->active_rcu_locks) { verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_RCU, + "Call bpf_rcu_read_unlock() before this operation on every path."); return -EINVAL; } if (check_lock && env->cur_state->active_preempt_locks) { verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix); + bpf_diag_ctx_active( + env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_PREEMPT, + "Call bpf_preempt_enable() before this operation on every path."); return -EINVAL; } @@ -10178,6 +10741,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn enum bpf_type_flag ret_flag; struct bpf_reg_state *regs; struct bpf_call_arg_meta meta; + const char *operation; int insn_idx = *insn_idx_p; bool changes_data; int i, err, func_id; @@ -10193,17 +10757,31 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) { verbose(env, "program of this type cannot use helper %s#%d\n", func_id_name(func_id), func_id); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, "this program type does not allow the helper", + "Use a helper allowed for this program type, or move the logic to a compatible program type."); return err; } /* eBPF programs must be GPL compatible to use GPL-ed functions */ if (!env->prog->gpl_compatible && fn->gpl_only) { verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, + "this helper is restricted to GPL-compatible programs", + "Use a GPL-compatible license, or replace the helper with one that is available to non-GPL programs."); return -EINVAL; } if (fn->allowed && !fn->allowed(env->prog)) { verbose(env, "helper call is not allowed in probe\n"); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, + "the helper-specific policy callback rejected this program", + "Use the helper only from an allowed attach point or program configuration."); return -EINVAL; } @@ -10215,7 +10793,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } memset(&meta, 0, sizeof(meta)); - meta.pkt_access = fn->pkt_access; err = check_func_proto(fn, &meta); if (err) { @@ -10226,6 +10803,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (fn->might_sleep && !in_sleepable_context(env)) { verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable helper %s#%d", + func_id_name(func_id), func_id); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Move the helper call outside the critical section, or use a non-sleepable helper."); return -EINVAL; } @@ -10234,9 +10815,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn env->insn_aux_data[insn_idx].non_sleepable = true; meta.func_id = func_id; + meta.fn = fn; /* check args */ for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { - err = check_func_arg(env, i, &meta, fn, insn_idx); + err = check_func_arg(env, i, &meta, insn_idx); if (err) return err; } @@ -10254,8 +10836,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* Mark slots with STACK_MISC in case of raw mode, stack offset * is inferred from register state. */ - for (i = 0; i < meta.access_size; i++) { - err = check_mem_access(env, insn_idx, regs + meta.regno, argno_from_reg(meta.regno), i, BPF_B, + for (i = 0; i < meta.arg_raw_mem.size; i++) { + err = check_mem_access(env, insn_idx, regs + meta.arg_raw_mem.regno, + argno_from_reg(meta.arg_raw_mem.regno), i, BPF_B, BPF_WRITE, -1, false, false); if (err) return err; @@ -10411,15 +10994,14 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return err; /* reset caller saved regs */ + bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); } invalidate_outgoing_stack_args(env, cur_func(env)); - /* helper call returns 64-bit value. */ - regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; - /* update return register (already marked as written above) */ ret_type = fn->ret_type; ret_flag = type_flag(ret_type); @@ -10453,10 +11035,12 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].map_ptr = meta.map.ptr; regs[BPF_REG_0].map_uid = meta.map.uid; regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag; - if (!type_may_be_null(ret_flag) && + if (type_may_be_null(ret_flag) || btf_record_has_field(meta.map.ptr->record, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK)) { regs[BPF_REG_0].id = ++env->id_gen; } + /* requires regs[BPF_REG_0].id to be set because of the map-in-map case */ + refine_map_lookup_value(®s[BPF_REG_0]); break; case RET_PTR_TO_SOCKET: mark_reg_known_zero(env, regs, BPF_REG_0); @@ -10473,7 +11057,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn case RET_PTR_TO_MEM: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; - regs[BPF_REG_0].mem_size = meta.mem_size; + regs[BPF_REG_0].mem_size = meta.ret_mem.size; break; case RET_PTR_TO_MEM_OR_BTF_ID: { @@ -10554,7 +11138,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } - if (type_may_be_null(regs[BPF_REG_0].type)) + if (type_may_be_null(regs[BPF_REG_0].type) && !regs[BPF_REG_0].id) regs[BPF_REG_0].id = ++env->id_gen; if (is_ptr_cast_function(func_id) && @@ -10566,6 +11150,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) return err; + bpf_diag_mod_end(env); + /* * In order for a release of any of the original or cast pointers * to invalidate all other pointers, reuse the same reference id for @@ -10582,6 +11168,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn __mark_reg_known_zero(r0); r0->type = SCALAR_VALUE; + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL; regs[BPF_REG_0].id = meta.ref_obj.id; } else if (is_acquire_function(func_id, meta.map.ptr)) { @@ -10600,6 +11187,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) return err; + bpf_diag_mod_end(env); + err = check_map_func_compatibility(env, meta.map.ptr, func_id); if (err) return err; @@ -10657,76 +11246,50 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return 0; } -/* mark_btf_func_reg_size() is used when the reg size is determined by - * the BTF func_proto's return value size and argument. - */ -static void __mark_btf_func_reg_size(struct bpf_verifier_env *env, struct bpf_reg_state *regs, - u32 regno, size_t reg_size) -{ - struct bpf_reg_state *reg = ®s[regno]; - - if (regno == BPF_REG_0) { - /* Function return value */ - reg->subreg_def = reg_size == sizeof(u64) ? - DEF_NOT_SUBREG : env->insn_idx + 1; - } else if (reg_size == sizeof(u64)) { - /* Function argument */ - mark_insn_zext(env, reg); - } -} - -static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, - size_t reg_size) -{ - return __mark_btf_func_reg_size(env, cur_regs(env), regno, reg_size); -} - -static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; } -static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_release(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RELEASE; } -static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_DESTRUCTIVE; } -static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RCU; } -static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RCU_PROTECTED; } static bool is_kfunc_arg_mem_size(const struct btf *btf, - const struct btf_param *arg, - const struct bpf_reg_state *reg) + const struct btf_param *arg) { const struct btf_type *t; t = btf_type_skip_modifiers(btf, arg->type, NULL); - if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + if (!btf_type_is_scalar(t)) return false; return btf_param_match_suffix(btf, arg, "__sz"); } static bool is_kfunc_arg_const_mem_size(const struct btf *btf, - const struct btf_param *arg, - const struct bpf_reg_state *reg) + const struct btf_param *arg) { const struct btf_type *t; t = btf_type_skip_modifiers(btf, arg->type, NULL); - if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + if (!btf_type_is_scalar(t)) return false; return btf_param_match_suffix(btf, arg, "__szk"); @@ -10747,6 +11310,11 @@ static bool is_kfunc_arg_map(const struct btf *btf, const struct btf_param *arg) return btf_param_match_suffix(btf, arg, "__map"); } +static bool is_kfunc_arg_const_map(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__const_map"); +} + static bool is_kfunc_arg_alloc_obj(const struct btf *btf, const struct btf_param *arg) { return btf_param_match_suffix(btf, arg, "__alloc"); @@ -10764,7 +11332,8 @@ static bool is_kfunc_arg_refcounted_kptr(const struct btf *btf, const struct btf static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param *arg) { - return btf_param_match_suffix(btf, arg, "__nullable"); + return btf_param_match_suffix(btf, arg, "__nullable") || + btf_param_match_suffix(btf, arg, "__arena"); } static bool is_kfunc_arg_nonown_allowed(const struct btf *btf, const struct btf_param *arg) @@ -10782,6 +11351,12 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param return btf_param_match_suffix(btf, arg, "__irq_flag"); } +static bool is_kfunc_arg_arena(const struct btf *btf, const struct btf_param *arg) +{ + return btf_param_match_suffix(btf, arg, "__arena__nullable") || + btf_param_match_suffix(btf, arg, "__arena"); +} + static bool is_kfunc_arg_scalar_with_name(const struct btf *btf, const struct btf_param *arg, const char *name) @@ -10922,7 +11497,7 @@ static bool is_kfunc_arg_prog_aux(const struct btf *btf, const struct btf_param * To determine whether an argument is implicit, we compare its position * against the number of arguments in the prototype w/o implicit args. */ -static bool is_kfunc_arg_implicit(const struct bpf_kfunc_call_arg_meta *meta, u32 arg_idx) +static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_idx) { const struct btf_type *func, *func_proto; u32 argn; @@ -10978,6 +11553,11 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, } enum kfunc_ptr_arg_type { + KF_ARG_CONST_MEM_SIZE, + KF_ARG_MEM_SIZE, + KF_ARG_CONST, + KF_ARG_CONST_ALLOC_SIZE_OR_ZERO, + KF_ARG_ANYTHING, KF_ARG_PTR_TO_CTX, KF_ARG_PTR_TO_ALLOC_BTF_ID, /* Allocated object */ KF_ARG_PTR_TO_REFCOUNTED_KPTR, /* Refcounted local kptr */ @@ -10987,18 +11567,17 @@ enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_LIST_NODE, KF_ARG_PTR_TO_BTF_ID, /* Also covers reg2btf_ids conversions */ KF_ARG_PTR_TO_MEM, - KF_ARG_PTR_TO_MEM_SIZE, /* Size derived from next argument, skip it */ KF_ARG_PTR_TO_CALLBACK, KF_ARG_PTR_TO_RB_ROOT, KF_ARG_PTR_TO_RB_NODE, - KF_ARG_PTR_TO_NULL, KF_ARG_PTR_TO_CONST_STR, - KF_ARG_PTR_TO_MAP, + KF_ARG_CONST_MAP_PTR, KF_ARG_PTR_TO_TIMER, KF_ARG_PTR_TO_WORKQUEUE, KF_ARG_PTR_TO_IRQ_FLAG, KF_ARG_PTR_TO_RES_SPIN_LOCK, KF_ARG_PTR_TO_TASK_WORK, + KF_ARG_PTR_TO_ARENA, }; enum special_kfunc_type { @@ -11068,10 +11647,7 @@ enum special_kfunc_type { KF_bpf_task_work_schedule_resume, KF_bpf_arena_alloc_pages, KF_bpf_arena_free_pages, - KF_bpf_arena_reserve_pages, KF_bpf_session_is_return, - KF_bpf_stream_vprintk, - KF_bpf_stream_print_stack, }; BTF_ID_LIST(special_kfunc_list) @@ -11161,14 +11737,11 @@ BTF_ID(func, bpf_task_work_schedule_signal) BTF_ID(func, bpf_task_work_schedule_resume) BTF_ID(func, bpf_arena_alloc_pages) BTF_ID(func, bpf_arena_free_pages) -BTF_ID(func, bpf_arena_reserve_pages) #ifdef CONFIG_BPF_EVENTS BTF_ID(func, bpf_session_is_return) #else BTF_ID_UNUSED #endif -BTF_ID(func, bpf_stream_vprintk) -BTF_ID(func, bpf_stream_print_stack) static bool is_bpf_obj_new_kfunc(u32 func_id) { @@ -11221,7 +11794,7 @@ static bool is_task_work_add_kfunc(u32 func_id) func_id == special_kfunc_list[KF_bpf_task_work_schedule_resume]; } -static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta) { if (is_bpf_refcount_acquire_kfunc(meta->func_id) && meta->arg_owning_ref) return false; @@ -11229,140 +11802,208 @@ static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta) return meta->kfunc_flags & KF_RET_NULL; } -static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock]; } -static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock]; } -static bool is_kfunc_bpf_preempt_disable(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable]; } -static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta) +static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable]; } -bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta) +bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta) { return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; } -static enum kfunc_ptr_arg_type -get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller, - struct bpf_reg_state *regs, struct bpf_kfunc_call_arg_meta *meta, - const struct btf_type *t, const struct btf_type *ref_t, - const char *ref_tname, const struct btf_param *args, - int arg, int nargs, argno_t argno, struct bpf_reg_state *reg) +static int +get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + const struct btf_param *args, int arg, int nargs) { - bool arg_mem_size = false; + const struct btf_type *t, *ref_t = NULL; + argno_t argno = argno_from_arg(arg + 1); + const char *ref_tname = NULL; + int arg_type; - if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || - meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || - meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) - return KF_ARG_PTR_TO_CTX; + t = btf_type_skip_modifiers(meta->btf, args[arg].type, NULL); - if (arg + 1 < nargs && - (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)) || - is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)))) - arg_mem_size = true; + /* Scalar arguments are classified from their BTF suffix/name alone. */ + if (btf_type_is_scalar(t)) { + if (is_kfunc_arg_constant(meta->btf, &args[arg])) + return KF_ARG_CONST; + if (is_kfunc_arg_const_mem_size(meta->btf, &args[arg])) + return KF_ARG_CONST_MEM_SIZE; + if (is_kfunc_arg_mem_size(meta->btf, &args[arg])) + return KF_ARG_MEM_SIZE; + if (is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdonly_buf_size") || + is_kfunc_arg_scalar_with_name(meta->btf, &args[arg], "rdwr_buf_size")) + return KF_ARG_CONST_ALLOC_SIZE_OR_ZERO; + return KF_ARG_ANYTHING; + } + + if (!btf_type_is_ptr(t)) { + verbose(env, "Unrecognized %s type %s\n", + reg_arg_name(env, argno), btf_type_str(t)); + return -EINVAL; + } + ref_t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + ref_tname = btf_name_by_offset(meta->btf, ref_t->name_off); /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register * type to our caller. When a set of conditions hold in the BTF type of * arguments, we resolve it to a known kfunc_ptr_arg_type. */ - if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) - return KF_ARG_PTR_TO_CTX; - - if (is_kfunc_arg_nullable(meta->btf, &args[arg]) && bpf_register_is_null(reg) && - !arg_mem_size) - return KF_ARG_PTR_TO_NULL; - - if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_ALLOC_BTF_ID; - - if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_REFCOUNTED_KPTR; - - if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_DYNPTR; - - if (is_kfunc_arg_iter(meta, arg, &args[arg])) - return KF_ARG_PTR_TO_ITER; - - if (is_kfunc_arg_list_head(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_LIST_HEAD; - - if (is_kfunc_arg_list_node(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_LIST_NODE; - - if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RB_ROOT; - - if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RB_NODE; - - if (is_kfunc_arg_const_str(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_CONST_STR; + if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || + meta->func_id == special_kfunc_list[KF_bpf_session_is_return] || + meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) + arg_type = KF_ARG_PTR_TO_CTX; + else if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg)) + arg_type = KF_ARG_PTR_TO_CTX; + else if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_ALLOC_BTF_ID; + else if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_REFCOUNTED_KPTR; + else if (is_kfunc_arg_dynptr(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_DYNPTR; + else if (is_kfunc_arg_iter(meta, arg, &args[arg])) + arg_type = KF_ARG_PTR_TO_ITER; + else if (is_kfunc_arg_list_head(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_LIST_HEAD; + else if (is_kfunc_arg_list_node(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_LIST_NODE; + else if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RB_ROOT; + else if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RB_NODE; + else if (is_kfunc_arg_const_str(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_CONST_STR; + else if (is_kfunc_arg_const_map(meta->btf, &args[arg])) + arg_type = KF_ARG_CONST_MAP_PTR; + else if (is_kfunc_arg_map(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_BTF_ID; + else if (is_kfunc_arg_wq(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_WORKQUEUE; + else if (is_kfunc_arg_timer(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_TIMER; + else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_TASK_WORK; + else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_IRQ_FLAG; + else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_RES_SPIN_LOCK; + else if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_CALLBACK; + else if (is_kfunc_arg_arena(meta->btf, &args[arg])) { + if (!bpf_jit_supports_arena_args()) { + verbose(env, "JIT does not support kfunc %s() with arena pointer arguments\n", + meta->func_name); + return -ENOTSUPP; + } + if (!env->prog->aux->arena) { + verbose(env, + "%s arena pointer requires a program with an associated arena\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + if (reg_from_argno(argno) < 0) { + verbose(env, "%s arena pointer cannot be a stack argument\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + /* + * Both suffixes accept a constant zero. The function model determines + * whether the JIT rebases it to the arena base or preserves NULL. + * The common nullable path below records that verifier property. + */ + arg_type = KF_ARG_PTR_TO_ARENA; + } else if (arg + 1 < nargs && + (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || + is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { + if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); + return -EINVAL; + } + arg_type = KF_ARG_PTR_TO_MEM; + } else if (btf_type_is_struct(ref_t)) + /* A pointer to a struct without a size argument is classified as KF_ARG_PTR_TO_BTF_ID */ + arg_type = KF_ARG_PTR_TO_BTF_ID; + else { + /* + * Otherwise this is a fixed-size memory buffer supported by + * check_helper_mem_access(): a pointer to a scalar or a struct of + * scalars. The access size is derived from the pointed-to BTF type. + */ + if (!btf_type_is_scalar(ref_t) && + !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", + reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); + return -EINVAL; + } + arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + } - if (is_kfunc_arg_map(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_MAP; + if (is_kfunc_arg_nullable(meta->btf, &args[arg])) + arg_type |= PTR_MAYBE_NULL; - if (is_kfunc_arg_wq(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_WORKQUEUE; + return arg_type; +} - if (is_kfunc_arg_timer(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_TIMER; +static int gen_kfunc_arg_proto(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, + struct bpf_func_proto *proto) +{ + const struct btf *btf = meta->btf; + const struct btf_param *args; + u32 i, nargs; + int arg_type; - if (is_kfunc_arg_task_work(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_TASK_WORK; + args = (const struct btf_param *)(meta->func_proto + 1); + nargs = btf_type_vlen(meta->func_proto); + if (nargs > MAX_BPF_FUNC_ARGS) { + verbose(env, "Function %s has %d > %d args\n", meta->func_name, + nargs, MAX_BPF_FUNC_ARGS); + return -EINVAL; + } + if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { + verbose(env, "JIT does not support kfunc %s() with %d args\n", + meta->func_name, nargs); + return -ENOTSUPP; + } - if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_IRQ_FLAG; + for (i = 0; i < nargs; i++) { + if (is_kfunc_arg_prog_aux(btf, &args[i]) || + is_kfunc_arg_ignore(btf, &args[i]) || + is_kfunc_arg_implicit(meta, i)) + continue; - if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) - return KF_ARG_PTR_TO_RES_SPIN_LOCK; + arg_type = get_kfunc_arg_type(env, meta, args, i, nargs); + if (arg_type < 0) + return arg_type; - if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) { - if (!btf_type_is_struct(ref_t)) { - verbose(env, "kernel function %s %s pointer type %s %s is not supported\n", - meta->func_name, reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname); - return -EINVAL; - } - return KF_ARG_PTR_TO_BTF_ID; + proto->arg_type[i] = arg_type; } - if (is_kfunc_arg_callback(env, meta->btf, &args[arg])) - return KF_ARG_PTR_TO_CALLBACK; - - /* This is the catch all argument type of register types supported by - * check_helper_mem_access. However, we only allow when argument type is - * pointer to scalar, or struct composed (recursively) of scalars. When - * arg_mem_size is true, the pointer can be void *. - */ - if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && - (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { - verbose(env, "%s pointer type %s %s must point to %sscalar, or struct with scalar\n", - reg_arg_name(env, argno), - btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : ""); - return -EINVAL; - } - return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM; + return 0; } static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, const struct btf_type *ref_t, const char *ref_tname, u32 ref_id, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, int arg, argno_t argno) { const struct btf_type *reg_ref_t; @@ -11414,7 +12055,8 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, ®_ref_id); reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off); struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value, - meta->btf, ref_id, strict_type_match); + meta->btf, ref_id, strict_type_match, + !type_is_alloc(reg->type)); /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow * caller to pass in the underlying type. @@ -11431,7 +12073,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, } static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { int err, spi, kfunc_class = IRQ_NATIVE_KFUNC; bool irq_save; @@ -11455,6 +12097,12 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * if (!is_irq_flag_reg_valid_uninit(env, reg)) { verbose(env, "expected uninitialized irq flag as %s\n", reg_arg_name(env, argno)); + bpf_diag_res(env, env->insn_idx, "IRQ flag is already initialized", + "Saving IRQ state requires an uninitialized stack slot for " + "the IRQ flag, but this slot already contains tracked IRQ " + "flag state.", + "Use a fresh stack slot for this save operation, or restore " + "the existing IRQ flag before reusing the slot."); return -EINVAL; } @@ -11471,6 +12119,11 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * if (err) { verbose(env, "expected an initialized irq flag as %s\n", reg_arg_name(env, argno)); + bpf_diag_res(env, env->insn_idx, "uninitialized IRQ flag restore", + "Restoring IRQ state requires a stack slot that was " + "initialized by a matching IRQ save operation on this path.", + "Pass the same stack slot that was previously initialized by " + "the matching IRQ save kfunc."); return err; } @@ -11483,11 +12136,13 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * err = unmark_stack_slot_irq_flag(env, reg, kfunc_class); if (err) return err; + + if (!in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); } return 0; } - static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { struct btf_record *rec = reg_btf_record(reg); @@ -11513,8 +12168,10 @@ static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id) { struct bpf_func_state *unused; struct bpf_reg_state *reg; + int err; - WARN_ON_ONCE(release_reference_nomark(env->cur_state, id)); + err = release_reference_nomark(env, id); + WARN_ON_ONCE(err); bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ if (reg->id == id) { @@ -11622,20 +12279,6 @@ static bool is_bpf_rbtree_api_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_rbtree_right]; } -static bool is_bpf_iter_num_api_kfunc(u32 btf_id) -{ - return btf_id == special_kfunc_list[KF_bpf_iter_num_new] || - btf_id == special_kfunc_list[KF_bpf_iter_num_next] || - btf_id == special_kfunc_list[KF_bpf_iter_num_destroy]; -} - -static bool is_bpf_graph_api_kfunc(u32 btf_id) -{ - return is_bpf_list_api_kfunc(btf_id) || - is_bpf_rbtree_api_kfunc(btf_id) || - is_bpf_refcount_acquire_kfunc(btf_id); -} - static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) { return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || @@ -11644,24 +12287,16 @@ static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]; } -static bool is_bpf_arena_kfunc(u32 btf_id) +static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset) { - return btf_id == special_kfunc_list[KF_bpf_arena_alloc_pages] || - btf_id == special_kfunc_list[KF_bpf_arena_free_pages] || - btf_id == special_kfunc_list[KF_bpf_arena_reserve_pages]; -} + struct bpf_kfunc_meta kfunc; + int err; -static bool is_bpf_stream_kfunc(u32 btf_id) -{ - return btf_id == special_kfunc_list[KF_bpf_stream_vprintk] || - btf_id == special_kfunc_list[KF_bpf_stream_print_stack]; -} + err = fetch_kfunc_meta(env, func_id, offset, &kfunc); + if (err || !kfunc.flags) + return false; -static bool kfunc_spin_allowed(u32 btf_id) -{ - return is_bpf_graph_api_kfunc(btf_id) || is_bpf_iter_num_api_kfunc(btf_id) || - is_bpf_res_spin_lock_kfunc(btf_id) || is_bpf_arena_kfunc(btf_id) || - is_bpf_stream_kfunc(btf_id); + return *kfunc.flags & KF_SPINLOCK_SAFE; } static bool is_sync_callback_calling_kfunc(u32 btf_id) @@ -11756,7 +12391,7 @@ static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, static int __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, enum btf_field_type head_field_type, struct btf_field **head_field) { @@ -11806,7 +12441,7 @@ __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_LIST_HEAD, &meta->arg_list_head.field); @@ -11814,7 +12449,7 @@ static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_RB_ROOT, &meta->arg_rbtree_root.field); @@ -11823,7 +12458,7 @@ static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, static int __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta, + struct bpf_call_arg_meta *meta, enum btf_field_type head_field_type, enum btf_field_type node_field_type, struct btf_field **node_field) @@ -11861,7 +12496,8 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id); t = btf_type_by_id(reg->btf, reg->btf_id); if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, - field->graph_root.value_btf_id, true)) { + field->graph_root.value_btf_id, true, + !type_is_alloc(reg->type))) { verbose(env, "operation on %s expects arg#1 %s at offset=%d " "in struct %s, but arg is at offset=%d in struct %s\n", btf_field_type_name(head_field_type), @@ -11887,7 +12523,7 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, BPF_LIST_HEAD, BPF_LIST_NODE, @@ -11896,7 +12532,7 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta, BPF_RB_ROOT, BPF_RB_NODE, @@ -11925,7 +12561,7 @@ static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) } } -static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, +static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int insn_idx) { const char *func_name = meta->func_name, *ref_tname; @@ -11939,18 +12575,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ args = (const struct btf_param *)(meta->func_proto + 1); nargs = btf_type_vlen(meta->func_proto); - if (nargs > MAX_BPF_FUNC_ARGS) { - verbose(env, "Function %s has %d > %d args\n", func_name, nargs, - MAX_BPF_FUNC_ARGS); - return -EINVAL; - } - if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { - verbose(env, "JIT does not support kfunc %s() with %d args\n", - func_name, nargs); - return -ENOTSUPP; - } - ret = check_outgoing_stack_args(env, caller, nargs); + ret = check_outgoing_stack_args(env, caller, nargs, func_name, btf, args); if (ret) return ret; @@ -11964,9 +12590,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); bool btf_id_fixed_off_ok = true; - u32 ref_id, type_size; - bool is_ret_buf_sz = false; - int kf_arg_type; + u32 ref_id = args[i].type, type_size; + int kf_arg_type = meta->fn->arg_type[i]; if (is_kfunc_arg_prog_aux(btf, &args[i])) { /* Reject repeated use bpf_prog_aux */ @@ -11989,153 +12614,67 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ t = btf_type_skip_modifiers(btf, args[i].type, NULL); - if (btf_type_is_scalar(t)) { - if (reg->type != SCALAR_VALUE) { - verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); - return -EINVAL; - } - - if (is_kfunc_arg_constant(meta->btf, &args[i])) { - if (meta->arg_constant.found) { - verifier_bug(env, "only one constant argument permitted"); - return -EFAULT; - } - if (!tnum_is_const(reg->var_off)) { - verbose(env, "%s must be a known constant\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (regno >= 0) - ret = mark_chain_precision(env, regno); - else - ret = mark_stack_arg_precision(env, i); - if (ret < 0) - return ret; - meta->arg_constant.found = true; - meta->arg_constant.value = reg->var_off.value; - } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { - meta->r0_rdonly = true; - is_ret_buf_sz = true; - } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdwr_buf_size")) { - is_ret_buf_sz = true; - } - - if (is_ret_buf_sz) { - if (meta->r0_size) { - verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); - return -EINVAL; - } - - if (!tnum_is_const(reg->var_off)) { - verbose(env, "%s is not a const\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - - meta->r0_size = reg->var_off.value; - if (regno >= 0) - ret = mark_chain_precision(env, regno); - else - ret = mark_stack_arg_precision(env, i); - if (ret) - return ret; - } - continue; + if (btf_type_is_ptr(t)) { + ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); } - if (!btf_type_is_ptr(t)) { - verbose(env, "Unrecognized %s type %s\n", - reg_arg_name(env, argno), btf_type_str(t)); - return -EINVAL; - } + if (btf_type_is_ptr(t) && + (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && + !type_may_be_null(kf_arg_type)) { + const char *expected_type; - if ((bpf_register_is_null(reg) || type_may_be_null(reg->type)) && - !is_kfunc_arg_nullable(meta->btf, &args[i])) { + expected_type = bpf_diag_fmt_btf_type(env, btf, args[i].type); verbose(env, "Possibly NULL pointer passed to trusted %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Add a NULL check and call the kfunc only on the non-NULL path.", + "the pointer may be NULL, but this kfunc requires a non-NULL value of type %s", + expected_type); return -EACCES; } if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", func_name, reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the resource-owning pointer returned by the matching acquire kfunc, and avoid calling the release kfunc after ownership has already been transferred or released.", + "release kfuncs require a resource-owning value of type %s returned by a matching acquire kfunc", + expected_type); return -EINVAL; } if (reg_is_referenced(env, reg)) update_ref_obj(&meta->ref_obj, reg); - ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); + if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) + continue; - kf_arg_type = get_kfunc_ptr_arg_type(env, caller, regs, meta, t, ref_t, ref_tname, - args, i, nargs, argno, reg); - if (kf_arg_type < 0) - return kf_arg_type; + if (is_kfunc_arg_map(btf, &args[i])) { + ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; + ref_t = btf_type_by_id(btf_vmlinux, ref_id); + ref_tname = btf_name_by_offset(btf, ref_t->name_off); + } - switch (kf_arg_type) { - case KF_ARG_PTR_TO_NULL: - continue; - case KF_ARG_PTR_TO_MAP: - if (!reg->map_ptr) { - verbose(env, "pointer in %s isn't map pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (meta->map.ptr && (reg->map_ptr->record->wq_off >= 0 || - reg->map_ptr->record->task_work_off >= 0)) { - /* Use map_uid (which is unique id of inner map) to reject: - * inner_map1 = bpf_map_lookup_elem(outer_map, key1) - * inner_map2 = bpf_map_lookup_elem(outer_map, key2) - * if (inner_map1 && inner_map2) { - * wq = bpf_map_lookup_elem(inner_map1); - * if (wq) - * // mismatch would have been allowed - * bpf_wq_init(wq, inner_map2); - * } - * - * Comparing map_ptr is enough to distinguish normal and outer maps. - */ - if (meta->map.ptr != reg->map_ptr || - meta->map.uid != reg->map_uid) { - if (reg->map_ptr->record->task_work_off >= 0) { - verbose(env, - "bpf_task_work pointer in R2 map_uid=%d doesn't match map pointer in R3 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - verbose(env, - "workqueue pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n", - meta->map.uid, reg->map_uid); - return -EINVAL; - } - } - meta->map.ptr = reg->map_ptr; - meta->map.uid = reg->map_uid; - fallthrough; + switch (base_type(kf_arg_type)) { + case KF_ARG_CONST: + case KF_ARG_CONST_MEM_SIZE: + case KF_ARG_MEM_SIZE: + case KF_ARG_ANYTHING: + case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: case KF_ARG_PTR_TO_ALLOC_BTF_ID: case KF_ARG_PTR_TO_BTF_ID: - if (!is_trusted_reg(env, reg)) { - if (!is_kfunc_rcu(meta)) { - verbose(env, "%s must be referenced or trusted\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - if (!is_rcu_reg(reg)) { - verbose(env, "%s must be a rcu pointer\n", - reg_arg_name(env, argno)); - return -EINVAL; - } - } - fallthrough; + case KF_ARG_CONST_MAP_PTR: case KF_ARG_PTR_TO_ITER: case KF_ARG_PTR_TO_LIST_HEAD: case KF_ARG_PTR_TO_LIST_NODE: case KF_ARG_PTR_TO_RB_ROOT: case KF_ARG_PTR_TO_RB_NODE: case KF_ARG_PTR_TO_MEM: - case KF_ARG_PTR_TO_MEM_SIZE: case KF_ARG_PTR_TO_CALLBACK: case KF_ARG_PTR_TO_CONST_STR: case KF_ARG_PTR_TO_WORKQUEUE: @@ -12143,6 +12682,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ case KF_ARG_PTR_TO_TASK_WORK: case KF_ARG_PTR_TO_IRQ_FLAG: case KF_ARG_PTR_TO_RES_SPIN_LOCK: + case KF_ARG_PTR_TO_ARENA: break; case KF_ARG_PTR_TO_DYNPTR: arg_type = ARG_PTR_TO_DYNPTR; @@ -12166,11 +12706,71 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ if (ret < 0) return ret; - switch (kf_arg_type) { + switch (base_type(kf_arg_type)) { + case KF_ARG_CONST: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + + ret = process_const_arg(env, reg, argno, meta); + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the kfunc requires this scalar argument to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return ret; + } + break; + case KF_ARG_ANYTHING: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + break; + case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + + if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) + meta->r0_rdonly = true; + ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a verifier-known constant size for this kfunc buffer argument.", + "the kfunc uses this argument as a return-buffer size, but %s is invalid or variable on this path", + reg_arg_name(env, argno)); + return ret; + } + break; case KF_ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { verbose(env, "%s expected pointer to ctx, but got %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the original program context pointer or preserve it before modifying registers.", + "the kfunc expects a context pointer, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -12181,6 +12781,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ meta->ret_btf_id = ret; } break; + case KF_ARG_PTR_TO_ARENA: + if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a pointer to arena or scalar\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + break; case KF_ARG_PTR_TO_ALLOC_BTF_ID: if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) { if (!is_bpf_obj_drop_kfunc(meta->func_id)) { @@ -12197,10 +12804,19 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } else { verbose(env, "%s expected pointer to allocated object\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer returned by the matching BPF object allocation path.", + "the kfunc expects an allocated object pointer, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the owned object pointer before it is released or transferred.", + "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", + reg_arg_name(env, argno)); return -EINVAL; } if (meta->btf == btf_vmlinux) { @@ -12237,8 +12853,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); } - ret = process_dynptr_func(env, reg, argno, insn_idx, dynptr_arg_type, - &meta->ref_obj, &meta->dynptr); + ret = process_dynptr_func(env, reg, argno, insn_idx, func_name, + dynptr_arg_type, &meta->ref_obj, &meta->dynptr); if (ret < 0) return ret; break; @@ -12335,75 +12951,181 @@ check_ok: if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_MAP: - /* If argument has '__map' suffix expect 'struct bpf_map *' */ - ref_id = *reg2btf_ids[CONST_PTR_TO_MAP]; - ref_t = btf_type_by_id(btf_vmlinux, ref_id); - ref_tname = btf_name_by_offset(btf, ref_t->name_off); - fallthrough; - case KF_ARG_PTR_TO_BTF_ID: - /* Only base_type is checked, further checks are done here */ - if ((base_type(reg->type) != PTR_TO_BTF_ID || - (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) && - !reg2btf_ids[base_type(reg->type)]) { - verbose(env, "%s is %s ", reg_arg_name(env, argno), - reg_type_str(env, reg->type)); - verbose(env, "expected %s or socket\n", - reg_type_str(env, base_type(reg->type) | - (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS))); + case KF_ARG_CONST_MAP_PTR: + if (base_type(reg->type) != CONST_PTR_TO_MAP || + type_may_be_null(reg->type)) { + verbose(env, "pointer in %s isn't map pointer\n", + reg_arg_name(env, argno)); return -EINVAL; } - ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); + ret = process_map_ptr_arg(env, reg, argno, meta); if (ret < 0) return ret; break; - case KF_ARG_PTR_TO_MEM: - resolve_ret = btf_resolve_size(btf, ref_t, &type_size); - if (IS_ERR(resolve_ret)) { - verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", - reg_arg_name(env, argno), btf_type_str(ref_t), - ref_tname, PTR_ERR(resolve_ret)); + case KF_ARG_PTR_TO_BTF_ID: + /* Only base_type is checked, further checks are done here */ + if (base_type(reg->type) == PTR_TO_BTF_ID || + reg2btf_ids[base_type(reg->type)]) { + if (!is_trusted_reg(env, reg) || + bpf_type_has_unsafe_modifiers(reg->type)) { + if (!is_kfunc_rcu(meta)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + verbose(env, "%s must be referenced or trusted\n", + reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.", + "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s", + expected_type, + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + if (!is_rcu_reg(reg)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + verbose(env, "%s must be a rcu pointer\n", + reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Use this kfunc with a pointer that is valid in an RCU read lock region.", + "the kfunc requires an RCU-protected pointer to %s, but %s is %s", + expected_type, + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; + } + } + + ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno); + if (ret < 0) + return ret; + break; + } + + if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); + const char *expected_type; + + verbose(env, "%s is %s expected %s %s", + reg_arg_name(env, argno), reg_type_str(env, reg->type), + btf_type_str(ref_t), ref_tname); + if (reg2btf_type != NOT_INIT) + verbose(env, " or %s", reg_type_str(env, reg2btf_type)); + verbose(env, "\n"); + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.", + "the kfunc expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer", + expected_type, + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } - ret = check_mem_reg(env, reg, argno, type_size); - if (ret < 0) - return ret; - break; - case KF_ARG_PTR_TO_MEM_SIZE: - { - struct bpf_reg_state *buff_reg = reg; - const struct btf_param *buff_arg = &args[i]; - struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1); - const struct btf_param *size_arg = &args[i + 1]; - argno_t next_argno = argno_from_arg(i + 2); - - if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { - ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg, - argno, next_argno); + + /* + * If the register does not contain btf id but the argument type is a pointer to + * scalar-only struct, allow verifying it as a fixed size memory. + */ + kf_arg_type = KF_ARG_PTR_TO_MEM | MEM_FIXED_SIZE; + fallthrough; + case KF_ARG_PTR_TO_MEM: + if (kf_arg_type & MEM_FIXED_SIZE) { + bool known_memory; + + resolve_ret = btf_resolve_size(btf, ref_t, &type_size); + if (IS_ERR(resolve_ret)) { + verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", + reg_arg_name(env, argno), btf_type_str(ref_t), + ref_tname, PTR_ERR(resolve_ret)); + return -EINVAL; + } + ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, + meta, &known_memory); if (ret < 0) { - verbose(env, "%s and ", reg_arg_name(env, argno)); - verbose(env, "%s memory, len pair leads to invalid memory access\n", - reg_arg_name(env, next_argno)); + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + if (known_memory) + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Pass memory with at least the required number of accessible bytes and suitable read and write access.", + "the kfunc expects %u bytes of memory for %s, but the verifier cannot prove that %s provides a readable and writable range of that size", + type_size, expected_type, + bpf_diag_reg_type_plain(env, reg->type)); + else + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", + "the kfunc expects %u bytes of memory for %s, but it is %s and not verifier-known memory", + type_size, expected_type, + bpf_diag_reg_type_plain(env, reg->type)); return ret; } } + break; + case KF_ARG_CONST_MEM_SIZE: + ret = process_const_arg(env, reg, argno, meta); + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the kfunc requires this memory size to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); + return ret; + } + fallthrough; + case KF_ARG_MEM_SIZE: + { + struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); + struct bpf_reg_state *size_reg = reg; + argno_t buff_argno = argno_from_arg(i); + enum bpf_mem_size_failure failure; - if (is_kfunc_arg_const_mem_size(meta->btf, size_arg, size_reg)) { - if (meta->arg_constant.found) { - verifier_bug(env, "only one constant argument permitted"); - return -EFAULT; - } - if (!tnum_is_const(size_reg->var_off)) { - verbose(env, "%s must be a known constant\n", - reg_arg_name(env, next_argno)); - return -EINVAL; - } - meta->arg_constant.found = true; - meta->arg_constant.value = size_reg->var_off.value; + if (reg->type != SCALAR_VALUE) { + verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar length for this memory argument.", + "the kfunc expects a scalar memory size, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); + return -EINVAL; } - /* Skip next '__sz' or '__szk' argument */ - i++; + if (bpf_register_is_null(buff_reg)) + break; + + ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, + BPF_READ | BPF_WRITE, true, meta, &failure); + if (ret < 0) { + const char *buff_arg, *size_arg; + + buff_arg = bpf_diag_arg_name(env, buff_argno); + size_arg = bpf_diag_arg_name(env, argno); + verbose(env, "%s and ", reg_arg_name(env, buff_argno)); + verbose(env, "%s memory, len pair leads to invalid memory access\n", + reg_arg_name(env, argno)); + if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { + bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, func_name, + "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", + "it is the memory pointer in a memory/length pair with %s, but %s does not describe verifier-readable memory for the requested length", + size_arg, buff_arg); + } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { + if (reg_smin(size_reg) < 0) + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may be negative because its signed minimum is %lld", + size_arg, reg_smin(size_reg)); + else + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", + size_arg, reg_umax(size_reg), BPF_MAX_VAR_SIZ); + } + return ret; + } break; } case KF_ARG_PTR_TO_CALLBACK: @@ -12417,6 +13139,11 @@ check_ok: if (!type_is_ptr_alloc_obj(reg->type)) { verbose(env, "%s is neither owning or non-owning ref\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an owning or non-owning pointer to a BPF-managed object containing a bpf_refcount field.", + "the kfunc expects a pointer to a BPF-managed refcounted object, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (!type_is_non_owning_ref(reg->type)) @@ -12441,6 +13168,11 @@ check_ok: if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a const string\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value.", + "the kfunc expects a pointer to a constant string stored in verifier-known memory, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } ret = check_arg_const_str(env, reg, argno); @@ -12463,7 +13195,7 @@ check_ok: reg_arg_name(env, argno)); return -EINVAL; } - ret = process_timer_kfunc(env, reg, argno, meta); + ret = process_timer_func(env, reg, argno, &meta->map); if (ret < 0) return ret; break; @@ -12481,6 +13213,11 @@ check_ok: if (reg->type != PTR_TO_STACK) { verbose(env, "%s doesn't point to an irq flag on stack\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave().", + "the kfunc expects a stack pointer to an IRQ flag slot, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } ret = process_irq_flag(env, reg, argno, meta); @@ -12519,16 +13256,17 @@ check_ok: int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, s16 offset, - struct bpf_kfunc_call_arg_meta *meta) + struct bpf_call_arg_meta *meta) { struct bpf_kfunc_meta kfunc; int err; + memset(meta, 0, sizeof(*meta)); + err = fetch_kfunc_meta(env, func_id, offset, &kfunc); if (err) return err; - memset(meta, 0, sizeof(*meta)); meta->btf = kfunc.btf; meta->func_id = kfunc.id; meta->func_proto = kfunc.proto; @@ -12678,7 +13416,7 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * int arg, int insn_idx) { struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; const struct btf_param *args; const struct btf_type *t, *ref_t; const struct btf *btf; @@ -12739,7 +13477,7 @@ out: * 0 - fall-through to 'else' branch * < 0 - not fall-through to 'else' branch, return error */ -static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, +static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, struct bpf_reg_state *regs, struct bpf_insn_aux_data *insn_aux, const struct btf_type *ptr_type, struct btf *desc_btf) { @@ -12918,11 +13656,12 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_reg_state *regs = cur_regs(env); const char *func_name, *ptr_type_name; const struct btf_type *t, *ptr_type; - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; struct bpf_insn_aux_data *insn_aux; + const char *operation; int err, insn_idx = *insn_idx_p; - const struct btf_param *args; u32 i, nargs, ptr_type_id; + struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -12931,14 +13670,26 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; err = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta); - if (err == -EACCES && meta.func_name) + if (err == -EACCES && meta.func_name) { verbose(env, "calling kernel function %s is not allowed\n", meta.func_name); + operation = bpf_diag_fmt(env, "kfunc %s", meta.func_name); + bpf_diag_policy( + env, insn_idx, operation, "this program cannot call the kfunc", + "Use a kfunc allowed for this program type and attach point, or change the program context."); + } if (err) return err; desc_btf = meta.btf; func_name = meta.func_name; insn_aux = &env->insn_aux_data[insn_idx]; + desc = find_kfunc_desc(env->prog, insn->imm, insn->off); + if (!desc) { + verifier_bug(env, "kfunc descriptor not found for func_id %u", insn->imm); + return -EFAULT; + } + meta.fn = &desc->proto; + insn_aux->is_iter_next = bpf_is_iter_next_kfunc(&meta); if (!insn->off && @@ -12965,7 +13716,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, verbose(env, "failed to mark s32 range for retval in forked state for lock\n"); return err; } - __mark_btf_func_reg_size(env, regs, BPF_REG_0, sizeof(u32)); } else if (!insn->off && insn->imm == special_kfunc_list[KF___bpf_trap]) { verbose(env, "unexpected __bpf_trap() due to uninitialized variable?\n"); return -EFAULT; @@ -12973,12 +13723,19 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); + operation = bpf_diag_fmt(env, "destructive kfunc %s", meta.func_name); + bpf_diag_policy( + env, insn_idx, operation, "destructive kfuncs require CAP_SYS_BOOT", + "Load the program with CAP_SYS_BOOT, or avoid destructive kfuncs."); return -EACCES; } sleepable = bpf_is_kfunc_sleepable(&meta); if (sleepable && !in_sleepable(env)) { verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); + operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."); return -EACCES; } @@ -13016,11 +13773,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } - if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) { - meta.r0_size = sizeof(u64); - meta.r0_rdonly = false; - } - if (is_bpf_wq_set_callback_kfunc(meta.func_id)) { err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_timer_callback_state); @@ -13049,26 +13801,46 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (rcu_lock) { env->cur_state->active_rcu_locks++; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, true, + env->cur_state->active_rcu_locks); } else if (rcu_unlock) { if (env->cur_state->active_rcu_locks == 0) { verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); + bpf_diag_ctx_underflow( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU, + "Remove the extra bpf_rcu_read_unlock() call, or ensure this path first enters an RCU read lock region."); return -EINVAL; } - if (--env->cur_state->active_rcu_locks == 0) + env->cur_state->active_rcu_locks--; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, false, + env->cur_state->active_rcu_locks); + if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); } else if (preempt_disable) { env->cur_state->active_preempt_locks++; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, true, + env->cur_state->active_preempt_locks); } else if (preempt_enable) { if (env->cur_state->active_preempt_locks == 0) { verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); + bpf_diag_ctx_underflow( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_PREEMPT, + "Remove the extra bpf_preempt_enable() call, or ensure this path first disables preemption."); return -EINVAL; } env->cur_state->active_preempt_locks--; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, false, + env->cur_state->active_preempt_locks); + if (!in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); } if (sleepable && !in_sleepable_context(env)) { verbose(env, "kernel func %s is sleepable within %s\n", func_name, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."); return -EACCES; } @@ -13079,6 +13851,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) { verbose(env, "kernel func %s requires RCU critical section protection\n", func_name); + bpf_diag_ctx_required( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU, + "Call this kfunc between bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced."); return -EACCES; } @@ -13116,11 +13891,12 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } + bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { u32 regno = caller_saved[i]; bpf_mark_reg_not_init(env, ®s[regno]); - regs[regno].subreg_def = DEF_NOT_SUBREG; } invalidate_outgoing_stack_args(env, cur_func(env)); @@ -13142,7 +13918,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); - mark_btf_func_reg_size(env, BPF_REG_0, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -13153,15 +13928,19 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* kfunc returning 'void *' is equivalent to returning scalar */ mark_reg_unknown(env, regs, BPF_REG_0); } else if (!__btf_type_is_struct(ptr_type)) { - if (!meta.r0_size) { + if (!meta.ret_mem.found) { __u32 sz; if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) { - meta.r0_size = sz; + meta.ret_mem.found = true; + meta.ret_mem.size = sz; meta.r0_rdonly = true; } + + if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) + meta.r0_rdonly = false; } - if (!meta.r0_size) { + if (!meta.ret_mem.found) { ptr_type_name = btf_name_by_offset(desc_btf, ptr_type->name_off); verbose(env, @@ -13174,7 +13953,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM; - regs[BPF_REG_0].mem_size = meta.r0_size; + regs[BPF_REG_0].mem_size = meta.ret_mem.size; if (meta.r0_rdonly) regs[BPF_REG_0].type |= MEM_RDONLY; @@ -13229,7 +14008,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ regs[BPF_REG_0].id = ++env->id_gen; } - mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); if (is_kfunc_acquire(&meta)) { id = acquire_reference(env, insn_idx, 0); if (id < 0) @@ -13266,17 +14044,11 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, caller_info->stack_arg_cnt = stack_arg_cnt; } - args = (const struct btf_param *)(meta.func_proto + 1); - for (i = 0; i < min_t(int, nargs, MAX_BPF_FUNC_REG_ARGS); i++) { - u32 regno = i + 1; - - t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); - if (btf_type_is_ptr(t)) - mark_btf_func_reg_size(env, regno, sizeof(void *)); - else - /* scalar. ensured by check_kfunc_args() */ - mark_btf_func_reg_size(env, regno, t->size); - } + /* + * Record R0 before process_iter_next_call() snapshots the alternate + * iterator path's diagnostic position. + */ + bpf_diag_mod_end(env); if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); @@ -13661,9 +14433,8 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, * If we return -EACCES, caller may want to try again treating pointer as a * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. */ -static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, - struct bpf_insn *insn, - const struct bpf_reg_state *ptr_reg, +static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, + u32 ptr_regno, const struct bpf_reg_state *ptr_reg, const struct bpf_reg_state *off_reg) { struct bpf_verifier_state *vstate = env->cur_state; @@ -13675,6 +14446,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_sanitize_info info = {}; u8 opcode = BPF_OP(insn->code); u32 dst = insn->dst_reg; + const char *reason; int ret, bounds_ret; dst_reg = ®s[dst]; @@ -13698,12 +14470,24 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, verbose(env, "R%d 32-bit pointer arithmetic prohibited\n", dst); + reason = bpf_diag_fmt( + env, "R%d holds %s. 32-bit ALU operations on pointers discard pointer tracking, so the verifier cannot keep the result as a safe pointer.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "32-bit pointer arithmetic", reason, + "Use a 64-bit ALU instruction with an allowed, bounded scalar offset."); return -EACCES; } if (ptr_reg->type & PTR_MAYBE_NULL) { verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", dst, reg_type_str(env, ptr_reg->type)); + reason = bpf_diag_fmt( + env, "R%d may be NULL (%s). Pointer arithmetic is allowed only after the program proves the pointer is non-NULL on this path.", + ptr_regno, reg_type_str(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer arithmetic before NULL check", reason, + "Make sure that a NULL check precedes any arithmetic performed on the pointer."); return -EACCES; } @@ -13733,6 +14517,12 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, default: verbose(env, "R%d pointer arithmetic on %s prohibited\n", dst, reg_type_str(env, ptr_reg->type)); + reason = bpf_diag_fmt( + env, "R%d holds %s. This pointer kind does not allow offset arithmetic.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer arithmetic is not allowed", reason, + "Do not change this pointer's offset; use it only in operations accepted for its kind."); return -EACCES; } @@ -13750,9 +14540,25 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) return 0; - if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) || - !check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) + if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "The scalar offset used with R%d is unbounded or outside the verifier's safe pointer-offset range [-%u, %u].", + ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Clamp or bounds-check the scalar offset before applying it to the pointer."); + return -EINVAL; + } + if (!check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "R%d already has an offset outside the verifier's safe range [-%u, %u] for %s.", + ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF, + bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Keep the base pointer within the verifier's allowed offset range before applying more arithmetic."); return -EINVAL; + } /* pointer types do not carry 32-bit bounds at the moment. */ __mark_reg32_unbounded(dst_reg); @@ -13795,6 +14601,13 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, /* scalar -= pointer. Creates an unknown scalar */ verbose(env, "R%d tried to subtract pointer from scalar\n", dst); + reason = bpf_diag_fmt( + env, "This operation subtracts pointer register R%d from scalar register R%d. " + "The verifier only tracks pointer-minus-scalar arithmetic for allowed pointer types.", + ptr_regno, dst); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer subtracted from scalar", reason, + "Keep the pointer as the base; only add or subtract bounded scalars when permitted."); return -EACCES; } /* We don't allow subtraction from FP, because (according to @@ -13804,6 +14617,12 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, if (ptr_reg->type == PTR_TO_STACK) { verbose(env, "R%d subtraction from stack pointer prohibited\n", dst); + reason = bpf_diag_fmt( + env, "R%d is a stack pointer. The verifier does not allow BPF_SUB to move stack pointers.", + ptr_regno); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "subtraction from stack pointer", reason, + "Use addition from R10 to form stack addresses within the tracked stack frame."); return -EACCES; } dst_reg->r64 = cnum64_add(ptr_reg->r64, cnum64_negate(off_reg->r64)); @@ -13829,16 +14648,38 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, /* bitwise ops on pointers are troublesome, prohibit. */ verbose(env, "R%d bitwise operator %s on pointer prohibited\n", dst, bpf_alu_string[opcode >> 4]); + reason = bpf_diag_fmt( + env, "R%d holds %s. Bitwise operator %s would destroy the pointer value the verifier is tracking.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type), + bpf_alu_string[opcode >> 4]); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "bitwise operation on pointer", reason, + "Do bitwise operations on scalar values, not on pointer-valued registers."); return -EACCES; default: /* other operators (e.g. MUL,LSH) produce non-pointer results */ verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", dst, bpf_alu_string[opcode >> 4]); + reason = bpf_diag_fmt( + env, "R%d holds %s. Operator %s is not one of the limited pointer arithmetic operations the verifier can track.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type), + bpf_alu_string[opcode >> 4]); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "invalid pointer arithmetic operator", reason, + "Use only verifier-supported addition or subtraction with a bounded scalar offset, or perform this operation on a scalar value."); return -EACCES; } - if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) + if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "After this arithmetic, R%d would be outside the verifier's safe offset range [-%u, %u] for %s.", + dst, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF, + bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range."); return -EINVAL; + } reg_bounds_sync(dst_reg); bounds_ret = sanitize_check_bounds(env, insn, dst_reg); if (bounds_ret == -EACCES) @@ -14770,14 +15611,14 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, if (dst_reg->type != PTR_TO_ARENA) *dst_reg = *src_reg; - dst_reg->subreg_def = env->insn_idx + 1; - - if (BPF_CLASS(insn->code) == BPF_ALU64) + if (BPF_CLASS(insn->code) == BPF_ALU64) { /* * 32-bit operations zero upper bits automatically. * 64-bit operations need to be converted to 32. */ aux->needs_zext = true; + aux->zext_dst = true; + } /* Any arithmetic operations are allowed on arena pointers */ return 0; @@ -14810,15 +15651,15 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, if (err) return err; off_reg = *dst_reg; - return adjust_ptr_min_max_vals(env, insn, src_reg, &off_reg); + return adjust_ptr_min_max_vals(env, insn, insn->src_reg, src_reg, + &off_reg); } } else if (ptr_reg) { /* pointer += scalar */ err = mark_chain_precision(env, insn->src_reg); if (err) return err; - return adjust_ptr_min_max_vals(env, insn, - dst_reg, src_reg); + return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, dst_reg, src_reg); } else if (dst_reg->precise) { /* if dst_reg is precise, src_reg should be precise as well */ err = mark_chain_precision(env, insn->src_reg); @@ -14833,8 +15674,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, __mark_reg_known(&off_reg, insn->imm); src_reg = &off_reg; if (ptr_reg) /* pointer += K */ - return adjust_ptr_min_max_vals(env, insn, - ptr_reg, src_reg); + return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, ptr_reg, src_reg); } /* Got here implies adding two SCALAR_VALUEs */ @@ -14920,6 +15760,8 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) u8 opcode = BPF_OP(insn->code); int err; + bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); + if (opcode == BPF_END || opcode == BPF_NEG) { /* check src operand */ err = check_reg_arg(env, insn->dst_reg, SRC_OP); @@ -14973,18 +15815,14 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) if (insn->imm) { /* off == BPF_ADDR_SPACE_CAST */ mark_reg_unknown(env, regs, insn->dst_reg); - if (insn->imm == 1) { /* cast from as(1) to as(0) */ + if (insn->imm == 1) /* cast from as(1) to as(0) */ dst_reg->type = PTR_TO_ARENA; - /* PTR_TO_ARENA is 32-bit */ - dst_reg->subreg_def = env->insn_idx + 1; - } } else if (insn->off == 0) { /* case: R1 = R2 * copy register state to dest reg */ assign_scalar_id_before_mov(env, src_reg); *dst_reg = *src_reg; - dst_reg->subreg_def = DEF_NOT_SUBREG; } else { /* case: R1 = (s8, s16 s32)R2 */ if (is_pointer_value(env, insn->src_reg)) { @@ -15002,7 +15840,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) if (!no_sext) clear_scalar_id(dst_reg); coerce_reg_to_size_sx(dst_reg, insn->off >> 3); - dst_reg->subreg_def = DEF_NOT_SUBREG; } else { mark_reg_unknown(env, regs, insn->dst_reg); } @@ -15027,7 +15864,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) */ if (!is_src_reg_u32) clear_scalar_id(dst_reg); - dst_reg->subreg_def = env->insn_idx + 1; } else { /* case: W1 = (s8, s16)W2 */ bool no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1)); @@ -15037,7 +15873,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) *dst_reg = *src_reg; if (!no_sext) clear_scalar_id(dst_reg); - dst_reg->subreg_def = env->insn_idx + 1; coerce_subreg_to_size_sx(dst_reg, insn->off >> 3); } } else { @@ -15100,7 +15935,12 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; } - return reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); + err = reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); + if (err) + return err; + + bpf_diag_mod_end(env); + return 0; } static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, @@ -15715,7 +16555,7 @@ static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, * No one could have freed the reference state before * doing the NULL check. */ - WARN_ON_ONCE(release_reference_nomark(vstate, id)); + WARN_ON_ONCE(__release_reference_nomark(vstate, id)); bpf_for_each_reg_in_vstate(vstate, state, reg, ({ mark_ptr_or_null_reg(state, reg, id, is_null); @@ -15906,12 +16746,8 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s continue; if ((!(reg->id & BPF_ADD_CONST) && !(known_reg->id & BPF_ADD_CONST)) || reg->delta == known_reg->delta) { - s32 saved_subreg_def = reg->subreg_def; - *reg = *known_reg; - reg->subreg_def = saved_subreg_def; } else { - s32 saved_subreg_def = reg->subreg_def; s32 saved_off = reg->delta; u32 saved_id = reg->id; @@ -15921,12 +16757,11 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s /* reg = known_reg; reg += delta */ *reg = *known_reg; /* - * Must preserve off, id and subreg_def flag, - * otherwise another sync_linked_regs() will be incorrect. + * Must preserve off and id, otherwise another sync_linked_regs() + * will be incorrect. */ reg->delta = saved_off; reg->id = saved_id; - reg->subreg_def = saved_subreg_def; scalar32_min_max_add(reg, &fake_reg); scalar_min_max_add(reg, &fake_reg); @@ -16199,11 +17034,13 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; dst_reg = ®s[insn->dst_reg]; + bpf_diag_mod_begin(env, dst_reg, NULL, BPF_DIAG_MOD_WRITE); if (insn->src_reg == 0) { u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; dst_reg->type = SCALAR_VALUE; __mark_reg_known(®s[insn->dst_reg], imm); + bpf_diag_mod_end(env); return 0; } @@ -16227,6 +17064,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) verifier_bug(env, "pseudo btf id: unexpected dst reg type"); return -EFAULT; } + bpf_diag_mod_end(env); return 0; } @@ -16246,6 +17084,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) dst_reg->type = PTR_TO_FUNC; dst_reg->subprogno = subprogno; + bpf_diag_mod_end(env); return 0; } @@ -16256,6 +17095,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) if (map->map_type == BPF_MAP_TYPE_ARENA) { __mark_reg_unknown(env, dst_reg); dst_reg->map_ptr = map; + bpf_diag_mod_end(env); return 0; } __mark_reg_known(dst_reg, aux->map_off); @@ -16273,6 +17113,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) return -EFAULT; } + bpf_diag_mod_end(env); return 0; } @@ -16351,6 +17192,8 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; /* reset caller saved regs to unreadable */ + bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); @@ -16361,8 +17204,7 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) * Already marked as written above. */ mark_reg_unknown(env, regs, BPF_REG_0); - /* ld_abs load up to 32-bit skb data. */ - regs[BPF_REG_0].subreg_def = env->insn_idx + 1; + bpf_diag_mod_end(env); /* * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0 * which must be explored by the verifier when in a subprog. @@ -16383,7 +17225,6 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return 0; } - static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_range *range) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -16672,12 +17513,11 @@ bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, struct bpf_call_summary *cs) { - struct bpf_kfunc_call_arg_meta meta; + struct bpf_call_arg_meta meta; const struct bpf_func_proto *fn; int i; if (bpf_helper_call(call)) { - if (bpf_get_helper_proto(env, call->imm, &fn) < 0) /* error would be reported later */ return false; @@ -16987,6 +17827,8 @@ static bool reg_type_mismatch_ok(enum bpf_reg_type type) case PTR_TO_BTF_ID: case PTR_TO_ARENA: return false; + case PTR_TO_MEM: + return !bpf_may_fault_on_deref(type); default: return true; } @@ -17010,27 +17852,29 @@ static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) !reg_type_mismatch_ok(prev)); } -static bool is_ptr_to_mem_or_btf_id(enum bpf_reg_type type) +static bool is_ptr_to_mem(enum bpf_reg_type type) { - switch (base_type(type)) { - case PTR_TO_MEM: - case PTR_TO_BTF_ID: - return true; - default: - return false; - } + return base_type(type) == PTR_TO_MEM; } -static bool is_ptr_to_mem(enum bpf_reg_type type) +static enum bpf_reg_type merge_ptr_types(enum bpf_reg_type type_a, + enum bpf_reg_type type_b) { - return base_type(type) == PTR_TO_MEM; + bool to_mem = is_ptr_to_mem(type_a) || is_ptr_to_mem(type_b); + enum bpf_reg_type type_merged = to_mem ? PTR_TO_MEM : PTR_TO_BTF_ID; + + if (bpf_may_fault_on_deref(type_a) || bpf_may_fault_on_deref(type_b)) + type_merged |= to_mem ? MEM_RDONLY | PTR_UNTRUSTED : + PTR_UNTRUSTED; + else + type_merged |= ((type_a | type_b) & MEM_RDONLY); + return type_merged; } static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch) { enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type; - enum bpf_reg_type merged_type; if (*prev_type == NOT_INIT) { /* Saw a valid insn @@ -17047,24 +17891,16 @@ static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type typ * Reject it. */ if (allow_trust_mismatch && - is_ptr_to_mem_or_btf_id(type) && - is_ptr_to_mem_or_btf_id(*prev_type)) { + bpf_is_ptr_to_mem_or_btf_id(type) && + bpf_is_ptr_to_mem_or_btf_id(*prev_type)) { /* * Have to support a use case when one path through - * the program yields TRUSTED pointer while another - * is UNTRUSTED. Fallback to UNTRUSTED to generate - * BPF_PROBE_MEM/BPF_PROBE_MEMSX. - * Same behavior of MEM_RDONLY flag. + * the program yields a TRUSTED pointer while another + * is UNTRUSTED. Merge them into a type which keeps + * the BPF_PROBE_MEM/BPF_PROBE_MEMSX rewrite when + * either side needs it. */ - if (is_ptr_to_mem(type) || is_ptr_to_mem(*prev_type)) - merged_type = PTR_TO_MEM; - else - merged_type = PTR_TO_BTF_ID; - if ((type & PTR_UNTRUSTED) || (*prev_type & PTR_UNTRUSTED)) - merged_type |= PTR_UNTRUSTED; - if ((type & MEM_RDONLY) || (*prev_type & MEM_RDONLY)) - merged_type |= MEM_RDONLY; - *prev_type = merged_type; + *prev_type = merge_ptr_types(type, *prev_type); } else { verbose(env, "same insn cannot be used with different pointers\n"); return -EINVAL; @@ -17288,9 +18124,13 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) insn->imm != BPF_FUNC_spin_unlock && insn->imm != BPF_FUNC_kptr_xchg) || (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && - (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) { + !kfunc_spin_allowed(env, insn->imm, insn->off))) { verbose(env, "function calls are not allowed while holding a lock\n"); + bpf_diag_ctx_active( + env, env->insn_idx, + "function call", BPF_DIAG_CONTEXT_LOCK, + "Release the BPF spin lock before making this call, or move the call outside the locked region."); return -EINVAL; } } @@ -17364,7 +18204,9 @@ static int do_check(struct bpf_verifier_env *env) insn = &insns[env->insn_idx]; insn_aux = &env->insn_aux_data[env->insn_idx]; - if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { + account_processed_insn(env); + + if (env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { verbose(env, "BPF program is too large. Processed %d insn\n", env->insn_processed); @@ -17373,6 +18215,27 @@ static int do_check(struct bpf_verifier_env *env) state->last_insn_idx = env->prev_insn_idx; state->insn_idx = env->insn_idx; + /* + * Record the incoming edge so active and queued paths use the same + * branch-recording path. A zero-offset conditional has identical + * successors, so its outcome cannot be reconstructed from the edge. + */ + if (!state->speculative && prev_insn_idx >= 0 && prev_insn_idx < insn_cnt) { + struct bpf_insn *prev_insn = &insns[prev_insn_idx]; + int fallthrough_idx = prev_insn_idx + 1; + int branch_idx = prev_insn_idx + bpf_jmp_offset(prev_insn) + 1; + u8 class = BPF_CLASS(prev_insn->code); + u8 opcode = BPF_OP(prev_insn->code); + + if ((class == BPF_JMP || class == BPF_JMP32) && + opcode != BPF_JA && opcode != BPF_CALL && opcode != BPF_EXIT && + opcode <= BPF_JCOND && branch_idx != fallthrough_idx) { + if (env->insn_idx == branch_idx) + bpf_diag_record_branch(env, prev_insn_idx, true); + else if (env->insn_idx == fallthrough_idx) + bpf_diag_record_branch(env, prev_insn_idx, false); + } + } if (bpf_is_prune_point(env, env->insn_idx)) { err = bpf_is_state_visited(env, env->insn_idx); @@ -17422,6 +18285,7 @@ static int do_check(struct bpf_verifier_env *env) env->prev_log_pos = env->log.end_pos; verbose(env, "%d: ", env->insn_idx); bpf_verbose_insn(env, insn); + verbose(env, "\n"); env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos; env->prev_log_pos = env->log.end_pos; } @@ -17504,6 +18368,7 @@ static int do_check(struct bpf_verifier_env *env) "speculation barrier after jump instruction may not have the desired effect")) return -EFAULT; process_bpf_exit: + account_current_path(env); mark_verifier_state_scratched(env); err = bpf_update_branch_counts(env, env->cur_state); if (err) @@ -17566,6 +18431,11 @@ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) if (env->used_btfs[i].btf == btf) goto ret_put; + if (env->signature) { + verbose(env, "signed program cannot bind any BTF\n"); + ret = -EACCES; + goto ret_put; + } if (env->used_btf_cnt >= MAX_USED_BTFS) { verbose(env, "The total number of btfs per program has reached the limit of %u\n", MAX_USED_BTFS); @@ -17758,7 +18628,9 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); return -EINVAL; } + } + if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) { if (is_tracing_prog_type(prog_type)) { verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); return -EINVAL; @@ -17830,6 +18702,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, return -EOPNOTSUPP; } env->prog->aux->arena = (void *)map; + env->prog->jit_required = true; if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { verbose(env, "arena's user address must be set via map_extra or mmap()\n"); return -EINVAL; @@ -17848,6 +18721,12 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) if (env->used_maps[i] == map) return i; + if (env->signature && + env->prog->aux->sig.verdict == BPF_SIG_VERIFIED) { + verbose(env, "signed program cannot bind map '%s' not covered by the signature\n", + map->name); + return -EACCES; + } if (env->used_map_cnt >= MAX_USED_MAPS) { verbose(env, "The total number of maps per program has reached the limit of %u\n", MAX_USED_MAPS); @@ -17877,6 +18756,7 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) return err; } env->insn_array_maps[env->insn_array_map_cnt++] = map; + env->prog->jit_required = true; } return env->used_map_cnt - 1; @@ -17900,6 +18780,48 @@ static int add_used_map(struct bpf_verifier_env *env, int fd) return __add_used_map(env, map); } +static int fd_array_get_map_idx_continuous(struct bpf_verifier_env *env, u32 idx) +{ + struct bpf_map *map; + + if (idx >= env->fd_array_cnt) { + verbose(env, "fd_idx %u out of bounds, fd_array_cnt %u\n", + idx, env->fd_array_cnt); + return -EINVAL; + } + map = fd_slot_map(env->fd_array[idx]); + if (!map) { + verbose(env, "fd_idx %u is not a map\n", idx); + return -EINVAL; + } + return __add_used_map(env, map); +} + +static int fd_array_get_map_idx_sparse(struct bpf_verifier_env *env, u32 idx) +{ + int fd; + + if (copy_from_bpfptr_offset(&fd, env->fd_array_raw, + (size_t)idx * sizeof(fd), sizeof(fd))) + return -EFAULT; + return add_used_map(env, fd); +} + +static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx) +{ + if (env->fd_array) + return fd_array_get_map_idx_continuous(env, idx); + if (env->signature) { + verbose(env, "signed program must bind maps via a continuous fd_array (fd_array_cnt)\n"); + return -EACCES; + } + if (!bpfptr_is_null(env->fd_array_raw)) + return fd_array_get_map_idx_sparse(env, idx); + + verbose(env, "fd_idx without fd_array is invalid\n"); + return -EPROTO; +} + static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn) { u8 class = BPF_CLASS(insn->code); @@ -18117,7 +19039,6 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) struct bpf_map *map; int map_idx; u64 addr; - u32 fd; if (i == insn_cnt - 1 || insn[1].code != 0 || insn[1].dst_reg != 0 || insn[1].src_reg != 0 || @@ -18169,21 +19090,17 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) switch (insn[0].src_reg) { case BPF_PSEUDO_MAP_IDX_VALUE: case BPF_PSEUDO_MAP_IDX: - if (bpfptr_is_null(env->fd_array)) { - verbose(env, "fd_idx without fd_array is invalid\n"); - return -EPROTO; - } - if (copy_from_bpfptr_offset(&fd, env->fd_array, - insn[0].imm * sizeof(fd), - sizeof(fd))) - return -EFAULT; + map_idx = fd_array_get_map_idx(env, insn[0].imm); break; default: - fd = insn[0].imm; + if (env->signature) { + verbose(env, "signed program cannot reference a map by fd, only via fd_array index\n"); + return -EINVAL; + } + map_idx = add_used_map(env, insn[0].imm); break; } - map_idx = add_used_map(env, fd); if (map_idx < 0) return map_idx; map = env->used_maps[map_idx]; @@ -18277,8 +19194,6 @@ static void release_insn_arrays(struct bpf_verifier_env *env) bpf_insn_array_release(env->insn_array_maps[i]); } - - /* The verifier does more data flow analysis than llvm and will not * explore branches that are dead at run time. Malicious programs can * have dead code too. Therefore replace all dead at-run-time code @@ -18306,8 +19221,6 @@ static void sanitize_dead_code(struct bpf_verifier_env *env) } } - - static void free_states(struct bpf_verifier_env *env) { struct bpf_verifier_state_list *sl; @@ -18358,6 +19271,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) struct bpf_prog_aux *aux = env->prog->aux; struct bpf_verifier_state *state; struct bpf_reg_state *regs; + u32 insn_processed = env->insn_processed; int ret, i; env->prev_linfo = NULL; @@ -18385,7 +19299,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) regs = state->frame[state->curframe]->regs; if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { - const char *sub_name = subprog_name(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); struct bpf_subprog_arg_info *arg; struct bpf_reg_state *reg; @@ -18496,9 +19410,22 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) ret = do_check(env); out: - if (!ret && pop_log) - bpf_vlog_reset(&env->log, 0); + account_current_path(env); + if (!ret) { + if (pop_log) + bpf_vlog_reset(&env->log, 0); + bpf_diag_event_log_restore(env, 0); + } free_states(env); + + /* + * The override is needed to account for async subprograms, which + * are verified with their own set of stack frames and thus are + * not accounted as callees by account_current_path(). + * Accumulate their total counts as total counts of the main or + * global subprog hosting the async call. + */ + env->subprog_info[subprog].insns_total = env->insn_processed - insn_processed; return ret; } @@ -18527,7 +19454,6 @@ static int do_check_subprogs(struct bpf_verifier_env *env) struct bpf_prog_aux *aux = env->prog->aux; struct bpf_func_info_aux *sub_aux; int i, ret, new_cnt; - u32 insn_processed; if (!aux->func_info) return 0; @@ -18542,8 +19468,6 @@ again: if (!bpf_subprog_is_global(env, i)) continue; - insn_processed = env->insn_processed; - sub_aux = subprog_aux(env, i); if (!sub_aux->called || sub_aux->verified) continue; @@ -18551,12 +19475,11 @@ again: env->insn_idx = env->subprog_info[i].start; WARN_ON_ONCE(env->insn_idx == 0); ret = do_check_common(env, i); - env->subprog_info[i].insn_processed = env->insn_processed - insn_processed; if (ret) { return ret; } else if (env->log.level & BPF_LOG_LEVEL) { verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", - i, subprog_name(env, i)); + i, bpf_subprog_name(env, i)); } /* We verified new global subprog, it might have called some @@ -18578,18 +19501,15 @@ again: static int do_check_main(struct bpf_verifier_env *env) { - u32 insn_processed = env->insn_processed; int ret; env->insn_idx = 0; ret = do_check_common(env, 0); - env->subprog_info[0].insn_processed = env->insn_processed - insn_processed; if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; } - static void print_verification_stats(struct bpf_verifier_env *env) { /* Skip over hidden subprogs which are not verified. */ @@ -18598,15 +19518,20 @@ static void print_verification_stats(struct bpf_verifier_env *env) if (env->log.level & BPF_LOG_STATS) { verbose(env, "verification time %lld usec\n", div_u64(env->verification_time, 1000)); - verbose(env, "stack depth %d", env->subprog_info[0].stack_depth); - for (i = 1; i < subprog_cnt; i++) - verbose(env, "+%d", env->subprog_info[i].stack_depth); - verbose(env, " max %d\n", env->max_stack_depth); - verbose(env, "insns processed %d", env->subprog_info[0].insn_processed); - for (i = 1; i < subprog_cnt; i++) - if (bpf_subprog_is_global(env, i)) - verbose(env, "+%d", env->subprog_info[i].insn_processed); - verbose(env, "\n"); + verbose(env, "stack depth max %d\n", env->max_stack_depth); + for (i = 0; i < subprog_cnt; i++) { + const char *name = env->subprog_info[i].name; + const char *kind; + + if (!name || !name[0]) + name = "<unknown>"; + kind = i == 0 ? "main" : + bpf_subprog_is_global(env, i) ? "global" : "static"; + verbose(env, "subprog %d (%s) %s insns_self %d insns_total %d stack %d\n", + i, name, kind, env->subprog_info[i].insns_self, + env->subprog_info[i].insns_total, + env->subprog_info[i].stack_depth); + } } verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " "total_states %d peak_states %d mark_read %d\n", @@ -18628,6 +19553,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) { const struct btf_type *t, *func_proto; const struct bpf_struct_ops_desc *st_ops_desc; + const struct bpf_struct_ops_arg_info *arg_info; const struct bpf_struct_ops *st_ops; const struct btf_member *member; struct bpf_prog *prog = env->prog; @@ -18706,10 +19632,23 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) return -EACCES; } - for (i = 0; i < st_ops_desc->arg_info[member_idx].cnt; i++) { - if (st_ops_desc->arg_info[member_idx].info[i].refcounted) { + arg_info = &st_ops_desc->arg_info[member_idx]; + for (i = 0; i < arg_info->cnt; i++) { + const struct bpf_ctx_arg_aux *info = &arg_info->info[i]; + + if (info->refcounted) has_refcounted_arg = true; - break; + if (base_type(info->reg_type) == PTR_TO_ARENA) { + if (!bpf_jit_supports_arena_args()) { + verbose(env, "JIT does not support arena arguments\n"); + return -ENOTSUPP; + } + if (!prog->aux->arena) { + verbose(env, + "arena argument of %s requires a program with an associated arena\n", + mname); + return -EINVAL; + } } } @@ -18730,8 +19669,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) prog->aux->attach_func_name = mname; env->ops = st_ops->verifier_ops; - return bpf_prog_ctx_arg_info_init(prog, st_ops_desc->arg_info[member_idx].info, - st_ops_desc->arg_info[member_idx].cnt); + return bpf_prog_ctx_arg_info_init(prog, arg_info->info, arg_info->cnt); } #define SECURITY_PREFIX "security_" @@ -18838,6 +19776,9 @@ static int btf_id_allow_sleepable(u32 btf_id, unsigned long addr, const struct b const struct btf_type *t; const char *tname; + if (!btf_is_kernel(btf)) + return -EINVAL; + switch (prog->type) { case BPF_PROG_TYPE_TRACING: t = btf_type_by_id(btf, btf_id); @@ -18922,6 +19863,20 @@ btf_attach_func_proto(struct bpf_verifier_log *log, struct btf *btf, u32 func_id return btf_type_by_id(btf, func->type); } +static bool attach_uses_trampoline_retval(enum bpf_attach_type type) +{ + switch (type) { + case BPF_MODIFY_RETURN: + case BPF_TRACE_FEXIT: + case BPF_TRACE_FEXIT_MULTI: + case BPF_TRACE_FSESSION: + case BPF_TRACE_FSESSION_MULTI: + return true; + default: + return false; + } +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -18981,6 +19936,16 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, bpf_log(log, "Subprog %s doesn't exist\n", tname); return -EINVAL; } + /* + * A struct_ops indirect trampoline converts arena arguments + * before invoking its program. A tracing or extension program + * attached to the main program would see the converted offset as a + * regular BTF pointer. + */ + if (subprog == 0 && bpf_prog_has_arena_ctx_arg(tgt_prog)) { + bpf_log(log, "Cannot attach to a target with arena context arguments\n"); + return -EOPNOTSUPP; + } if (aux->func && aux->func[subprog]->aux->exception_cb) { bpf_log(log, "%s programs cannot attach to exception callback\n", @@ -19186,6 +20151,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, if (ret < 0) return ret; + if (tgt_info->fmodel.ret_size > 8 && + attach_uses_trampoline_retval(prog->expected_attach_type)) { + bpf_log(log, + "Attach to function %s with a >8 byte return value is not supported for this attach type\n", + tname); + return -EOPNOTSUPP; + } + /* * *.multi programs don't need an address during program * verification, we just take the module ref if needed. @@ -19421,7 +20394,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) return -ENOMEM; if (tgt_prog && tgt_prog->aux->tail_call_reachable) - tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX; + bpf_trampoline_set_flags(tr, BPF_TRAMP_F_TAIL_CALL_CTX); prog->aux->dst_trampoline = tr; return 0; @@ -19460,6 +20433,9 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt err = btf_distill_func_proto(NULL, btf, t, tname, &tgt_info->fmodel); if (err < 0) return err; + if (tgt_info->fmodel.ret_size > 8 && + attach_uses_trampoline_retval(prog->expected_attach_type)) + return -EOPNOTSUPP; if (btf_is_module(btf)) { /* The bpf program already holds reference to module. */ if (WARN_ON_ONCE(!prog->aux->mod)) @@ -19483,13 +20459,25 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct btf *bpf_get_btf_vmlinux(void) { - if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { - mutex_lock(&bpf_verifier_lock); - if (!btf_vmlinux) - btf_vmlinux = btf_parse_vmlinux(); - mutex_unlock(&bpf_verifier_lock); + /* Pairs with the smp_store_release() on the parse path below. */ + struct btf *btf = smp_load_acquire(&btf_vmlinux); + + if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { + mutex_lock(&btf_vmlinux_lock); + btf = btf_vmlinux; + if (!btf) { + btf = btf_parse_vmlinux(); + /* + * Order the parsed BTF contents and the globals the + * parse populated (e.g. bpf_ctx_convert.t) before + * the pointer publication. Pairs with the acquire + * on the lockless fast path above. + */ + smp_store_release(&btf_vmlinux, btf); + } + mutex_unlock(&btf_vmlinux_lock); } - return btf_vmlinux; + return btf; } /* @@ -19497,7 +20485,7 @@ struct btf *bpf_get_btf_vmlinux(void) * this case expect that every file descriptor in the array is either a map or * a BTF. Everything else is considered to be trash. */ -static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) +static int add_fd_from_fd_array(struct bpf_verifier_env *env, u32 idx, int fd) { struct bpf_map *map; struct btf *btf; @@ -19509,51 +20497,83 @@ static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) err = __add_used_map(env, map); if (err < 0) return err; + fd_slot_set_map(&env->fd_array[idx], map); return 0; } btf = __btf_get_by_fd(f); if (!IS_ERR(btf)) { btf_get(btf); - return __add_used_btf(env, btf); + err = __add_used_btf(env, btf); + if (err < 0) + return err; + fd_slot_set_btf(&env->fd_array[idx], btf); + return 0; } verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); return PTR_ERR(map); } -static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr) +/* + * A continuous fd_array is resolved into an in-memory cache with one slot + * per entry. The bound here is deliberately generous and not derived from + * the per-program object limits: Duplicate entries /are/ permitted, and + * the number of distinct maps and BTFs a program can bind is enforced when + * each entry is resolved by __add_used_map() and __add_used_btf(). + */ +#define MAX_FD_ARRAY_CNT 4096 + +static int process_fd_array_continuous(struct bpf_verifier_env *env, + bpfptr_t fd_array, u32 cnt) { - size_t size = sizeof(int); - int ret; - int fd; + int fd, ret; u32 i; - env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); - - /* - * The only difference between old (no fd_array_cnt is given) and new - * APIs is that in the latter case the fd_array is expected to be - * continuous and is scanned for map fds right away - */ - if (!attr->fd_array_cnt) - return 0; - - /* Check for integer overflow */ - if (attr->fd_array_cnt >= (U32_MAX / size)) { - verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt); - return -EINVAL; + if (cnt > MAX_FD_ARRAY_CNT) { + verbose(env, "fd_array has too many entries (%u, max %u)\n", + cnt, MAX_FD_ARRAY_CNT); + return -E2BIG; } - for (i = 0; i < attr->fd_array_cnt; i++) { - if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size)) + env->fd_array = kvcalloc(cnt, sizeof(*env->fd_array), + GFP_KERNEL_ACCOUNT); + if (!env->fd_array) + return -ENOMEM; + env->fd_array_cnt = cnt; + for (i = 0; i < cnt; i++) { + if (copy_from_bpfptr_offset(&fd, fd_array, + (size_t)i * sizeof(fd), sizeof(fd))) return -EFAULT; - - ret = add_fd_from_fd_array(env, fd); + ret = add_fd_from_fd_array(env, i, fd); if (ret) return ret; } + return 0; +} + +static int process_fd_array(struct bpf_verifier_env *env, + union bpf_attr *attr, bpfptr_t uattr) +{ + bpfptr_t fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); + if (bpfptr_is_null(fd_array)) { + if (attr->fd_array_cnt) { + verbose(env, "fd_array_cnt %u without fd_array is invalid\n", + attr->fd_array_cnt); + return -EINVAL; + } + return 0; + } + /* + * New API: the caller passes fd_array_cnt and a continuous array that + * is resolved and bound up front. Legacy API (no fd_array_cnt): keep + * the caller's array and resolve entries on the spot at each reference. + */ + if (attr->fd_array_cnt) + return process_fd_array_continuous(env, fd_array, + attr->fd_array_cnt); + env->fd_array_raw = fd_array; return 0; } @@ -19753,6 +20773,51 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, insn_buf[4] = BPF_ALU64_REG(BPF_SUB, BPF_REG_0, BPF_REG_1); insn_buf[5] = BPF_ALU64_IMM(BPF_NEG, BPF_REG_0, 0); *cnt = 6; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_new]) { + /* inline bpf_iter_num_new(&it, start, end); R1=&it, R2=start, R3=end */ + int i = 0; + + /* if (start > end) goto einval; */ + insn_buf[i++] = BPF_JMP32_REG(BPF_JSGT, BPF_REG_2, BPF_REG_3, 8); + /* r0 = (u32)end - (u32)start; if (r0 > BPF_MAX_LOOPS) goto e2big; */ + insn_buf[i++] = BPF_MOV32_REG(BPF_REG_0, BPF_REG_3); + insn_buf[i++] = BPF_ALU32_REG(BPF_SUB, BPF_REG_0, BPF_REG_2); + insn_buf[i++] = BPF_JMP_IMM(BPF_JGT, BPF_REG_0, BPF_MAX_LOOPS, 8); + /* s->cur = start - 1; s->end = end; return 0; */ + insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_2, -1); + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_2, 0); + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, 4); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); + insn_buf[i++] = BPF_JMP_A(5); + /* einval: s->cur = s->end = 0; return -EINVAL; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); + insn_buf[i++] = BPF_JMP_A(2); + /* e2big: s->cur = s->end = 0; return -E2BIG; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, -E2BIG); + *cnt = i; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_next]) { + /* inline bpf_iter_num_next(&it); R1=&it, returns &s->cur or NULL */ + int i = 0; + + /* r0 = s->cur + 1; if ((s32)r0 >= s->end) goto done; */ + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, 0); + insn_buf[i++] = BPF_ALU32_IMM(BPF_ADD, BPF_REG_0, 1); + insn_buf[i++] = BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 4); + insn_buf[i++] = BPF_JMP32_REG(BPF_JSGE, BPF_REG_0, BPF_REG_2, 3); + /* s->cur = r0; return &s->cur; */ + insn_buf[i++] = BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0); + insn_buf[i++] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1); + insn_buf[i++] = BPF_JMP_A(2); + /* done: s->cur = s->end = 0; return NULL; */ + insn_buf[i++] = BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0); + insn_buf[i++] = BPF_MOV64_IMM(BPF_REG_0, 0); + *cnt = i; + } else if (desc->func_id == special_kfunc_list[KF_bpf_iter_num_destroy]) { + /* bpf_iter_num_destroy() is a no-op; emit a nop to drop the call */ + insn_buf[0] = BPF_JMP_A(0); + *cnt = 1; } if (env->insn_aux_data[insn_idx].arg_prog) { @@ -19768,6 +20833,146 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) +{ + switch (keyring_id) { + case 0: + return BPF_SIG_KEYRING_BUILTIN; + case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING: + return BPF_SIG_KEYRING_SECONDARY; + case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: + return BPF_SIG_KEYRING_PLATFORM; + default: + return BPF_SIG_KEYRING_USER; + } +} + +/* + * Verify the PKCS#7 signature of a loaded program. Called from bpf_check() + * once the program's metadata maps have been resolved into used_maps, so + * the exact maps folded into the signature are the ones the program binds. + * + * The signature covers the instructions followed by the frozen contents of + * each map, in @maps order: insns || map_0 || map_1 || [...]. On success the + * verdict and keyring info are recorded on prog->aux. + */ +static int bpf_prog_verify_signature(struct bpf_verifier_env *env, + union bpf_attr *attr, bool is_kernel) +{ + bpfptr_t usig = make_bpfptr(attr->signature, is_kernel); + struct bpf_dynptr_kern sig_ptr, data_ptr; + struct bpf_prog *prog = env->prog; + struct bpf_map **maps = env->used_maps; + struct bpf_key *key = NULL; + void *sig, *data = NULL; + u32 map_cnt = env->used_map_cnt; + u32 i, off, insns_sz; + u64 data_sz; + int err = 0; + + /* + * Don't attempt to use kmalloc_large or vmalloc for signatures. + * Practical signature for BPF program should be below this limit. + */ + if (!attr->signature_size || + attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + return -EINVAL; + if (system_keyring_id_check(attr->keyring_id) == 0) + key = bpf_lookup_system_key(attr->keyring_id); + else + key = bpf_lookup_user_key(attr->keyring_id, 0); + if (!key) { + verbose(env, "cannot resolve signing keyring with keyring_id %d\n", + attr->keyring_id); + return -EINVAL; + } + + sig = kvmemdup_bpfptr(usig, attr->signature_size); + if (IS_ERR(sig)) { + bpf_key_put(key); + return PTR_ERR(sig); + } + + insns_sz = prog->len * sizeof(struct bpf_insn); + data_sz = insns_sz; + for (i = 0; i < map_cnt; i++) { + struct bpf_map *map = maps[i]; + + if (map->map_type != BPF_MAP_TYPE_ARRAY || + !map->ops->map_direct_value_addr) { + verbose(env, "signed program metadata map '%s' must be an array\n", + map->name); + err = -EINVAL; + goto out; + } + if (!READ_ONCE(map->frozen)) { + verbose(env, "signed program metadata map '%s' must be frozen\n", + map->name); + err = -EPERM; + goto out; + } + if (bpf_map_write_active(map)) { + verbose(env, "signed program metadata map '%s' has active writers\n", + map->name); + err = -EBUSY; + goto out; + } + if (!map->excl_prog_sha) { + verbose(env, "signed program metadata map '%s' must be exclusive\n", + map->name); + err = -EPERM; + goto out; + } + data_sz += map->value_size; + } + if (bpf_dynptr_check_size(data_sz)) { + verbose(env, "signed payload too large: %llu bytes\n", data_sz); + err = -E2BIG; + goto out; + } + data = kvmalloc(data_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!data) { + err = -ENOMEM; + goto out; + } + memcpy(data, prog->insnsi, insns_sz); + off = insns_sz; + for (i = 0; i < map_cnt; i++) { + struct bpf_map *map = maps[i]; + u64 addr; + + err = map->ops->map_direct_value_addr(map, &addr, 0); + if (err) { + verbose(env, "failed to read signed metadata map '%s': %d\n", + map->name, err); + goto out; + } + memcpy(data + off, (void *)(unsigned long)addr, + map->value_size); + off += map->value_size; + } + + bpf_dynptr_init(&data_ptr, data, BPF_DYNPTR_TYPE_LOCAL, 0, data_sz); + bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0, + attr->signature_size); + + err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&data_ptr, + (struct bpf_dynptr *)&sig_ptr, key); + if (err) { + verbose(env, "signature verification failed: %d\n", err); + } else { + verbose(env, "signature verification passed\n"); + prog->aux->sig.keyring_serial = bpf_key_serial(key); + prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id); + prog->aux->sig.verdict = BPF_SIG_VERIFIED; + } +out: + kvfree(data); + bpf_key_put(key); + kvfree(sig); + return err; +} + int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log) { @@ -19790,18 +20995,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, return -ENOMEM; env->bt.env = env; - - len = (*prog)->len; - env->insn_aux_data = - vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); - ret = -ENOMEM; - if (!env->insn_aux_data) - goto err_free_env; - for (i = 0; i < len; i++) - env->insn_aux_data[i].orig_idx = i; - env->succ = bpf_iarray_realloc(NULL, 2); - if (!env->succ) - goto err_free_env; env->prog = *prog; env->ops = bpf_verifier_ops[env->prog->type]; @@ -19810,22 +21003,55 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); - - bpf_get_btf_vmlinux(); - - /* grab the mutex to protect few globals used by verifier */ - if (!is_priv) - mutex_lock(&bpf_verifier_lock); + env->signature = attr->signature; /* user could have requested verbose verifier output * and supplied buffer to store the verification trace */ ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); if (ret) - goto err_unlock; + goto err_free_env; + ret = bpf_diag_init(env); + if (ret) + goto err_prep; + if (env->signature) { + ret = bpf_prog_calc_tag(env->prog); + if (ret < 0) + goto err_prep; + } ret = process_fd_array(env, attr, uattr); if (ret) + goto err_prep; + + if (env->signature) { + ret = bpf_prog_verify_signature(env, attr, uattr.is_kernel); + if (ret) + goto err_prep; + } + + ret = security_bpf_prog_load(env->prog, attr, env->prog->aux->token, + uattr.is_kernel); + if (ret) + goto err_prep; + + bpf_get_btf_vmlinux(); + + /* Serialize verification of unprivileged programs. */ + if (!is_priv) + mutex_lock(&bpf_verifier_lock); + + len = env->insn_aux_data_len = env->prog->len; + env->insn_aux_data = + __vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len), + GFP_KERNEL_ACCOUNT | __GFP_ZERO); + ret = -ENOMEM; + if (!env->insn_aux_data) + goto skip_full_check; + for (i = 0; i < len; i++) + env->insn_aux_data[i].orig_idx = i; + env->succ = bpf_iarray_realloc(NULL, 2); + if (!env->succ) goto skip_full_check; mark_verifier_state_clean(env); @@ -19858,11 +21084,13 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, INIT_LIST_HEAD(&env->explored_states[i]); INIT_LIST_HEAD(&env->free_list); - ret = bpf_check_btf_info_early(env, attr, uattr); + /* Prepare BTF and func_info needed to discover all subprograms. */ + ret = bpf_prepare_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; - ret = add_subprog_and_kfunc(env); + /* Discover all subprograms before validating their layout and BTF. */ + ret = add_subprogs(env); if (ret < 0) goto skip_full_check; @@ -19870,14 +21098,21 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* Validate BTF against the complete subprogram layout and apply CO-RE. */ ret = bpf_check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; + /* Validate instructions and resolve the program's referenced resources. */ ret = check_and_resolve_insns(env); if (ret < 0) goto skip_full_check; + /* Build kfunc prototypes after resolving program resources. */ + ret = add_kfuncs(env); + if (ret < 0) + goto skip_full_check; + if (bpf_prog_is_offloaded(env->prog->aux)) { ret = bpf_prog_offload_verifier_prep(env->prog); if (ret) @@ -20049,17 +21284,27 @@ err_release_maps: *prog = env->prog; module_put(env->attach_btf_mod); -err_unlock: if (!is_priv) mutex_unlock(&bpf_verifier_lock); - bpf_clear_insn_aux_data(env, 0, env->prog->len); + goto err_free_env; +err_prep: + err = bpf_log_attr_finalize(attr_log, &env->log); + if (err) + ret = err; + release_insn_arrays(env); + release_maps(env); + release_btfs(env); err_free_env: + if (env->insn_aux_data) + bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); + vfree(env->insn_aux_data); + kvfree(env->fd_array); bpf_stack_liveness_free(env); kvfree(env->cfg.insn_postorder); kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); - vfree(env->insn_aux_data); + bpf_diag_free(env); kvfree(env); return ret; } diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 75495a5c3507..29260951aa87 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -221,7 +221,7 @@ const struct bpf_func_proto bpf_probe_read_user_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -258,7 +258,7 @@ const struct bpf_func_proto bpf_probe_read_user_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -273,7 +273,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -308,7 +308,7 @@ const struct bpf_func_proto bpf_probe_read_kernel_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -328,7 +328,7 @@ static const struct bpf_func_proto bpf_probe_read_compat_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; @@ -347,7 +347,7 @@ static const struct bpf_func_proto bpf_probe_read_compat_str_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_ANYTHING, }; #endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */ @@ -383,7 +383,7 @@ static const struct bpf_func_proto bpf_probe_write_user_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; #define MAX_TRACE_PRINTK_VARARGS 3 @@ -418,7 +418,7 @@ static const struct bpf_func_proto bpf_trace_printk_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, }; static void __set_printk_clr_event(struct work_struct *work) @@ -474,9 +474,9 @@ static const struct bpf_func_proto bpf_trace_vprintk_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, }; const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void) @@ -518,9 +518,9 @@ static const struct bpf_func_proto bpf_seq_printf_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len) @@ -535,7 +535,7 @@ static const struct bpf_func_proto bpf_seq_write_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr, @@ -559,7 +559,7 @@ static const struct bpf_func_proto bpf_seq_printf_btf_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_seq_file_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -633,7 +633,7 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = { .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void) @@ -730,7 +730,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static DEFINE_PER_CPU(int, bpf_event_output_nest_level); @@ -996,7 +996,7 @@ static const struct bpf_func_proto bpf_d_path_proto = { .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_d_path_btf_ids[0], .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .allowed = bpf_d_path_allowed, }; @@ -1053,9 +1053,9 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg2_type = ARG_CONST_SIZE, + .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -1218,7 +1218,7 @@ const struct bpf_func_proto bpf_get_branch_snapshot_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value) @@ -1421,7 +1421,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map, @@ -1462,7 +1462,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_tp = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1524,12 +1524,12 @@ clear: } static const struct bpf_func_proto bpf_perf_prog_read_value_proto = { - .func = bpf_perf_prog_read_value, - .gpl_only = true, - .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .func = bpf_perf_prog_read_value, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_PTR_TO_UNINIT_MEM, + .arg3_type = ARG_MEM_SIZE, }; BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx, @@ -1566,7 +1566,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -1646,7 +1646,7 @@ static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; extern const struct bpf_func_proto bpf_skb_output_proto; @@ -1701,7 +1701,7 @@ static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -3687,7 +3687,7 @@ static void bpf_tracing_multi_link_release(struct bpf_link *link) struct bpf_tracing_multi_link *tr_link = container_of(link, struct bpf_tracing_multi_link, link); - WARN_ON_ONCE(bpf_trampoline_multi_detach(link->prog, tr_link)); + bpf_trampoline_multi_detach(link->prog, tr_link); } static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) @@ -3700,6 +3700,60 @@ static void bpf_tracing_multi_link_dealloc(struct bpf_link *link) kvfree(tr_link); } +static int bpf_tracing_multi_link_fill_link_info(const struct bpf_link *link, + struct bpf_link_info *info) +{ + u64 __user *ucookies = u64_to_user_ptr(info->tracing_multi.cookies); + u64 __user *uaddrs = u64_to_user_ptr(info->tracing_multi.addrs); + u32 __user *uids = u64_to_user_ptr(info->tracing_multi.ids); + struct bpf_tracing_multi_link *tr_link; + u32 ucount = info->tracing_multi.count; + bool has_cookies, show_addrs; + int err = 0; + + if ((uids || ucookies || uaddrs) && !ucount) + return -EINVAL; + + tr_link = container_of(link, struct bpf_tracing_multi_link, link); + + info->tracing_multi.attach_type = tr_link->link.attach_type; + info->tracing_multi.count = tr_link->nodes_cnt; + info->tracing_multi.btf_obj_id = btf_obj_id(tr_link->link.prog->aux->attach_btf); + + if (!uids && !ucookies && !uaddrs) + return 0; + + if (ucount < tr_link->nodes_cnt) + err = -ENOSPC; + else + ucount = tr_link->nodes_cnt; + + has_cookies = !!tr_link->cookies; + show_addrs = kallsyms_show_value(current_cred()); + + for (int i = 0; i < ucount; i++) { + struct bpf_tracing_multi_node *mnode = &tr_link->nodes[i]; + u64 addr, cookie; + u32 id; + + bpf_trampoline_unpack_key(mnode->trampoline->key, NULL, &id); + + addr = show_addrs ? mnode->trampoline->ip : 0; + cookie = has_cookies ? tr_link->cookies[i] : 0; + + if (uids && put_user(id, uids + i)) + return -EFAULT; + if (uaddrs && put_user(addr, uaddrs + i)) + return -EFAULT; + if (ucookies && put_user(cookie, ucookies + i)) + return -EFAULT; + + cond_resched(); + } + + return err; +} + #ifdef CONFIG_PROC_FS static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, struct seq_file *seq) @@ -3730,6 +3784,7 @@ static void bpf_tracing_multi_show_fdinfo(const struct bpf_link *link, static const struct bpf_link_ops bpf_tracing_multi_link_lops = { .release = bpf_tracing_multi_link_release, .dealloc_deferred = bpf_tracing_multi_link_dealloc, + .fill_link_info = bpf_tracing_multi_link_fill_link_info, #ifdef CONFIG_PROC_FS .show_fdinfo = bpf_tracing_multi_show_fdinfo, #endif diff --git a/net/core/Makefile b/net/core/Makefile index b3fdcb4e355f..c20e520358b8 100644 --- a/net/core/Makefile +++ b/net/core/Makefile @@ -44,6 +44,9 @@ obj-$(CONFIG_FAILOVER) += failover.o obj-$(CONFIG_NET_SOCK_MSG) += skmsg.o obj-$(CONFIG_BPF_SYSCALL) += sock_map.o obj-$(CONFIG_BPF_SYSCALL) += bpf_sk_storage.o +ifdef CONFIG_INET +obj-$(CONFIG_BPF_SYSCALL) += bpf_ksock.o +endif obj-$(CONFIG_OF) += of_net.o obj-$(CONFIG_NET_TEST) += net_test.o obj-$(CONFIG_NET_DEVMEM) += devmem.o diff --git a/net/core/bpf_ksock.c b/net/core/bpf_ksock.c new file mode 100644 index 000000000000..e9943aeeadd3 --- /dev/null +++ b/net/core/bpf_ksock.c @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Isovalent */ + +#include <linux/bpf.h> +#include <linux/bpf_ksock.h> +#include <linux/btf.h> +#include <linux/btf_ids.h> +#include <linux/in.h> +#include <linux/in6.h> +#include <linux/net.h> +#include <linux/refcount.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/socket.h> +#include <linux/unaligned.h> +#include <linux/workqueue.h> +#include <net/sock.h> + +/** + * struct bpf_ksock - refcounted BPF kernel socket context + * @sock: The underlying kernel socket. + * @usage: Reference counter. + * @rwork: RCU work for deferred cleanup (sock_release may sleep). + */ +struct bpf_ksock { + struct socket *sock; + refcount_t usage; + struct rcu_work rwork; +}; + +static void ksock_release_work_fn(struct work_struct *work) +{ + struct bpf_ksock *ks; + + ks = container_of(to_rcu_work(work), struct bpf_ksock, rwork); + sock_release(ks->sock); + kfree(ks); +} + +static bool bpf_ksock_has_user_task_context(void) +{ + /* + * Task work can run from do_exit() after exit_nsproxy_namespaces() + * cleared current->nsproxy, while current is still not a kthread. + */ + return !(current->flags & PF_KTHREAD) && current->nsproxy; +} + +__bpf_kfunc_start_defs(); + +/** + * bpf_ksock_create() - Create a BPF kernel socket. + * + * Allocates and creates a kernel socket. + * + * The returned context must either be stored in a map as a kptr, or + * freed with bpf_ksock_release(). + * + * This function may sleep (sock_create), so it can only be used + * in sleepable BPF programs (SYSCALL). + * It cannot be called from a BPF workqueue callback because that callback + * does not retain the invoking task's namespace or security context. + * + * @opts: Pointer to struct bpf_ksock_create_opts with socket parameters. + * @opts__sz: Size of the opts struct. + * @err__uninit: Integer to store error code when NULL is returned. + */ +__bpf_kfunc struct bpf_ksock * +bpf_ksock_create(const struct bpf_ksock_create_opts *opts, u32 opts__sz, + int *err__uninit) +{ + struct bpf_ksock_create_opts opts_copy; + struct bpf_ksock *ks; + int err; + + /* + * sock_create() derives the network namespace, credentials, and cgroup + * from current. Kernel threads, including BPF workqueue callbacks, do + * not carry the context of the task that invoked the BPF program. + */ + if (!bpf_ksock_has_user_task_context()) { + err = -EOPNOTSUPP; + goto err_out; + } + + if (!opts || opts__sz != sizeof(struct bpf_ksock_create_opts)) { + err = -EINVAL; + goto err_out; + } + + opts_copy = (struct bpf_ksock_create_opts){ + .family = READ_ONCE(opts->family), + .type = READ_ONCE(opts->type), + .protocol = READ_ONCE(opts->protocol), + .reserved = READ_ONCE(opts->reserved), + }; + + if (opts_copy.reserved) { + err = -EINVAL; + goto err_out; + } + + if (opts_copy.family != AF_INET && opts_copy.family != AF_INET6) { + err = -EAFNOSUPPORT; + goto err_out; + } + + if (opts_copy.type != SOCK_DGRAM) { + err = -EPROTONOSUPPORT; + goto err_out; + } + + if (opts_copy.protocol != IPPROTO_UDP && opts_copy.protocol != 0) { + err = -EPROTONOSUPPORT; + goto err_out; + } + + ks = kzalloc_obj(*ks); + if (!ks) { + err = -ENOMEM; + goto err_out; + } + + /* + * Use the normal current-task socket path so LSM/cgroup policy, + * socket labels, and the active netns reference match a socket(2) + * created by the BPF program's caller. + */ + err = sock_create(opts_copy.family, opts_copy.type, opts_copy.protocol, + &ks->sock); + if (err) + goto err_free; + + ks->sock->sk->sk_rcvbuf = SOCK_MIN_RCVBUF; + ks->sock->sk->sk_userlocks |= SOCK_RCVBUF_LOCK; + + refcount_set(&ks->usage, 1); + put_unaligned(0, err__uninit); + return ks; + +err_free: + kfree(ks); +err_out: + put_unaligned(err, err__uninit); + return NULL; +} + +/** + * bpf_ksock_connect() - Connect a BPF kernel socket to a remote address. + * @ks: The BPF kernel socket context. + * @addr: Pointer to an IPv4 or IPv6 socket address. + * @addr__sz: Size of the address union. + * + * Connects the socket to the specified remote address and port. + * + * This function may sleep while connecting the socket, so it can only be used + * in sleepable BPF programs (SYSCALL). + * + * Return: 0 on success, negative errno on error. + */ +__bpf_kfunc int bpf_ksock_connect(struct bpf_ksock *ks, + const union bpf_ksock_addr *addr, + u32 addr__sz) +{ + struct sockaddr_storage sa; + int addrlen; + + if (!bpf_ksock_has_user_task_context()) + return -EOPNOTSUPP; + + if (!addr || addr__sz != sizeof(*addr)) + return -EINVAL; + + /* Kfunc memory arguments may be unaligned. */ + memcpy(&sa, addr, sizeof(*addr)); + + switch (sa.ss_family) { + case AF_INET: + addrlen = sizeof(struct sockaddr_in); + break; + case AF_INET6: + addrlen = sizeof(struct sockaddr_in6); + break; + default: + return -EAFNOSUPPORT; + } + + return connect_socket(ks->sock, &sa, addrlen, 0); +} + +/** + * bpf_ksock_acquire() - Acquire a reference to a BPF kernel socket. + * @ks: The BPF kernel socket context to acquire. Must be a + * trusted pointer (e.g. RCU-protected kptr from a map). + * + * The acquired context must either be stored in a map as a kptr, or + * freed with bpf_ksock_release(). + */ +__bpf_kfunc struct bpf_ksock *bpf_ksock_acquire(struct bpf_ksock *ks) +{ + if (!refcount_inc_not_zero(&ks->usage)) + return NULL; + return ks; +} + +/** + * bpf_ksock_release() - Release a BPF kernel socket. + * @ks: The BPF kernel socket context to release. + * + * When the final reference is released, the socket is cleaned up via + * queue_rcu_work() (since sock_release may sleep). + */ +__bpf_kfunc void bpf_ksock_release(struct bpf_ksock *ks) +{ + if (refcount_dec_and_test(&ks->usage)) { + INIT_RCU_WORK(&ks->rwork, ksock_release_work_fn); + queue_rcu_work(system_dfl_wq, &ks->rwork); + } +} + +__bpf_kfunc void bpf_ksock_release_dtor(void *ks) +{ + bpf_ksock_release(ks); +} +CFI_NOSEAL(bpf_ksock_release_dtor); + +/** + * bpf_ksock_send() - Send data through a BPF kernel socket. + * @ks: The BPF kernel socket context. Must be an acquired reference. + * @data: Pointer to the data to send. + * @data__sz: Size of the data to send. + * + * Sends data on a connected socket, best-effort and nonblocking. This may sleep + * (kernel_sendmsg), so it can only be called from sleepable BPF programs. + * + * Return: Number of bytes sent on success, negative errno on error. + */ +__bpf_kfunc int bpf_ksock_send(struct bpf_ksock *ks, const void *data, + u32 data__sz) +{ + struct msghdr msg = { + .msg_flags = MSG_DONTWAIT, + }; + struct kvec iov = { + .iov_base = (void *)data, + .iov_len = data__sz, + }; + int ret; + + if (!bpf_ksock_has_user_task_context()) + return -EOPNOTSUPP; + + ret = kernel_sendmsg(ks->sock, &msg, &iov, 1, data__sz); + + return ret; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(ksock_init_kfunc_btf_ids) +BTF_ID_FLAGS(func, bpf_ksock_create, KF_ACQUIRE | KF_RET_NULL | KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_ksock_connect, KF_SLEEPABLE) +BTF_KFUNCS_END(ksock_init_kfunc_btf_ids) + +static const struct btf_kfunc_id_set ksock_init_kfunc_set = { + .owner = THIS_MODULE, + .set = &ksock_init_kfunc_btf_ids, +}; + +BTF_KFUNCS_START(ksock_kfunc_btf_ids) +BTF_ID_FLAGS(func, bpf_ksock_release, KF_RELEASE) +BTF_ID_FLAGS(func, bpf_ksock_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_ksock_send, KF_SLEEPABLE) +BTF_KFUNCS_END(ksock_kfunc_btf_ids) + +#ifdef CONFIG_BPF_LSM +BTF_ID_LIST_SINGLE(bpf_lsm_socket_sendmsg_id, func, bpf_lsm_socket_sendmsg) +#endif + +static int bpf_ksock_kfunc_filter(const struct bpf_prog *prog, u32 kfunc_id) +{ + if (!btf_id_set8_contains(&ksock_kfunc_btf_ids, kfunc_id)) + return 0; + + if (prog->type == BPF_PROG_TYPE_SYSCALL) + return 0; + +#ifdef CONFIG_BPF_LSM + if (prog->type == BPF_PROG_TYPE_LSM && + prog->aux->attach_btf_id != bpf_lsm_socket_sendmsg_id[0]) + return 0; +#endif + + return -EACCES; +} + +static const struct btf_kfunc_id_set ksock_kfunc_set = { + .owner = THIS_MODULE, + .set = &ksock_kfunc_btf_ids, + .filter = bpf_ksock_kfunc_filter, +}; + +BTF_ID_LIST(bpf_ksock_dtor_ids) +BTF_ID(struct, bpf_ksock) +BTF_ID(func, bpf_ksock_release_dtor) + +static int __init bpf_ksock_kfunc_init(void) +{ + int ret; + const struct btf_id_dtor_kfunc bpf_ksock_dtors[] = { + { + .btf_id = bpf_ksock_dtor_ids[0], + .kfunc_btf_id = bpf_ksock_dtor_ids[1], + }, + }; + + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &ksock_init_kfunc_set); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, + &ksock_kfunc_set); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LSM, + &ksock_kfunc_set); + return ret ?: register_btf_id_dtor_kfuncs(bpf_ksock_dtors, + ARRAY_SIZE(bpf_ksock_dtors), + THIS_MODULE); +} + +late_initcall(bpf_ksock_kfunc_init); diff --git a/net/core/filter.c b/net/core/filter.c index 16845987b244..61940e753552 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -56,6 +56,7 @@ #include <net/sock_reuseport.h> #include <net/busy_poll.h> #include <net/tcp.h> +#include <net/gre.h> #include <net/xfrm.h> #include <net/udp.h> #include <linux/bpf_trace.h> @@ -84,6 +85,9 @@ #include <linux/un.h> #include <net/xdp_sock_drv.h> #include <net/inet_dscp.h> +#include <linux/icmpv6.h> +#include <net/icmp.h> +#include <net/ip6_route.h> #include "dev.h" @@ -1744,7 +1748,7 @@ static const struct bpf_func_proto bpf_skb_store_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -1781,7 +1785,7 @@ static const struct bpf_func_proto bpf_skb_load_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len) @@ -1820,7 +1824,7 @@ static const struct bpf_func_proto bpf_flow_dissector_load_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb, @@ -1864,7 +1868,7 @@ static const struct bpf_func_proto bpf_skb_load_bytes_relative_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -2060,9 +2064,9 @@ static const struct bpf_func_proto bpf_csum_diff_proto = { .pkt_access = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg2_type = ARG_CONST_SIZE_OR_ZERO, + .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE_OR_ZERO, + .arg4_type = ARG_MEM_SIZE_OR_ZERO, .arg5_type = ARG_ANYTHING, }; @@ -2529,16 +2533,18 @@ int skb_do_redirect(struct sk_buff *skb) if (unlikely(!dev)) goto out_drop; if (flags & BPF_F_PEER) { - if (unlikely(!skb_at_tc_ingress(skb))) - goto out_drop; dev = skb_get_peer_dev(dev); if (unlikely(!dev || !(dev->flags & IFF_UP) || net_eq(net, dev_net(dev)))) goto out_drop; + skb_scrub_packet(skb, false); + if (flags & BPF_F_EGRESS) + return __bpf_redirect(skb, dev, 0); + if (unlikely(!skb_at_tc_ingress(skb))) + goto out_drop; skb->dev = dev; dev_sw_netstats_rx_add(dev, skb->len); - skb_scrub_packet(skb, false); return -EAGAIN; } return flags & BPF_F_NEIGH ? @@ -2577,11 +2583,11 @@ BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags) { struct bpf_redirect_info *ri; - if (unlikely(!bpf_net_ctx_get() || flags)) + if (unlikely(!bpf_net_ctx_get() || (flags & ~BPF_F_EGRESS))) return TC_ACT_SHOT; ri = bpf_net_ctx_get_ri(); - ri->flags = BPF_F_PEER; + ri->flags = BPF_F_PEER | flags; ri->tgt_index = ifindex; return TC_ACT_REDIRECT; @@ -2621,7 +2627,7 @@ static const struct bpf_func_proto bpf_redirect_neigh_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, .arg2_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, }; @@ -3567,14 +3573,27 @@ static u32 bpf_skb_net_base_len(const struct sk_buff *skb) #define BPF_F_ADJ_ROOM_DECAP_L3_MASK (BPF_F_ADJ_ROOM_DECAP_L3_IPV4 | \ BPF_F_ADJ_ROOM_DECAP_L3_IPV6) -#define BPF_F_ADJ_ROOM_MASK (BPF_F_ADJ_ROOM_FIXED_GSO | \ - BPF_F_ADJ_ROOM_ENCAP_L3_MASK | \ +#define BPF_F_ADJ_ROOM_DECAP_L4_MASK (BPF_F_ADJ_ROOM_DECAP_L4_UDP | \ + BPF_F_ADJ_ROOM_DECAP_L4_GRE) + +#define BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK (BPF_F_ADJ_ROOM_DECAP_IPXIP4 | \ + BPF_F_ADJ_ROOM_DECAP_IPXIP6) + +#define BPF_F_ADJ_ROOM_ENCAP_MASK (BPF_F_ADJ_ROOM_ENCAP_L3_MASK | \ BPF_F_ADJ_ROOM_ENCAP_L4_GRE | \ BPF_F_ADJ_ROOM_ENCAP_L4_UDP | \ BPF_F_ADJ_ROOM_ENCAP_L2_ETH | \ BPF_F_ADJ_ROOM_ENCAP_L2( \ - BPF_ADJ_ROOM_ENCAP_L2_MASK) | \ - BPF_F_ADJ_ROOM_DECAP_L3_MASK) + BPF_ADJ_ROOM_ENCAP_L2_MASK)) + +#define BPF_F_ADJ_ROOM_DECAP_MASK (BPF_F_ADJ_ROOM_DECAP_L3_MASK | \ + BPF_F_ADJ_ROOM_DECAP_L4_MASK | \ + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK) + +#define BPF_F_ADJ_ROOM_MASK (BPF_F_ADJ_ROOM_FIXED_GSO | \ + BPF_F_ADJ_ROOM_ENCAP_MASK | \ + BPF_F_ADJ_ROOM_DECAP_MASK | \ + BPF_F_ADJ_ROOM_NO_CSUM_RESET) static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff, u64 flags) @@ -3697,8 +3716,8 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff, bool decap = flags & BPF_F_ADJ_ROOM_DECAP_L3_MASK; int ret; - if (unlikely(flags & ~(BPF_F_ADJ_ROOM_FIXED_GSO | - BPF_F_ADJ_ROOM_DECAP_L3_MASK | + if (unlikely(flags & ~(BPF_F_ADJ_ROOM_DECAP_MASK | + BPF_F_ADJ_ROOM_FIXED_GSO | BPF_F_ADJ_ROOM_NO_CSUM_RESET))) return -EINVAL; @@ -3735,9 +3754,48 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff, if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO)) skb_increase_gso_size(shinfo, len_diff); + /* Selective GSO flag clearing based on decap type. + * Only clear the flags for the tunnel layer being removed. + */ + if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_UDP) && + (shinfo->gso_type & (SKB_GSO_UDP_TUNNEL | + SKB_GSO_UDP_TUNNEL_CSUM))) + shinfo->gso_type &= ~(SKB_GSO_UDP_TUNNEL | + SKB_GSO_UDP_TUNNEL_CSUM); + if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_GRE) && + (shinfo->gso_type & (SKB_GSO_GRE | SKB_GSO_GRE_CSUM))) + shinfo->gso_type &= ~(SKB_GSO_GRE | + SKB_GSO_GRE_CSUM); + if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP4) && + (shinfo->gso_type & SKB_GSO_IPXIP4)) + shinfo->gso_type &= ~SKB_GSO_IPXIP4; + if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP6) && + (shinfo->gso_type & SKB_GSO_IPXIP6)) + shinfo->gso_type &= ~SKB_GSO_IPXIP6; + + /* Clear encapsulation flag only when no tunnel GSO flags remain */ + if (flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK | + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) { + if (!(shinfo->gso_type & (SKB_GSO_UDP_TUNNEL | + SKB_GSO_UDP_TUNNEL_CSUM | + SKB_GSO_GRE | + SKB_GSO_GRE_CSUM | + SKB_GSO_IPXIP4 | + SKB_GSO_IPXIP6 | + SKB_GSO_ESP))) + if (skb->encapsulation) + skb->encapsulation = 0; + } + /* Header must be checked, and gso_segs recomputed. */ shinfo->gso_type |= SKB_GSO_DODGY; shinfo->gso_segs = 0; + } else { + /* For non-GSO packets, clear encapsulation if decap flags are set */ + if ((flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK | + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) && + skb->encapsulation) + skb->encapsulation = 0; } return 0; @@ -3797,8 +3855,7 @@ BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff, u32 off; int ret; - if (unlikely(flags & ~(BPF_F_ADJ_ROOM_MASK | - BPF_F_ADJ_ROOM_NO_CSUM_RESET))) + if (unlikely(flags & ~BPF_F_ADJ_ROOM_MASK)) return -EINVAL; if (unlikely(len_diff_abs > 0xfffU)) return -EFAULT; @@ -3817,20 +3874,53 @@ BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff, return -ENOTSUPP; } - if (flags & BPF_F_ADJ_ROOM_DECAP_L3_MASK) { + if (flags & BPF_F_ADJ_ROOM_DECAP_MASK) { + u32 len_decap_min = 0; + if (!shrink) return -EINVAL; - switch (flags & BPF_F_ADJ_ROOM_DECAP_L3_MASK) { - case BPF_F_ADJ_ROOM_DECAP_L3_IPV4: + /* Reject mutually exclusive decap flag pairs. */ + if ((flags & BPF_F_ADJ_ROOM_DECAP_L3_MASK) == + BPF_F_ADJ_ROOM_DECAP_L3_MASK) + return -EINVAL; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_MASK) == + BPF_F_ADJ_ROOM_DECAP_L4_MASK) + return -EINVAL; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK) == + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK) + return -EINVAL; + + /* Reject mutually exclusive decap tunnel type flags. */ + if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_MASK) && + (flags & BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) + return -EINVAL; + + if (flags & BPF_F_ADJ_ROOM_DECAP_L4_MASK) + len_decap_min += bpf_skb_net_base_len(skb); + + if (flags & BPF_F_ADJ_ROOM_DECAP_L4_UDP) + len_decap_min += sizeof(struct udphdr); + + if (flags & BPF_F_ADJ_ROOM_DECAP_L4_GRE) + len_decap_min += sizeof(struct gre_base_hdr); + + if (flags & BPF_F_ADJ_ROOM_DECAP_IPXIP4) + len_decap_min += sizeof(struct iphdr); + + if (flags & BPF_F_ADJ_ROOM_DECAP_IPXIP6) + len_decap_min += sizeof(struct ipv6hdr); + + if (len_diff_abs < len_decap_min) + return -EINVAL; + + if (flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV4) len_min = sizeof(struct iphdr); - break; - case BPF_F_ADJ_ROOM_DECAP_L3_IPV6: + + if (flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV6) len_min = sizeof(struct ipv6hdr); - break; - default: - return -EINVAL; - } } len_cur = skb->len - skb_network_offset(skb); @@ -4194,7 +4284,7 @@ static const struct bpf_func_proto bpf_xdp_load_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len) @@ -4226,7 +4316,7 @@ static const struct bpf_func_proto bpf_xdp_store_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len) @@ -4789,7 +4879,7 @@ static const struct bpf_func_proto bpf_skb_event_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BTF_ID_LIST_SINGLE(bpf_skb_output_btf_ids, struct, sk_buff) @@ -4803,7 +4893,7 @@ const struct bpf_func_proto bpf_skb_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; static unsigned short bpf_tunnel_key_af(u64 flags) @@ -4886,7 +4976,7 @@ static const struct bpf_func_proto bpf_skb_get_tunnel_key_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -4921,7 +5011,7 @@ static const struct bpf_func_proto bpf_skb_get_tunnel_opt_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; static struct metadata_dst __percpu *md_dst; @@ -5003,7 +5093,7 @@ static const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -5031,7 +5121,7 @@ static const struct bpf_func_proto bpf_skb_set_tunnel_opt_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; static const struct bpf_func_proto * @@ -5203,7 +5293,7 @@ static const struct bpf_func_proto bpf_xdp_event_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BTF_ID_LIST_SINGLE(bpf_xdp_output_btf_ids, struct, xdp_buff) @@ -5217,7 +5307,7 @@ const struct bpf_func_proto bpf_xdp_output_proto = { .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE_OR_ZERO, + .arg5_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_1(bpf_get_socket_cookie, struct sk_buff *, skb) @@ -5640,6 +5730,7 @@ static int sol_ip_sockopt(struct sock *sk, int optname, switch (optname) { case IP_TOS: + case IP_TRANSPARENT: if (*optlen != sizeof(int)) return -EINVAL; break; @@ -5666,6 +5757,7 @@ static int sol_ipv6_sockopt(struct sock *sk, int optname, switch (optname) { case IPV6_TCLASS: case IPV6_AUTOFLOWLABEL: + case IPV6_TRANSPARENT: if (*optlen != sizeof(int)) return -EINVAL; break; @@ -5764,7 +5856,7 @@ const struct bpf_func_proto bpf_sk_setsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sk_getsockopt, struct sock *, sk, int, level, @@ -5781,7 +5873,7 @@ const struct bpf_func_proto bpf_sk_getsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_UNINIT_MEM, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sk_setsockopt_nodelay, struct sock *, sk, int, level, @@ -5805,7 +5897,7 @@ const struct bpf_func_proto bpf_sk_setsockopt_nodelay_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_unlocked_sk_setsockopt, struct sock *, sk, int, level, @@ -5822,7 +5914,7 @@ const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_unlocked_sk_getsockopt, struct sock *, sk, int, level, @@ -5839,7 +5931,7 @@ const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_UNINIT_MEM, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sock_addr_setsockopt, struct bpf_sock_addr_kern *, ctx, @@ -5856,7 +5948,7 @@ static const struct bpf_func_proto bpf_sock_addr_setsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sock_addr_getsockopt, struct bpf_sock_addr_kern *, ctx, @@ -5873,7 +5965,7 @@ static const struct bpf_func_proto bpf_sock_addr_getsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_UNINIT_MEM, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; static int sk_bpf_set_get_bypass_prot_mem(struct sock *sk, @@ -5918,7 +6010,7 @@ static const struct bpf_func_proto bpf_sock_create_setsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sock_create_getsockopt, struct sock *, sk, int, level, @@ -5944,7 +6036,7 @@ static const struct bpf_func_proto bpf_sock_create_getsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_UNINIT_MEM, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_sock_ops_setsockopt, struct bpf_sock_ops_kern *, bpf_sock, @@ -5970,7 +6062,7 @@ static const struct bpf_func_proto bpf_sock_ops_setsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; static int bpf_sock_ops_get_syn(struct bpf_sock_ops_kern *bpf_sock, @@ -6080,7 +6172,7 @@ static const struct bpf_func_proto bpf_sock_ops_getsockopt_proto = { .arg2_type = ARG_ANYTHING, .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_UNINIT_MEM, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_2(bpf_sock_ops_cb_flags_set, struct bpf_sock_ops_kern *, bpf_sock, @@ -6147,7 +6239,7 @@ static const struct bpf_func_proto bpf_bind_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, }; #ifdef CONFIG_XFRM @@ -6200,27 +6292,66 @@ static const struct bpf_func_proto bpf_skb_get_xfrm_state_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; #endif #if IS_ENABLED(CONFIG_INET) || IS_ENABLED(CONFIG_IPV6) -static int bpf_fib_set_fwd_params(struct bpf_fib_lookup *params, u32 mtu) +static int bpf_fib_set_fwd_params(struct net_device *dev, + struct bpf_fib_lookup *params, + u32 flags, u32 mtu, u32 in_ifindex) { params->h_vlan_TCI = 0; params->h_vlan_proto = 0; + +#if IS_ENABLED(CONFIG_VLAN_8021Q) + if ((flags & BPF_FIB_LOOKUP_VLAN) && is_vlan_dev(dev)) { + struct net_device *real_dev = vlan_dev_priv(dev)->real_dev; + + if (!is_vlan_dev(real_dev) && + net_eq(dev_net(real_dev), dev_net(dev))) { + params->h_vlan_proto = vlan_dev_vlan_proto(dev); + params->h_vlan_TCI = htons(vlan_dev_vlan_id(dev)); + params->ifindex = real_dev->ifindex; + } else { + params->ifindex = in_ifindex; + return BPF_FIB_LKUP_RET_VLAN_FAILURE; + } + } +#endif + if (mtu) params->mtu_result = mtu; /* union with tot_len */ return 0; } + +static struct net_device *bpf_fib_vlan_input_dev(struct net_device *dev, + const struct bpf_fib_lookup *params) +{ + __be16 proto = params->h_vlan_proto; + struct net_device *vlan_dev; + u16 vid; + + if (proto != htons(ETH_P_8021Q) && proto != htons(ETH_P_8021AD)) + return ERR_PTR(-EINVAL); + + vid = ntohs(params->h_vlan_TCI) & VLAN_VID_MASK; + vlan_dev = __vlan_find_dev_deep_rcu(dev, proto, vid); + if (!vlan_dev || !(vlan_dev->flags & IFF_UP) || + !net_eq(dev_net(vlan_dev), dev_net(dev))) + return NULL; + + return vlan_dev; +} #endif #if IS_ENABLED(CONFIG_INET) static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, u32 flags, bool check_mtu) { + u32 in_ifindex = params->ifindex; struct neighbour *neigh = NULL; struct fib_nh_common *nhc; struct in_device *in_dev; @@ -6234,6 +6365,14 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, if (unlikely(!dev)) return -ENODEV; + if (flags & BPF_FIB_LOOKUP_VLAN_INPUT) { + dev = bpf_fib_vlan_input_dev(dev, params); + if (IS_ERR(dev)) + return PTR_ERR(dev); + if (!dev) + return BPF_FIB_LKUP_RET_NOT_FWDED; + } + /* verify forwarding is enabled on this interface */ in_dev = __in_dev_get_rcu(dev); if (unlikely(!in_dev || !IN_DEV_FORWARD(in_dev))) @@ -6243,7 +6382,11 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, fl4.flowi4_iif = 1; fl4.flowi4_oif = params->ifindex; } else { - fl4.flowi4_iif = params->ifindex; + /* + * dev->ifindex, not params->ifindex: VLAN_INPUT may have + * resolved dev to a subinterface above. + */ + fl4.flowi4_iif = dev->ifindex; fl4.flowi4_oif = 0; } fl4.flowi4_dscp = inet_dsfield_to_dscp(params->tos); @@ -6352,7 +6495,7 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, memcpy(params->smac, dev->dev_addr, ETH_ALEN); set_fwd_params: - return bpf_fib_set_fwd_params(params, mtu); + return bpf_fib_set_fwd_params(dev, params, flags, mtu, in_ifindex); } #endif @@ -6362,6 +6505,7 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, { struct in6_addr *src = (struct in6_addr *) params->ipv6_src; struct in6_addr *dst = (struct in6_addr *) params->ipv6_dst; + u32 in_ifindex = params->ifindex; struct fib6_result res = {}; struct neighbour *neigh; struct net_device *dev; @@ -6379,6 +6523,14 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, if (unlikely(!dev)) return -ENODEV; + if (flags & BPF_FIB_LOOKUP_VLAN_INPUT) { + dev = bpf_fib_vlan_input_dev(dev, params); + if (IS_ERR(dev)) + return PTR_ERR(dev); + if (!dev) + return BPF_FIB_LKUP_RET_NOT_FWDED; + } + idev = __in6_dev_get_safely(dev); if (unlikely(!idev || !READ_ONCE(idev->cnf.forwarding))) return BPF_FIB_LKUP_RET_FWD_DISABLED; @@ -6387,7 +6539,12 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, fl6.flowi6_iif = 1; oif = fl6.flowi6_oif = params->ifindex; } else { - oif = fl6.flowi6_iif = params->ifindex; + /* + * dev->ifindex, not params->ifindex: VLAN_INPUT may have + * resolved dev to a subinterface above. + */ + oif = dev->ifindex; + fl6.flowi6_iif = oif; fl6.flowi6_oif = 0; strict = RT6_LOOKUP_F_HAS_SADDR; } @@ -6491,13 +6648,26 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, memcpy(params->smac, dev->dev_addr, ETH_ALEN); set_fwd_params: - return bpf_fib_set_fwd_params(params, mtu); + return bpf_fib_set_fwd_params(dev, params, flags, mtu, in_ifindex); } #endif #define BPF_FIB_LOOKUP_MASK (BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT | \ BPF_FIB_LOOKUP_SKIP_NEIGH | BPF_FIB_LOOKUP_TBID | \ - BPF_FIB_LOOKUP_SRC | BPF_FIB_LOOKUP_MARK) + BPF_FIB_LOOKUP_SRC | BPF_FIB_LOOKUP_MARK | \ + BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_VLAN_INPUT) + +static bool bpf_fib_lookup_flags_ok(u32 flags) +{ + if (flags & ~BPF_FIB_LOOKUP_MASK) + return false; + + if ((flags & BPF_FIB_LOOKUP_VLAN_INPUT) && + (flags & (BPF_FIB_LOOKUP_TBID | BPF_FIB_LOOKUP_OUTPUT))) + return false; + + return true; +} BPF_CALL_4(bpf_xdp_fib_lookup, struct xdp_buff *, ctx, struct bpf_fib_lookup *, params, int, plen, u32, flags) @@ -6505,7 +6675,7 @@ BPF_CALL_4(bpf_xdp_fib_lookup, struct xdp_buff *, ctx, if (plen < sizeof(*params)) return -EINVAL; - if (flags & ~BPF_FIB_LOOKUP_MASK) + if (!bpf_fib_lookup_flags_ok(flags)) return -EINVAL; switch (params->family) { @@ -6529,7 +6699,7 @@ static const struct bpf_func_proto bpf_xdp_fib_lookup_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -6543,7 +6713,10 @@ BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb, if (plen < sizeof(*params)) return -EINVAL; - if (flags & ~BPF_FIB_LOOKUP_MASK) + if (!bpf_fib_lookup_flags_ok(flags)) + return -EINVAL; + + if (flags & BPF_FIB_LOOKUP_VLAN) return -EINVAL; if (params->tot_len) @@ -6586,7 +6759,7 @@ static const struct bpf_func_proto bpf_skb_fib_lookup_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -6784,7 +6957,7 @@ static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE + .arg4_type = ARG_MEM_SIZE }; static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = { @@ -6794,7 +6967,7 @@ static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE + .arg4_type = ARG_MEM_SIZE }; #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF) @@ -6838,7 +7011,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_store_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE + .arg4_type = ARG_MEM_SIZE }; static void bpf_update_srh_state(struct sk_buff *skb) @@ -6927,7 +7100,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_action_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg4_type = ARG_CONST_SIZE + .arg4_type = ARG_MEM_SIZE }; BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset, @@ -7080,6 +7253,28 @@ out: } static struct sock * +bpf_sk_lookup_full_sk(struct sock *sk) +{ + struct sock *sk2 = sk_to_full_sk(sk); + + /* + * sk_to_full_sk() may return sk->rsk_listener, make sure the original + * sk sock refcnt is decremented to prevent a request_sock leak. + */ + if (sk2 != sk) { + sock_gen_put(sk); + /* Ensure there is no need to bump sk2 refcnt. */ + if (unlikely(sk2 && !sock_flag(sk2, SOCK_RCU_FREE))) { + WARN_ONCE(1, "Found non-RCU, unreferenced socket!"); + return NULL; + } + sk = sk2; + } + + return sk; +} + +static struct sock * __bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len, struct net *caller_net, u32 ifindex, u8 proto, u64 netns_id, u64 flags, int sdif) @@ -7087,24 +7282,8 @@ __bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len, struct sock *sk = __bpf_skc_lookup(skb, tuple, len, caller_net, ifindex, proto, netns_id, flags, sdif); - - if (sk) { - struct sock *sk2 = sk_to_full_sk(sk); - - /* sk_to_full_sk() may return (sk)->rsk_listener, so make sure the original sk - * sock refcnt is decremented to prevent a request_sock leak. - */ - if (sk2 != sk) { - sock_gen_put(sk); - /* Ensure there is no need to bump sk2 refcnt */ - if (unlikely(sk2 && !sock_flag(sk2, SOCK_RCU_FREE))) { - WARN_ONCE(1, "Found non-RCU, unreferenced socket!"); - return NULL; - } - sk = sk2; - } - } - + if (sk) + sk = bpf_sk_lookup_full_sk(sk); return sk; } @@ -7133,24 +7312,8 @@ bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len, { struct sock *sk = bpf_skc_lookup(skb, tuple, len, proto, netns_id, flags); - - if (sk) { - struct sock *sk2 = sk_to_full_sk(sk); - - /* sk_to_full_sk() may return (sk)->rsk_listener, so make sure the original sk - * sock refcnt is decremented to prevent a request_sock leak. - */ - if (sk2 != sk) { - sock_gen_put(sk); - /* Ensure there is no need to bump sk2 refcnt */ - if (unlikely(sk2 && !sock_flag(sk2, SOCK_RCU_FREE))) { - WARN_ONCE(1, "Found non-RCU, unreferenced socket!"); - return NULL; - } - sk = sk2; - } - } - + if (sk) + sk = bpf_sk_lookup_full_sk(sk); return sk; } @@ -7168,7 +7331,7 @@ static const struct bpf_func_proto bpf_skc_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7187,7 +7350,7 @@ static const struct bpf_func_proto bpf_sk_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7206,7 +7369,7 @@ static const struct bpf_func_proto bpf_sk_lookup_udp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7230,7 +7393,7 @@ static const struct bpf_func_proto bpf_tc_skc_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7254,7 +7417,7 @@ static const struct bpf_func_proto bpf_tc_sk_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7278,7 +7441,7 @@ static const struct bpf_func_proto bpf_tc_sk_lookup_udp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7316,7 +7479,7 @@ static const struct bpf_func_proto bpf_xdp_sk_lookup_udp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7340,7 +7503,7 @@ static const struct bpf_func_proto bpf_xdp_skc_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7364,7 +7527,7 @@ static const struct bpf_func_proto bpf_xdp_sk_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7384,7 +7547,7 @@ static const struct bpf_func_proto bpf_sock_addr_skc_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCK_COMMON_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7403,7 +7566,7 @@ static const struct bpf_func_proto bpf_sock_addr_sk_lookup_tcp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7422,7 +7585,7 @@ static const struct bpf_func_proto bpf_sock_addr_sk_lookup_udp_proto = { .ret_type = RET_PTR_TO_SOCKET_OR_NULL, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, .arg5_type = ARG_ANYTHING, }; @@ -7742,9 +7905,9 @@ static const struct bpf_func_proto bpf_tcp_check_syncookie_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_5(bpf_tcp_gen_syncookie, struct sock *, sk, void *, iph, u32, iph_len, @@ -7811,9 +7974,9 @@ static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg5_type = ARG_CONST_SIZE, + .arg5_type = ARG_MEM_SIZE, }; BPF_CALL_3(bpf_sk_assign, struct sk_buff *, skb, struct sock *, sk, u64, flags) @@ -7967,7 +8130,7 @@ static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -8045,7 +8208,7 @@ static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE, + .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; @@ -8140,7 +8303,7 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv4_proto = { .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct iphdr), .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_3(bpf_tcp_raw_gen_syncookie_ipv6, struct ipv6hdr *, iph, @@ -8172,7 +8335,7 @@ static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = { .arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY, .arg1_size = sizeof(struct ipv6hdr), .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, - .arg3_type = ARG_CONST_SIZE_OR_ZERO, + .arg3_type = ARG_MEM_SIZE_OR_ZERO, }; BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv4, struct iphdr *, iph, @@ -8350,10 +8513,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_CGROUP_UNIX_SENDMSG: case BPF_CGROUP_INET4_GETPEERNAME: case BPF_CGROUP_INET6_GETPEERNAME: - case BPF_CGROUP_UNIX_GETPEERNAME: case BPF_CGROUP_INET4_GETSOCKNAME: case BPF_CGROUP_INET6_GETSOCKNAME: - case BPF_CGROUP_UNIX_GETSOCKNAME: return &bpf_sock_addr_setsockopt_proto; default: return NULL; @@ -8373,10 +8534,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_CGROUP_UNIX_SENDMSG: case BPF_CGROUP_INET4_GETPEERNAME: case BPF_CGROUP_INET6_GETPEERNAME: - case BPF_CGROUP_UNIX_GETPEERNAME: case BPF_CGROUP_INET4_GETSOCKNAME: case BPF_CGROUP_INET6_GETSOCKNAME: - case BPF_CGROUP_UNIX_GETSOCKNAME: return &bpf_sock_addr_getsockopt_proto; default: return NULL; @@ -11645,7 +11804,7 @@ static const struct bpf_func_proto sk_reuseport_load_bytes_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, }; BPF_CALL_5(sk_reuseport_load_bytes_relative, @@ -11663,7 +11822,7 @@ static const struct bpf_func_proto sk_reuseport_load_bytes_relative_proto = { .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_ANYTHING, .arg3_type = ARG_PTR_TO_UNINIT_MEM, - .arg4_type = ARG_CONST_SIZE, + .arg4_type = ARG_MEM_SIZE, .arg5_type = ARG_ANYTHING, }; @@ -12551,6 +12710,87 @@ __bpf_kfunc int bpf_xdp_pull_data(struct xdp_md *x, u32 len) return 0; } +/** + * bpf_icmp_send - Send an ICMP control message + * @skb_ctx: Packet that triggered the control message + * @type: ICMP type (only ICMP_DEST_UNREACH/ICMPV6_DEST_UNREACH supported) + * @code: ICMP code (0-15 except ICMP_FRAG_NEEDED for IPv4, 0-6 for IPv6) + * + * Sends an ICMP control message in response to the packet. The original packet + * is cloned before sending the ICMP message, so the BPF program can still let + * the packet pass if desired. + * + * Currently only ICMP_DEST_UNREACH (IPv4) and ICMPV6_DEST_UNREACH (IPv6) are + * supported. + * + * Return: 0 on success (send attempt), negative error code on failure: + * -EBUSY: Recursion detected + * -EPROTONOSUPPORT: Non-IP protocol + * -EOPNOTSUPP: Unsupported ICMP type + * -EINVAL: Invalid code parameter + * -ENETUNREACH: No usable route/dst for the ICMP reply + * -ENOMEM: Memory allocation failed + */ +__bpf_kfunc int bpf_icmp_send(struct __sk_buff *skb_ctx, int type, int code) +{ + struct sk_buff *skb = (struct sk_buff *)skb_ctx; + struct sock *sk; + + sk = skb_to_full_sk(skb); + if (sk && sk->sk_kern_sock && + (sk->sk_protocol == IPPROTO_ICMP || sk->sk_protocol == IPPROTO_ICMPV6)) + return -EBUSY; + + if (!skb_valid_dst(skb)) + return -ENETUNREACH; + + switch (skb->protocol) { +#if IS_ENABLED(CONFIG_INET) + case htons(ETH_P_IP): { + struct sk_buff *nskb; + + if (type != ICMP_DEST_UNREACH) + return -EOPNOTSUPP; + if (code < 0 || code > NR_ICMP_UNREACH || + code == ICMP_FRAG_NEEDED) /* needs a valid next-hop MTU */ + return -EINVAL; + + nskb = skb_clone(skb, GFP_ATOMIC); + if (!nskb) + return -ENOMEM; + + memset(IPCB(nskb), 0, sizeof(*IPCB(nskb))); + icmp_send(nskb, type, code, 0); + consume_skb(nskb); + break; + } +#endif +#if IS_ENABLED(CONFIG_IPV6) + case htons(ETH_P_IPV6): { + struct sk_buff *nskb; + + if (type != ICMPV6_DEST_UNREACH) + return -EOPNOTSUPP; + if (code < 0 || code > ICMPV6_REJECT_ROUTE) + return -EINVAL; + + nskb = skb_clone(skb, GFP_ATOMIC); + if (!nskb) + return -ENOMEM; + + memset(IP6CB(nskb), 0, sizeof(*IP6CB(nskb))); + icmpv6_send(nskb, type, code, 0); + consume_skb(nskb); + break; + } +#endif + default: + return -EPROTONOSUPPORT; + } + + return 0; +} + __bpf_kfunc_end_defs(); int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags, @@ -12593,6 +12833,10 @@ BTF_KFUNCS_START(bpf_kfunc_check_set_sock_ops) BTF_ID_FLAGS(func, bpf_sock_ops_enable_tx_tstamp) BTF_KFUNCS_END(bpf_kfunc_check_set_sock_ops) +BTF_KFUNCS_START(bpf_kfunc_check_set_icmp_send) +BTF_ID_FLAGS(func, bpf_icmp_send) +BTF_KFUNCS_END(bpf_kfunc_check_set_icmp_send) + static const struct btf_kfunc_id_set bpf_kfunc_set_skb = { .owner = THIS_MODULE, .set = &bpf_kfunc_check_set_skb, @@ -12623,6 +12867,11 @@ static const struct btf_kfunc_id_set bpf_kfunc_set_sock_ops = { .set = &bpf_kfunc_check_set_sock_ops, }; +static const struct btf_kfunc_id_set bpf_kfunc_set_icmp_send = { + .owner = THIS_MODULE, + .set = &bpf_kfunc_check_set_icmp_send, +}; + static int __init bpf_kfunc_init(void) { int ret; @@ -12644,6 +12893,7 @@ static int __init bpf_kfunc_init(void) ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SOCK_ADDR, &bpf_kfunc_set_sock_addr); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_tcp_reqsk); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &bpf_kfunc_set_icmp_send); return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SOCK_OPS, &bpf_kfunc_set_sock_ops); } late_initcall(bpf_kfunc_init); diff --git a/net/core/lwt_bpf.c b/net/core/lwt_bpf.c index 652952d416f2..da49364ec63d 100644 --- a/net/core/lwt_bpf.c +++ b/net/core/lwt_bpf.c @@ -167,10 +167,10 @@ static int bpf_output(struct net *net, struct sock *sk, struct sk_buff *skb) return dst->lwtstate->orig_output(net, sk, skb); } -static int xmit_check_hhlen(struct sk_buff *skb, int hh_len) +static int xmit_check_headroom(struct sk_buff *skb, int hroom) { - if (skb_headroom(skb) < hh_len) { - int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); + if (skb_headroom(skb) < hroom) { + int nhead = hroom - skb_headroom(skb); if (pskb_expand_head(skb, nhead, 0, GFP_ATOMIC)) return -ENOMEM; @@ -282,7 +282,7 @@ static int bpf_xmit(struct sk_buff *skb) bpf = bpf_lwt_lwtunnel(dst->lwtstate); if (bpf->xmit.prog) { - int hh_len = dst->dev->hard_header_len; + int hroom = LL_RESERVED_SPACE(dst->dev); __be16 proto = skb->protocol; int ret; @@ -298,9 +298,12 @@ static int bpf_xmit(struct sk_buff *skb) return -EINVAL; } /* If the header was expanded, headroom might be too - * small for L2 header to come, expand as needed. + * small for the L2 header to come, expand as needed. + * neigh_hh_output() copies the cached header in + * HH_DATA_MOD aligned chunks, so match the reservation + * made before LWT xmit. */ - ret = xmit_check_hhlen(skb, hh_len); + ret = xmit_check_headroom(skb, hroom); if (unlikely(ret)) return ret; diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 9efbd8ca7db8..ca49bc7f8687 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -392,8 +392,8 @@ static void *sock_map_lookup(struct bpf_map *map, void *key) sk = __sock_map_lookup_elem(map, *(u32 *)key); if (!sk) return NULL; - if (sk_is_refcounted(sk) && !refcount_inc_not_zero(&sk->sk_refcnt)) - return NULL; + if (sk_is_refcounted(sk)) + sock_hold(sk); return sk; } @@ -1218,8 +1218,8 @@ static void *sock_hash_lookup(struct bpf_map *map, void *key) sk = __sock_hash_lookup_elem(map, key); if (!sk) return NULL; - if (sk_is_refcounted(sk) && !refcount_inc_not_zero(&sk->sk_refcnt)) - return NULL; + if (sk_is_refcounted(sk)) + sock_hold(sk); return sk; } diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 7f413f509d7d..190c7af4cf92 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -2931,8 +2931,8 @@ static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter, { union bpf_tcp_iter_batch_item *new_batch; - new_batch = kvmalloc(sizeof(*new_batch) * new_batch_sz, - flags | __GFP_NOWARN); + new_batch = kvmalloc_array(new_batch_sz, sizeof(*new_batch), + flags | __GFP_NOWARN); if (!new_batch) return -ENOMEM; diff --git a/net/socket.c b/net/socket.c index b0256cd222f8..c05d86e63abf 100644 --- a/net/socket.c +++ b/net/socket.c @@ -2128,6 +2128,20 @@ SYSCALL_DEFINE3(accept, int, fd, struct sockaddr __user *, upeer_sockaddr, return __sys_accept4(fd, upeer_sockaddr, upeer_addrlen, 0); } +int connect_socket(struct socket *sock, struct sockaddr_storage *address, + int addrlen, int flags) +{ + int err; + + err = security_socket_connect(sock, (struct sockaddr *)address, + addrlen); + if (err) + return err; + + return READ_ONCE(sock->ops)->connect(sock, (struct sockaddr_unsized *)address, + addrlen, flags); +} + /* * Attempt to connect to a socket with the server address. The address * is in user space so we verify it is OK and move it to kernel space. @@ -2144,23 +2158,13 @@ int __sys_connect_file(struct file *file, struct sockaddr_storage *address, int addrlen, int file_flags) { struct socket *sock; - int err; sock = sock_from_file(file); - if (!sock) { - err = -ENOTSOCK; - goto out; - } - - err = - security_socket_connect(sock, (struct sockaddr *)address, addrlen); - if (err) - goto out; + if (!sock) + return -ENOTSOCK; - err = READ_ONCE(sock->ops)->connect(sock, (struct sockaddr_unsized *)address, - addrlen, sock->file->f_flags | file_flags); -out: - return err; + return connect_socket(sock, address, addrlen, + sock->file->f_flags | file_flags); } int __sys_connect(int fd, struct sockaddr __user *uservaddr, int addrlen) diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index 7794740fa80f..270a37da293f 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -3655,8 +3655,8 @@ static int bpf_iter_unix_realloc_batch(struct bpf_unix_iter_state *iter, { struct sock **new_batch; - new_batch = kvmalloc(sizeof(*new_batch) * new_batch_sz, - GFP_USER | __GFP_NOWARN); + new_batch = kvmalloc_array(new_batch_sz, sizeof(*new_batch), + GFP_USER | __GFP_NOWARN); if (!new_batch) return -ENOMEM; diff --git a/scripts/Makefile.btf b/scripts/Makefile.btf index e66e13e79653..a1812985a61a 100644 --- a/scripts/Makefile.btf +++ b/scripts/Makefile.btf @@ -14,9 +14,7 @@ pahole-flags-$(call test-ge, $(pahole-ver), 125) += --skip_encoding_btf_inconsis else # Switch to using --btf_features for v1.26 and later. -pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs - -pahole-flags-$(call test-ge, $(pahole-ver), 130) += --btf_features=attributes +pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func pahole-flags-$(call test-ge, $(pahole-ver), 131) += --btf_features=layout diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 271a7dc77273..b0f7168e7943 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -99,7 +99,7 @@ endif HOST_LDFLAGS := $(LDFLAGS) # Remove warnings for libbpf bootstrap build -LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat -Wformat-signedness,$(HOST_CFLAGS)) +LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat%,$(HOST_CFLAGS)) INSTALL ?= install RM ?= rm -f diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index 6ef908adf3a4..c9589026da8d 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -179,7 +179,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, case BTF_KIND_STRUCT: case BTF_KIND_UNION: { const struct btf_member *m = (const void *)(t + 1); - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); if (json_output) { jsonw_uint_field(w, "size", t->size); @@ -193,7 +193,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, const char *name = btf_str(btf, m->name_off); __u32 bit_off, bit_sz; - if (BTF_INFO_KFLAG(t->info)) { + if (btf_kflag(t)) { bit_off = BTF_MEMBER_BIT_OFFSET(m->offset); bit_sz = BTF_MEMBER_BITFIELD_SIZE(m->offset); } else { @@ -224,7 +224,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, } case BTF_KIND_ENUM: { const struct btf_enum *v = (const void *)(t + 1); - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); const char *encoding; encoding = btf_kflag(t) ? "SIGNED" : "UNSIGNED"; @@ -300,8 +300,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, break; } case BTF_KIND_FWD: { - const char *fwd_kind = BTF_INFO_KFLAG(t->info) ? "union" - : "struct"; + const char *fwd_kind = btf_kflag(t) ? "union" : "struct"; if (json_output) jsonw_string_field(w, "fwd_kind", fwd_kind); @@ -322,7 +321,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, } case BTF_KIND_FUNC_PROTO: { const struct btf_param *p = (const void *)(t + 1); - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); if (json_output) { jsonw_uint_field(w, "ret_type_id", t->type); @@ -365,7 +364,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, case BTF_KIND_DATASEC: { const struct btf_var_secinfo *v = (const void *)(t + 1); const struct btf_type *vt; - __u32 i, vlen = BTF_INFO_VLEN(t->info); + __u32 i, vlen = btf_vlen(t); if (json_output) { jsonw_uint_field(w, "size", t->size); diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index 9dc8425b1789..e4075824343f 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -476,8 +476,8 @@ static int btf_dumper_struct(const struct btf_dumper *d, __u32 type_id, if (!t) return -EINVAL; - kind_flag = BTF_INFO_KFLAG(t->info); - vlen = BTF_INFO_VLEN(t->info); + kind_flag = btf_kflag(t); + vlen = btf_vlen(t); jsonw_start_object(d->jw); m = (struct btf_member *)(t + 1); @@ -535,7 +535,7 @@ static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id, if (!t) return -EINVAL; - vlen = BTF_INFO_VLEN(t->info); + vlen = btf_vlen(t); vsi = (struct btf_var_secinfo *)(t + 1); jsonw_start_object(d->jw); @@ -557,7 +557,7 @@ static int btf_dumper_do_type(const struct btf_dumper *d, __u32 type_id, { const struct btf_type *t = btf__type_by_id(d->btf, type_id); - switch (BTF_INFO_KIND(t->info)) { + switch (btf_kind(t)) { case BTF_KIND_INT: return btf_dumper_int(t, bit_offset, data, d->jw, d->is_plain_text); @@ -631,7 +631,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, t = btf__type_by_id(btf, type_id); - switch (BTF_INFO_KIND(t->info)) { + switch (btf_kind(t)) { case BTF_KIND_INT: case BTF_KIND_TYPEDEF: case BTF_KIND_FLOAT: @@ -661,7 +661,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, break; case BTF_KIND_FWD: BTF_PRINT_ARG("%s %s ", - BTF_INFO_KFLAG(t->info) ? "union" : "struct", + btf_kflag(t) ? "union" : "struct", btf__name_by_offset(btf, t->name_off)); break; case BTF_KIND_VOLATILE: @@ -718,7 +718,7 @@ static int btf_dump_func(const struct btf *btf, char *func_sig, BTF_PRINT_ARG("%s(", btf__name_by_offset(btf, func->name_off)); else BTF_PRINT_ARG("("); - vlen = BTF_INFO_VLEN(func_proto->info); + vlen = btf_vlen(func_proto); for (i = 0; i < vlen; i++) { struct btf_param *arg = &((struct btf_param *)(func_proto + 1))[i]; diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index 8bfcff9e2f63..ef366ccc9650 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -832,6 +832,8 @@ static int prog_fd_by_nametag(void *nametag, int **fds, bool tag) fd = bpf_prog_get_fd_by_id(id); if (fd < 0) { + if (errno == ENOENT) + continue; p_err("can't get prog by id (%u): %s", id, strerror(errno)); goto err_close_fds; @@ -996,6 +998,8 @@ static int map_fd_by_name(char *name, int **fds, opts_ro.open_flags = BPF_F_RDONLY; fd = bpf_map_get_fd_by_id_opts(id, &opts_ro); if (fd < 0) { + if (errno == ENOENT) + continue; p_err("can't get map by id (%u): %s", id, strerror(errno)); goto err_close_fds; diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c index 6ae7262ebe0c..a50540ef6521 100644 --- a/tools/bpf/bpftool/gen.c +++ b/tools/bpf/bpftool/gen.c @@ -101,6 +101,12 @@ static bool get_map_ident(const struct bpf_map *map, char *buf, size_t buf_sz) return true; } + if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) { + snprintf(buf, buf_sz, "%s", name + 1); + sanitize_identifier(buf); + return true; + } + for (i = 0, n = ARRAY_SIZE(sfxs); i < n; i++) { const char *sfx = sfxs[i], *p; @@ -117,7 +123,7 @@ static bool get_map_ident(const struct bpf_map *map, char *buf, size_t buf_sz) static bool get_datasec_ident(const char *sec_name, char *buf, size_t buf_sz) { - static const char *pfxs[] = { ".data", ".rodata", ".bss", ".kconfig" }; + static const char *pfxs[] = { ".data", ".rodata", ".bss", ".percpu", ".kconfig" }; int i, n; /* recognize hard coded LLVM section name */ @@ -254,7 +260,7 @@ static const struct btf_type *find_type_for_map(struct btf *btf, const char *map return NULL; } -static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz) +static bool is_skel_data(const struct bpf_map *map, char *buf, size_t sz) { size_t tmp_sz; @@ -263,13 +269,24 @@ static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz) return true; } - if (!bpf_map__is_internal(map) || !(bpf_map__map_flags(map) & BPF_F_MMAPABLE)) + if (!bpf_map__is_internal(map)) return false; if (!get_map_ident(map, buf, sz)) return false; - return true; + if (bpf_map__map_flags(map) & BPF_F_MMAPABLE) + return true; + + if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) + return bpf_map__btf_value_type_id(map) != 0; + + return false; +} + +static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz) +{ + return is_skel_data(map, buf, sz) && bpf_map__type(map) != BPF_MAP_TYPE_PERCPU_ARRAY; } static int codegen_datasecs(struct bpf_object *obj, const char *obj_name) @@ -287,7 +304,7 @@ static int codegen_datasecs(struct bpf_object *obj, const char *obj_name) bpf_object__for_each_map(map, obj) { /* only generate definitions for memory-mapped internal maps */ - if (!is_mmapable_map(map, map_ident, sizeof(map_ident))) + if (!is_skel_data(map, map_ident, sizeof(map_ident))) continue; sec = find_type_for_map(btf, map_ident); @@ -517,7 +534,7 @@ static void codegen_asserts(struct bpf_object *obj, const char *obj_name) ", obj_name); bpf_object__for_each_map(map, obj) { - if (!is_mmapable_map(map, map_ident, sizeof(map_ident))) + if (!is_skel_data(map, map_ident, sizeof(map_ident))) continue; sec = find_type_for_map(btf, map_ident); @@ -668,8 +685,7 @@ static void codegen_destroy(struct bpf_object *obj, const char *obj_name) bpf_object__for_each_map(map, obj) { if (!get_map_ident(map, ident, sizeof(ident))) continue; - if (bpf_map__is_internal(map) && - (bpf_map__map_flags(map) & BPF_F_MMAPABLE)) + if (is_skel_data(map, ident, sizeof(ident))) printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zu);\n", ident, bpf_map_mmap_sz(map)); codegen("\ @@ -741,7 +757,7 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h const void *mmap_data = NULL; size_t mmap_size = 0; - if (!is_mmapable_map(map, ident, sizeof(ident))) + if (!is_skel_data(map, ident, sizeof(ident))) continue; codegen("\ @@ -793,6 +809,8 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h if (sign_progs) { sopts.insns = opts.insns; sopts.insns_sz = opts.insns_sz; + sopts.data = opts.data; + sopts.data_sz = opts.data_sz; sopts.excl_prog_hash = prog_sha; sopts.excl_prog_hash_sz = sizeof(prog_sha); sopts.signature = sig_buf; @@ -847,9 +865,23 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h bpf_object__for_each_map(map, obj) { const char *mmap_flags; - if (!is_mmapable_map(map, ident, sizeof(ident))) + if (!is_skel_data(map, ident, sizeof(ident))) continue; + if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) { + codegen("\ + \n\ + err = skel_protect_map_data(skel->%1$s, &skel->maps.%1$s.initial_value, %2$zd);\n\ + if (err) \n\ + return err; \n\ + #ifdef __KERNEL__ \n\ + skel->%1$s = NULL; \n\ + #endif \n\ + ", + ident, bpf_map_mmap_sz(map)); + continue; + } + if (bpf_map__map_flags(map) & BPF_F_RDONLY_PROG) mmap_flags = "PROT_READ"; else @@ -953,8 +985,7 @@ codegen_maps_skeleton(struct bpf_object *obj, size_t map_cnt, bool mmaped, bool map->map = &obj->maps.%s; \n\ ", i, bpf_map__name(map), ident); - /* memory-mapped internal maps */ - if (mmaped && is_mmapable_map(map, ident, sizeof(ident))) { + if (mmaped && is_skel_data(map, ident, sizeof(ident))) { printf("\tmap->mmaped = (void **)&obj->%s;\n", ident); } diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index bdcd717b0348..088d1d206065 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c @@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val) return (__u64 *) u64_to_ptr(val); } +static __u32 *u64_to_u32_arr(__u64 val) +{ + return (__u32 *)u64_to_ptr(val); +} + +static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled) +{ + struct kernel_sym *sym; + + if (!addr) + return NULL; + + sym = kernel_syms_search(&dd, addr); + if (!sym && is_ibt_enabled && addr >= 4) + sym = kernel_syms_search(&dd, addr - 4); + + return sym; +} + static void show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) { @@ -404,6 +423,52 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr) } static void +show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr) +{ + bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol; + __u64 *addrs, *cookies; + __u32 i, *ids; + + if (!dd.sym_count) + kernel_syms_load(&dd); + show_symbol = !!dd.sym_count; + + show_link_attach_type_json(info->tracing_multi.attach_type, wtr); + jsonw_uint_field(wtr, "func_cnt", info->tracing_multi.count); + jsonw_uint_field(wtr, "btf_obj_id", info->tracing_multi.btf_obj_id); + jsonw_name(wtr, "funcs"); + + jsonw_start_array(wtr); + + ids = u64_to_u32_arr(info->tracing_multi.ids); + addrs = u64_to_arr(info->tracing_multi.addrs); + cookies = u64_to_arr(info->tracing_multi.cookies); + + for (i = 0; i < info->tracing_multi.count; i++) { + struct kernel_sym *sym; + __u64 addr = addrs[i]; + + sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL; + + jsonw_start_object(wtr); + jsonw_uint_field(wtr, "id", ids[i]); + jsonw_uint_field(wtr, "addr", addr); + if (sym) { + jsonw_string_field(wtr, "func", sym->name); + if (sym->module[0] == '\0') { + jsonw_name(wtr, "module"); + jsonw_null(wtr); + } else { + jsonw_string_field(wtr, "module", sym->module); + } + } + jsonw_uint_field(wtr, "cookie", cookies[i]); + jsonw_end_object(wtr); + } + jsonw_end_array(wtr); +} + +static void show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr) { jsonw_bool_field(wtr, "retprobe", info->perf_event.type == BPF_PERF_EVENT_KRETPROBE); @@ -589,6 +654,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info) case BPF_LINK_TYPE_UPROBE_MULTI: show_uprobe_multi_json(info, json_wtr); break; + case BPF_LINK_TYPE_TRACING_MULTI: + show_tracing_multi_json(info, json_wtr); + break; case BPF_LINK_TYPE_PERF_EVENT: switch (info->perf_event.type) { case BPF_PERF_EVENT_EVENT: @@ -833,6 +901,46 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info) } } +static void show_tracing_multi_plain(struct bpf_link_info *info) +{ + bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol; + __u64 *addrs, *cookies; + __u32 i, *ids; + + if (!info->tracing_multi.count) + return; + + if (!dd.sym_count) + kernel_syms_load(&dd); + show_symbol = !!dd.sym_count; + + printf("\n\t"); + show_link_attach_type_plain(info->tracing_multi.attach_type); + printf("btf_obj_id %u ", info->tracing_multi.btf_obj_id); + printf("count %u ", info->tracing_multi.count); + + printf("\n\t%-16s %-16s %-16s %s", + "btf_id", "addr", "cookie", "func [module]"); + + ids = u64_to_u32_arr(info->tracing_multi.ids); + addrs = u64_to_arr(info->tracing_multi.addrs); + cookies = u64_to_arr(info->tracing_multi.cookies); + + for (i = 0; i < info->tracing_multi.count; i++) { + __u64 addr = addrs[i]; + struct kernel_sym *sym; + + sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL; + + printf("\n\t%-16u %016llx %-16llu", ids[i], addr, cookies[i]); + if (sym) { + printf(" %s", sym->name); + if (sym->module[0] != '\0') + printf(" [%s]", sym->module); + } + } +} + static void show_perf_event_kprobe_plain(struct bpf_link_info *info) { const char *buf; @@ -989,6 +1097,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) case BPF_LINK_TYPE_UPROBE_MULTI: show_uprobe_multi_plain(info); break; + case BPF_LINK_TYPE_TRACING_MULTI: + show_tracing_multi_plain(info); + break; case BPF_LINK_TYPE_PERF_EVENT: switch (info->perf_event.type) { case BPF_PERF_EVENT_EVENT: @@ -1029,6 +1140,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) static int do_show_link(int fd) { __u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL; + __u32 *ids = NULL; struct bpf_link_info info; __u32 len = sizeof(info); char path_buf[PATH_MAX]; @@ -1114,6 +1226,26 @@ again: goto again; } } + if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) { + count = info.tracing_multi.count; + if (count) { + ids = calloc(count, sizeof(__u32)); + addrs = calloc(count, sizeof(__u64)); + cookies = calloc(count, sizeof(__u64)); + if (!ids || !addrs || !cookies) { + p_err("mem alloc failed"); + close(fd); + free(cookies); + free(addrs); + free(ids); + return -ENOMEM; + } + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.addrs = ptr_to_u64(addrs); + info.tracing_multi.cookies = ptr_to_u64(cookies); + goto again; + } + } if (info.type == BPF_LINK_TYPE_PERF_EVENT) { switch (info.perf_event.type) { case BPF_PERF_EVENT_TRACEPOINT: @@ -1153,6 +1285,7 @@ again: free(cookies); free(offsets); free(addrs); + free(ids); close(fd); return 0; } diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 71a45d96617e..684a8fb72414 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -659,8 +659,6 @@ static int do_show_subset(int argc, char **argv) show_map_close_json(fds[i], &info); else show_map_close_plain(fds[i], &info); - - close(fds[i]); } if (json_output && nb_fds > 1) jsonw_end_array(json_wtr); /* root array */ @@ -790,6 +788,12 @@ static int maps_have_btf(int *fds, int nb_fds) static struct btf *btf_vmlinux; +static void free_btf_vmlinux(void) +{ + btf__free(btf_vmlinux); + btf_vmlinux = NULL; +} + static int get_map_kv_btf(const struct bpf_map_info *info, struct btf **btf) { int err = 0; @@ -889,7 +893,6 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, exit_free: free(key); free(value); - close(fd); free_map_kv_btf(btf); return err; @@ -938,6 +941,7 @@ static int do_dump(int argc, char **argv) for (i = 0; i < nb_fds; i++) { if (bpf_map_get_info_by_fd(fds[i], &info, &len)) { p_err("can't get map info: %s", strerror(errno)); + err = -1; break; } err = map_dump(fds[i], &info, wtr, nb_fds > 1); @@ -958,7 +962,7 @@ exit_close: close(fds[i]); exit_free: free(fds); - btf__free(btf_vmlinux); + free_btf_vmlinux(); return err; } @@ -1049,7 +1053,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, btf_wtr = get_btf_writer(); if (!btf_wtr) { p_info("failed to create json writer for btf. falling back to plain output"); - btf__free(btf); + free_map_kv_btf(btf); btf = NULL; print_entry_plain(info, key, value); } else { @@ -1065,7 +1069,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, } else { print_entry_plain(info, key, value); } - btf__free(btf); + free_map_kv_btf(btf); } static int do_lookup(int argc, char **argv) diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c index f9b742f4bb10..88726a6db6d0 100644 --- a/tools/bpf/bpftool/sign.c +++ b/tools/bpf/bpftool/sign.c @@ -135,9 +135,21 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) CMS_ContentInfo *cms = NULL; long actual_sig_len = 0; X509 *x509 = NULL; + void *data = NULL; + size_t data_sz; int err = 0; - bd_in = BIO_new_mem_buf(opts->insns, opts->insns_sz); + data_sz = (size_t)opts->insns_sz + opts->data_sz; + data = malloc(data_sz); + if (!data) { + err = -ENOMEM; + goto cleanup; + } + memcpy(data, opts->insns, opts->insns_sz); + if (opts->data_sz) + memcpy((char *)data + opts->insns_sz, opts->data, opts->data_sz); + + bd_in = BIO_new_mem_buf(data, data_sz); if (!bd_in) { err = -ENOMEM; goto cleanup; @@ -175,10 +187,13 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } - EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash, - &opts->excl_prog_hash_sz, EVP_sha256(), NULL); + if (EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash, + &opts->excl_prog_hash_sz, EVP_sha256(), NULL) != 1) { + err = -EIO; + goto cleanup; + } - bd_out = BIO_new(BIO_s_mem()); + bd_out = BIO_new(BIO_s_mem()); if (!bd_out) { err = -ENOMEM; goto cleanup; @@ -212,6 +227,7 @@ cleanup: X509_free(x509); EVP_PKEY_free(private_key); BIO_free(bd_in); + free(data); DISPLAY_OSSL_ERR(err < 0); return err; } diff --git a/tools/bpf/bpftool/struct_ops.c b/tools/bpf/bpftool/struct_ops.c index aa43dead249c..835e5e561f7f 100644 --- a/tools/bpf/bpftool/struct_ops.c +++ b/tools/bpf/bpftool/struct_ops.c @@ -643,6 +643,10 @@ int do_struct_ops(int argc, char **argv) err = cmd_select(cmds, argc, argv, do_help); btf__free(btf_vmlinux); + btf_vmlinux = NULL; + map_info_type = NULL; + map_info_alloc_len = 0; + map_info_type_id = 0; return err; } diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c index 5e7cb8b36fef..5579173a61e3 100644 --- a/tools/bpf/bpftool/xlated_dumper.c +++ b/tools/bpf/bpftool/xlated_dumper.c @@ -107,14 +107,7 @@ print_insn_for_graph(void *private_data, const char *fmt, ...) p = buf; while (*p != '\0') { - if (*p == '\n') { - memmove(p + 3, p, strlen(buf) + 1 - (p - buf)); - /* Align each instruction dump row left. */ - *p++ = '\\'; - *p++ = 'l'; - /* Output multiline concatenation. */ - *p++ = '\\'; - } else if (*p == '<' || *p == '>' || *p == '|' || *p == '&') { + if (*p == '<' || *p == '>' || *p == '|' || *p == '&') { memmove(p + 1, p, strlen(buf) + 1 - (p - buf)); /* Escape special character. */ *p++ = '\\'; @@ -129,16 +122,10 @@ print_insn_for_graph(void *private_data, const char *fmt, ...) static void __printf(2, 3) print_insn_json(void *private_data, const char *fmt, ...) { - unsigned int l = strlen(fmt); - char chomped_fmt[l]; va_list args; va_start(args, fmt); - if (l > 0) { - strncpy(chomped_fmt, fmt, l - 1); - chomped_fmt[l - 1] = '\0'; - } - jsonw_vprintf_enquote(json_wtr, chomped_fmt, args); + jsonw_vprintf_enquote(json_wtr, fmt, args); va_end(args); } @@ -351,6 +338,7 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, printf("%4u: ", i); print_bpf_insn(&cbs, insn + i, true); + printf("\n"); if (opcodes) { printf(" "); @@ -417,6 +405,7 @@ void dump_xlated_for_graph(struct dump_data *dd, void *buf_start, void *buf_end, printf("%u: ", insn_off); print_bpf_insn(&cbs, cur, true); + printf("\\l\\\n"); if (opcodes) { printf("\\ \\ \\ \\ "); diff --git a/tools/bpf/resolve_btfids/Makefile b/tools/bpf/resolve_btfids/Makefile index 7672208f65e4..bb0463b380af 100644 --- a/tools/bpf/resolve_btfids/Makefile +++ b/tools/bpf/resolve_btfids/Makefile @@ -40,7 +40,6 @@ LIBBPF_DESTDIR := $(LIBBPF_OUT) LIBBPF_INCLUDE := $(LIBBPF_DESTDIR)include SUBCMD_DESTDIR := $(SUBCMD_OUT) -SUBCMD_INCLUDE := $(SUBCMD_DESTDIR)include BINARY := $(OUTPUT)/resolve_btfids BINARY_IN := $(BINARY)-in.o @@ -56,7 +55,7 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT): $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \ DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \ - $(abspath $@) install_headers + $(abspath $@) $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT) $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT) \ @@ -77,7 +76,7 @@ HOSTCFLAGS_resolve_btfids += -g \ -I$(srctree)/tools/include \ -I$(srctree)/tools/include/uapi \ -I$(LIBBPF_INCLUDE) \ - -I$(SUBCMD_INCLUDE) \ + -I$(srctree)/tools/lib \ $(LIBELF_FLAGS) \ -Wall -Werror diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c index f8a91fa7584f..37d7e7224207 100644 --- a/tools/bpf/resolve_btfids/main.c +++ b/tools/bpf/resolve_btfids/main.c @@ -58,6 +58,17 @@ * __BTF_ID__func__vfs_fallocate__5: * .zero 4 * .word (1 << 3) | (1 << 1) | (1 << 2) + * + * In addition to resolving BTF IDs, resolve_btfids performs kernel-specific + * BTF-to-BTF transformations for kfuncs found in BTF_SET8_KFUNCS sets. For + * each such kfunc it: + * + * - emits a "bpf_kfunc" decl tag, and "bpf_fastcall" when KF_FASTCALL is set; + * - wraps the return value and/or arguments that use arena pointers + * with the "address_space(1)" type attribute; + * - rewrites the prototype of KF_IMPLICIT_ARGS kfuncs. + * + * These kfunc annotations were historically produced by pahole. */ #define _GNU_SOURCE @@ -119,6 +130,11 @@ struct btf_id { Elf64_Addr addr[ADDR_CNT]; }; +struct addr_sym { + Elf64_Addr addr; + const char *name; +}; + struct object { const char *path; const char *btf_path; @@ -150,12 +166,27 @@ struct object { int nr_structs; int nr_unions; int nr_typedefs; + + struct addr_sym *addr_syms; + u32 addr_syms_cnt; + u32 addr_syms_cap; }; +#define DECL_TAG_FASTCALL "bpf_fastcall" +#define DECL_TAG_KFUNC "bpf_kfunc" + +#define KF_FASTCALL (1 << 12) +#define KF_ARENA_RET (1 << 13) +#define KF_ARENA_ARG1 (1 << 14) +#define KF_ARENA_ARG2 (1 << 15) #define KF_IMPLICIT_ARGS (1 << 16) #define KF_IMPL_SUFFIX "_impl" +#define TYPE_ATTR_ARENA "address_space(1)" +#define PARAM_SUFFIX_ARENA "__arena" +#define PARAM_SUFFIX_ARENA_NULLABLE "__arena__nullable" struct kfunc { + struct rb_node rb_node; const char *name; u32 btf_id; u32 flags; @@ -166,9 +197,7 @@ struct btf2btf_context { u32 *decl_tags; u32 nr_decl_tags; u32 max_decl_tags; - struct kfunc *kfuncs; - u32 nr_kfuncs; - u32 max_kfuncs; + struct rb_root kfuncs; }; static int verbose; @@ -201,6 +230,35 @@ static int eprintf(int level, int var, const char *fmt, ...) #define pr_info(fmt, ...) \ eprintf(0, verbose, pr_fmt(fmt), ##__VA_ARGS__) +/* + * Grow *data so it can hold at least cnt elements of elem_sz bytes each. + * *cap is the capacity in elements and is updated on growth. + */ +static int __ensure_mem(void **data, u32 *cap, u32 cnt, size_t elem_sz) +{ + u32 new_cap, old_cap = *cap; + void *arr; + + if (cnt <= old_cap) + return 0; + + new_cap = max(old_cap + 256, old_cap * 2); + if (new_cap < cnt) + new_cap = cnt; + + arr = realloc(*data, elem_sz * new_cap); + if (!arr) + return -ENOMEM; + + *data = arr; + *cap = new_cap; + + return 0; +} + +#define ensure_mem(arr_ptr, cap_ptr, cnt) \ + __ensure_mem((void **)(arr_ptr), (cap_ptr), (cnt), sizeof(**(arr_ptr))) + static bool is_btf_id(const char *name) { return name && !strncmp(name, BTF_ID_PREFIX, sizeof(BTF_ID_PREFIX) - 1); @@ -480,6 +538,40 @@ static int elf_collect(struct object *obj) return 0; } +static int push_addr_sym(struct object *obj, Elf64_Addr addr, const char *name) +{ + if (ensure_mem(&obj->addr_syms, &obj->addr_syms_cap, obj->addr_syms_cnt + 1)) + return -ENOMEM; + + obj->addr_syms[obj->addr_syms_cnt++] = (struct addr_sym){ + .addr = addr, + .name = name, + }; + + return 0; +} + +static int cmp_addr_sym(const void *a, const void *b) +{ + Elf64_Addr aa = ((const struct addr_sym *)a)->addr; + Elf64_Addr ab = ((const struct addr_sym *)b)->addr; + + return (aa > ab) - (aa < ab); +} + +static const char *find_name_by_addr(struct object *obj, Elf64_Addr addr) +{ + struct addr_sym key = { .addr = addr }; + struct addr_sym *res; + + if (!obj->addr_syms_cnt) + return NULL; + + res = bsearch(&key, obj->addr_syms, obj->addr_syms_cnt, + sizeof(*obj->addr_syms), cmp_addr_sym); + return res ? res->name : NULL; +} + static int symbols_collect(struct object *obj) { Elf_Scn *scn = NULL; @@ -573,8 +665,15 @@ static int symbols_collect(struct object *obj) return -1; } id->addr[id->addr_cnt++] = sym.st_value; + + if (push_addr_sym(obj, sym.st_value, id->name)) + return -1; } + if (obj->addr_syms_cnt) + qsort(obj->addr_syms, obj->addr_syms_cnt, + sizeof(*obj->addr_syms), cmp_addr_sym); + return 0; } @@ -890,17 +989,8 @@ static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, s3 static int push_decl_tag_id(struct btf2btf_context *ctx, u32 decl_tag_id) { - u32 *arr = ctx->decl_tags; - u32 cap = ctx->max_decl_tags; - - if (ctx->nr_decl_tags + 1 > cap) { - cap = max(cap + 256, cap * 2); - arr = realloc(arr, sizeof(u32) * cap); - if (!arr) - return -ENOMEM; - ctx->max_decl_tags = cap; - ctx->decl_tags = arr; - } + if (ensure_mem(&ctx->decl_tags, &ctx->max_decl_tags, ctx->nr_decl_tags + 1)) + return -ENOMEM; ctx->decl_tags[ctx->nr_decl_tags++] = decl_tag_id; @@ -909,23 +999,57 @@ static int push_decl_tag_id(struct btf2btf_context *ctx, u32 decl_tag_id) static int push_kfunc(struct btf2btf_context *ctx, struct kfunc *kfunc) { - struct kfunc *arr = ctx->kfuncs; - u32 cap = ctx->max_kfuncs; + struct rb_node **p = &ctx->kfuncs.rb_node; + struct rb_node *parent = NULL; + struct kfunc *k; - if (ctx->nr_kfuncs + 1 > cap) { - cap = max(cap + 256, cap * 2); - arr = realloc(arr, sizeof(struct kfunc) * cap); - if (!arr) - return -ENOMEM; - ctx->max_kfuncs = cap; - ctx->kfuncs = arr; + /* + * Dedup by BTF ID: collecting the same kfunc twice is a no-op, + * UNLESS the kfunc flags are inconsistent, in which case we + * fail hard because it indicates a bug in a kfunc set declaration. + */ + while (*p) { + parent = *p; + k = rb_entry(parent, struct kfunc, rb_node); + + if (kfunc->btf_id < k->btf_id) { + p = &(*p)->rb_left; + } else if (kfunc->btf_id > k->btf_id) { + p = &(*p)->rb_right; + } else if (k->flags == kfunc->flags) { + return 0; + } else { + pr_err("ERROR: resolve_btfids: kfunc %s has inconsistent flags across BTF ID sets: 0x%x != 0x%x\n", + kfunc->name, k->flags, kfunc->flags); + return -EINVAL; + } } - ctx->kfuncs[ctx->nr_kfuncs++] = *kfunc; + k = zalloc(sizeof(*k)); + if (!k) + return -ENOMEM; + + *k = *kfunc; + rb_link_node(&k->rb_node, parent, p); + rb_insert_color(&k->rb_node, &ctx->kfuncs); return 0; } +static void free_kfuncs(struct rb_root *root) +{ + struct rb_node *next; + struct kfunc *kfunc; + + next = rb_first(root); + while (next) { + kfunc = rb_entry(next, struct kfunc, rb_node); + next = rb_next(&kfunc->rb_node); + rb_erase(&kfunc->rb_node, root); + free(kfunc); + } +} + static int collect_decl_tags(struct btf2btf_context *ctx) { const u32 type_cnt = btf__type_cnt(ctx->btf); @@ -945,94 +1069,76 @@ static int collect_decl_tags(struct btf2btf_context *ctx) return 0; } -/* - * To find the kfunc flags having its struct btf_id (with ELF addresses) - * we need to find the address that is in range of a set8. - * If a set8 is found, then the flags are located at addr + 4 bytes. - * Return 0 (no flags!) if not found. - */ -static u32 find_kfunc_flags(struct object *obj, struct btf_id *kfunc_id) +static bool param_name_has_suffix(const char *name, const char *suffix) { - const u32 *elf_data_ptr = obj->efile.idlist->d_buf; - u64 set_lower_addr, set_upper_addr, addr; - struct btf_id *set_id; - struct rb_node *next; - u32 flags; - u64 idx; + size_t name_len = strlen(name); + size_t suffix_len = strlen(suffix); - for (next = rb_first(&obj->sets); next; next = rb_next(next)) { - set_id = rb_entry(next, struct btf_id, rb_node); - if (set_id->kind != BTF_ID_KIND_SET8 || set_id->addr_cnt != 1) - continue; - - set_lower_addr = set_id->addr[0]; - set_upper_addr = set_lower_addr + set_id->cnt * sizeof(u64); + return name_len >= suffix_len && !strcmp(name + name_len - suffix_len, suffix); +} - for (u32 i = 0; i < kfunc_id->addr_cnt; i++) { - addr = kfunc_id->addr[i]; - /* - * Lower bound is exclusive to skip the 8-byte header of the set. - * Upper bound is inclusive to capture the last entry at offset 8*cnt. - */ - if (set_lower_addr < addr && addr <= set_upper_addr) { - pr_debug("found kfunc %s in BTF_ID_FLAGS %s\n", - kfunc_id->name, set_id->name); - idx = addr - obj->efile.idlist_addr; - idx = idx / sizeof(u32) + 1; - flags = elf_data_ptr[idx]; - - return flags; - } - } - } +static bool is_arena_param(const struct btf *btf, const struct btf_param *param) +{ + const char *name = btf__name_by_offset(btf, param->name_off); - return 0; + return param_name_has_suffix(name, PARAM_SUFFIX_ARENA) || + param_name_has_suffix(name, PARAM_SUFFIX_ARENA_NULLABLE); } static int collect_kfuncs(struct object *obj, struct btf2btf_context *ctx) { - const char *tag_name, *func_name; + Elf_Data *idlist = obj->efile.idlist; struct btf *btf = ctx->btf; - const struct btf_type *t; - u32 flags, func_id; - struct kfunc kfunc; - struct btf_id *id; - int err; + struct rb_node *next; - if (ctx->nr_decl_tags == 0) + if (!idlist || !idlist->d_buf) return 0; - for (u32 i = 0; i < ctx->nr_decl_tags; i++) { - t = btf__type_by_id(btf, ctx->decl_tags[i]); - if (btf_kflag(t) || btf_decl_tag(t)->component_idx != -1) - continue; + for (next = rb_first(&obj->sets); next; next = rb_next(next)) { + struct btf_id_set8 *set8; + struct btf_id *set_id; + u64 set_addr; - tag_name = btf__name_by_offset(btf, t->name_off); - if (strcmp(tag_name, "bpf_kfunc") != 0) + set_id = rb_entry(next, struct btf_id, rb_node); + if (set_id->kind != BTF_ID_KIND_SET8 || set_id->addr_cnt != 1) continue; - func_id = t->type; - t = btf__type_by_id(btf, func_id); - if (!btf_is_func(t)) + set_addr = set_id->addr[0]; + set8 = idlist->d_buf + (set_addr - obj->efile.idlist_addr); + if (!(set8->flags & BTF_SET8_KFUNCS)) continue; - func_name = btf__name_by_offset(btf, t->name_off); - if (!func_name) - continue; + for (u32 i = 0; i < set_id->cnt; i++) { + size_t off = (char *)&set8->pairs[i] - (char *)set8; + const char *name = find_name_by_addr(obj, set_addr + off); + struct kfunc kfunc; + s32 func_id; + int err; - id = btf_id__find(&obj->funcs, func_name); - if (!id || id->kind != BTF_ID_KIND_SYM) - continue; + if (!name) { + pr_err("WARN: resolve_btfids: no BTF ID symbol for %s entry %u\n", + set_id->name, i); + warnings++; + continue; + } - flags = find_kfunc_flags(obj, id); + func_id = btf__find_by_name_kind_own(btf, name, BTF_KIND_FUNC); + if (func_id < 0) { + pr_err("WARN: resolve_btfids: no BTF func for kfunc %s in %s\n", + name, set_id->name); + warnings++; + continue; + } - kfunc.name = id->name; - kfunc.btf_id = func_id; - kfunc.flags = flags; + pr_debug("found kfunc %s in %s\n", name, set_id->name); - err = push_kfunc(ctx, &kfunc); - if (err) - return err; + kfunc.name = name; + kfunc.btf_id = func_id; + kfunc.flags = set8->pairs[i].flags; + err = push_kfunc(ctx, &kfunc); + if (err) + return err; + } } return 0; @@ -1141,7 +1247,7 @@ static int process_kfunc_with_implicit_args(struct btf2btf_context *ctx, struct return -E2BIG; } - if (btf__find_by_name_kind(btf, tmp_name, BTF_KIND_FUNC) > 0) { + if (btf__find_by_name_kind_own(btf, tmp_name, BTF_KIND_FUNC) > 0) { pr_debug("resolve_btfids: function %s already exists in BTF\n", tmp_name); goto add_new_proto; } @@ -1160,7 +1266,7 @@ static int process_kfunc_with_implicit_args(struct btf2btf_context *ctx, struct continue; tag_name = btf__name_by_offset(btf, t->name_off); - if (strcmp(tag_name, "bpf_kfunc") == 0) + if (strcmp(tag_name, DECL_TAG_KFUNC) == 0) continue; idx = btf_decl_tag(t)->component_idx; @@ -1211,22 +1317,187 @@ add_new_proto: return 0; } +static bool is_arena_arg(const struct btf *btf, const struct kfunc *kfunc, + const struct btf_param *param, u32 idx) +{ + if (is_arena_param(btf, param)) + return true; + + switch (idx) { + case 0: + return kfunc->flags & KF_ARENA_ARG1; + case 1: + return kfunc->flags & KF_ARENA_ARG2; + default: + return false; + } +} + +static s32 arena_tag_ptr(struct btf *btf, u32 ptr_id, struct kfunc *kfunc) +{ + const struct btf_type *ptr = btf__type_by_id(btf, ptr_id); + s32 tag_id, new_ptr_id; + + if (!btf_is_ptr(ptr)) { + pr_err("ERROR: resolve_btfids: kfunc %s: arena type is not a pointer\n", + kfunc->name); + return -EINVAL; + } + + tag_id = btf__add_type_attr(btf, TYPE_ATTR_ARENA, ptr->type); + if (tag_id < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a type attr to BTF: %d\n", + kfunc->name, tag_id); + return tag_id; + } + + new_ptr_id = btf__add_ptr(btf, tag_id); + if (new_ptr_id < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a pointer to BTF: %d\n", + kfunc->name, new_ptr_id); + } + + return new_ptr_id; +} + +/* + * Add a FUNC_PROTO for @kfunc with each arena pointer tagged with an + * "address_space(1)" attribute. The original proto may be shared with + * other FUNCs, so it is never modified in place. Returns the original + * proto id when @kfunc has no arena return value or arguments. + */ +static s32 add_arena_tagged_proto(struct btf *btf, struct kfunc *kfunc) +{ + const struct btf_type *func = btf__type_by_id(btf, kfunc->btf_id); + u32 proto_id = func->type; + const struct btf_type *proto = btf__type_by_id(btf, proto_id); + const struct btf_param *params = btf_params(proto); + u32 nr_params = btf_vlen(proto); + s32 ret_type_id = proto->type; + const struct btf_type *t; + struct btf_param *tag_params; + s32 new_proto_id, id; + const char *name; + bool has_arena_arg = false; + int err, i; + + for (i = 0; i < nr_params; i++) { + if (is_arena_arg(btf, kfunc, ¶ms[i], i)) { + has_arena_arg = true; + break; + } + } + + if (!(kfunc->flags & KF_ARENA_RET) && !has_arena_arg) + return proto_id; + + if (kfunc->flags & KF_ARENA_RET) { + ret_type_id = arena_tag_ptr(btf, ret_type_id, kfunc); + if (ret_type_id < 0) + return ret_type_id; + } + + new_proto_id = btf__add_func_proto(btf, ret_type_id); + if (new_proto_id < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a func proto to BTF: %d\n", + kfunc->name, new_proto_id); + return new_proto_id; + } + + for (i = 0; i < nr_params; i++) { + /* btf__add_func_param() below may move the proto, re-fetch */ + proto = btf__type_by_id(btf, proto_id); + name = btf__name_by_offset(btf, btf_params(proto)[i].name_off); + + err = btf__add_func_param(btf, name ?: "", btf_params(proto)[i].type); + if (err < 0) { + pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a proto param to BTF: %d\n", + kfunc->name, err); + return err; + } + } + + for (i = 0; i < nr_params; i++) { + t = btf__type_by_id(btf, new_proto_id); + tag_params = btf_params(t); + if (!is_arena_arg(btf, kfunc, &tag_params[i], i)) + continue; + + id = arena_tag_ptr(btf, tag_params[i].type, kfunc); + if (id < 0) + return id; + + t = btf__type_by_id(btf, new_proto_id); + tag_params = btf_params(t); + tag_params[i].type = id; + } + + pr_debug("added arena-tagged proto for kfunc %s: %d\n", kfunc->name, new_proto_id); + + return new_proto_id; +} + +static int process_kfunc_with_arena_attrs(struct btf2btf_context *ctx, + struct kfunc *kfunc) +{ + struct btf_type *t; + s32 proto_id; + + proto_id = add_arena_tagged_proto(ctx->btf, kfunc); + if (proto_id < 0) + return proto_id; + + t = (struct btf_type *)btf__type_by_id(ctx->btf, kfunc->btf_id); + t->type = proto_id; + + return 0; +} + +static int add_decl_tag(struct btf2btf_context *ctx, const char *tag_name, + u32 target_btf_id, int component_idx) +{ + s32 new_id; + + new_id = btf__add_decl_tag(ctx->btf, tag_name, target_btf_id, component_idx); + if (new_id < 0) { + pr_err("ERROR: resolve_btfids: failed to add '%s' decl tag for BTF id %u: %d\n", + tag_name, target_btf_id, new_id); + return new_id; + } + + return push_decl_tag_id(ctx, new_id); +} + static int btf2btf(struct object *obj) { struct btf2btf_context ctx = {}; + struct rb_node *next; int err; err = build_btf2btf_context(obj, &ctx); if (err) goto out; - for (u32 i = 0; i < ctx.nr_kfuncs; i++) { - struct kfunc *kfunc = &ctx.kfuncs[i]; + for (next = rb_first(&ctx.kfuncs); next; next = rb_next(next)) { + struct kfunc *kfunc = rb_entry(next, struct kfunc, rb_node); - if (!(kfunc->flags & KF_IMPLICIT_ARGS)) - continue; + err = add_decl_tag(&ctx, DECL_TAG_KFUNC, kfunc->btf_id, -1); + if (err) + goto out; + + if (kfunc->flags & KF_FASTCALL) { + err = add_decl_tag(&ctx, DECL_TAG_FASTCALL, kfunc->btf_id, -1); + if (err) + goto out; + } + + if (kfunc->flags & KF_IMPLICIT_ARGS) { + err = process_kfunc_with_implicit_args(&ctx, kfunc); + if (err) + goto out; + } - err = process_kfunc_with_implicit_args(&ctx, kfunc); + err = process_kfunc_with_arena_attrs(&ctx, kfunc); if (err) goto out; } @@ -1234,7 +1505,7 @@ static int btf2btf(struct object *obj) err = 0; out: free(ctx.decl_tags); - free(ctx.kfuncs); + free_kfuncs(&ctx.kfuncs); return err; } @@ -1309,6 +1580,12 @@ static int finalize_btf(struct object *obj) struct btf *base_btf = obj->base_btf, *btf = obj->btf; int err; + err = btf__dedup(obj->btf, NULL); + if (err) { + pr_err("FAILED to dedup BTF: %s\n", strerror(errno)); + goto out_err; + } + if (obj->base_btf && obj->distill_base) { err = btf__distill_base(obj->btf, &base_btf, &btf); if (err) { @@ -1575,6 +1852,7 @@ out: btf_id__free_all(&obj.typedefs); btf_id__free_all(&obj.funcs); btf_id__free_all(&obj.sets); + free(obj.addr_syms); if (obj.efile.elf) { elf_end(obj.efile.elf); close(obj.efile.fd); diff --git a/tools/include/linux/btf_ids.h b/tools/include/linux/btf_ids.h index 72ea363d434d..4fe5c5f1558c 100644 --- a/tools/include/linux/btf_ids.h +++ b/tools/include/linux/btf_ids.h @@ -10,6 +10,9 @@ struct btf_id_set { u32 ids[]; }; +/* This flag implies BTF_SET8 holds kfunc(s) */ +#define BTF_SET8_KFUNCS (1 << 0) + struct btf_id_set8 { u32 cnt; u32 flags; @@ -22,6 +25,7 @@ struct btf_id_set8 { #ifdef CONFIG_DEBUG_INFO_BTF #include <linux/compiler.h> /* for __PASTE */ +#include <linux/stringify.h> /* * Following macros help to define lists of BTF IDs placed @@ -35,7 +39,7 @@ struct btf_id_set8 { #define BTF_IDS_SECTION ".BTF_ids" -#define ____BTF_ID(symbol) \ +#define ____BTF_ID(symbol, word) \ asm( \ ".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ ".local " #symbol " ; \n" \ @@ -43,10 +47,11 @@ asm( \ ".size " #symbol ", 4; \n" \ #symbol ": \n" \ ".zero 4 \n" \ +word \ ".popsection; \n"); -#define __BTF_ID(symbol) \ - ____BTF_ID(symbol) +#define __BTF_ID(symbol, word) \ + ____BTF_ID(symbol, word) #define __ID(prefix) \ __PASTE(__PASTE(prefix, __COUNTER__), __LINE__) @@ -56,7 +61,14 @@ asm( \ * to 4 zero bytes. */ #define BTF_ID(prefix, name) \ - __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__)) + __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), "") + +#define ____BTF_ID_FLAGS(prefix, name, flags) \ + __BTF_ID(__ID(__BTF_ID__##prefix##__##name##__), ".long " #flags "\n") +#define __BTF_ID_FLAGS(prefix, name, flags, ...) \ + ____BTF_ID_FLAGS(prefix, name, flags) +#define BTF_ID_FLAGS(prefix, name, ...) \ + __BTF_ID_FLAGS(prefix, name, ##__VA_ARGS__, 0) /* * The BTF_ID_LIST macro defines pure (unsorted) list @@ -155,10 +167,58 @@ asm( \ ".popsection; \n"); \ extern struct btf_id_set name; +/* + * The BTF_SET8_START/END macros pair defines sorted list of + * BTF IDs and their flags plus its members count, with the + * following layout: + * + * BTF_SET8_START(list) + * BTF_ID_FLAGS(type1, name1, flags) + * BTF_ID_FLAGS(type2, name2, flags) + * BTF_SET8_END(list) + * + * __BTF_ID__set8__list: + * .zero 8 + * list: + * __BTF_ID__type1__name1__3: + * .zero 4 + * .word (1 << 0) | (1 << 2) + * __BTF_ID__type2__name2__5: + * .zero 4 + * .word (1 << 3) | (1 << 1) | (1 << 2) + * + */ +#define __BTF_SET8_START(name, scope, flags) \ +__BTF_ID_LIST(name, local) \ +asm( \ +".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ +"." #scope " __BTF_ID__set8__" #name "; \n" \ +"__BTF_ID__set8__" #name ":; \n" \ +".zero 4 \n" \ +".long " __stringify(flags) "\n" \ +".popsection; \n"); + +#define BTF_SET8_START(name) \ +__BTF_SET8_START(name, local, 0) + +#define BTF_SET8_END(name) \ +asm( \ +".pushsection " BTF_IDS_SECTION ",\"a\"; \n" \ +".size __BTF_ID__set8__" #name ", .-" #name " \n" \ +".popsection; \n"); \ +extern struct btf_id_set8 name; + +#define BTF_KFUNCS_START(name) \ +__BTF_SET8_START(name, local, BTF_SET8_KFUNCS) + +#define BTF_KFUNCS_END(name) \ +BTF_SET8_END(name) + #else -#define BTF_ID_LIST(name) static u32 __maybe_unused name[5]; +#define BTF_ID_LIST(name) static u32 __maybe_unused name[128]; #define BTF_ID(prefix, name) +#define BTF_ID_FLAGS(prefix, name, ...) #define BTF_ID_UNUSED #define BTF_ID_LIST_GLOBAL(name, n) u32 __maybe_unused name[n]; #define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 __maybe_unused name[1]; @@ -166,6 +226,10 @@ extern struct btf_id_set name; #define BTF_SET_START(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_START_GLOBAL(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_END(name) +#define BTF_SET8_START(name) static struct btf_id_set8 __maybe_unused name = { 0 }; +#define BTF_SET8_END(name) +#define BTF_KFUNCS_START(name) static struct btf_id_set8 __maybe_unused name = { .flags = BTF_SET8_KFUNCS }; +#define BTF_KFUNCS_END(name) #endif /* CONFIG_DEBUG_INFO_BTF */ @@ -215,5 +279,9 @@ MAX_BTF_TRACING_TYPE, }; extern u32 btf_tracing_ids[]; +extern u32 bpf_cgroup_btf_id[]; +extern u32 bpf_local_storage_map_btf_id[]; +extern u32 btf_bpf_map_id[]; +extern u32 bpf_kmem_cache_btf_id[]; #endif diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 89b36de5fdbb..732b35cc08d1 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -3038,8 +3038,34 @@ union bpf_attr { * * * **BPF_F_ADJ_ROOM_DECAP_L3_IPV4**, * **BPF_F_ADJ_ROOM_DECAP_L3_IPV6**: - * Indicate the new IP header version after decapsulating the outer - * IP header. Used when the inner and outer IP versions are different. + * Indicate the new IP header version after decapsulating the + * outer IP header. Used when the inner and outer IP versions + * are different. These flags only trigger a protocol change + * without clearing any tunnel-specific GSO flags. + * + * * **BPF_F_ADJ_ROOM_DECAP_L4_GRE**: + * Clear GRE tunnel GSO flags (SKB_GSO_GRE and SKB_GSO_GRE_CSUM) + * when decapsulating a GRE tunnel. + * + * * **BPF_F_ADJ_ROOM_DECAP_L4_UDP**: + * Clear UDP tunnel GSO flags (SKB_GSO_UDP_TUNNEL and + * SKB_GSO_UDP_TUNNEL_CSUM) when decapsulating a UDP tunnel. + * + * * **BPF_F_ADJ_ROOM_DECAP_IPXIP4**: + * Clear IPIP/SIT tunnel GSO flag (SKB_GSO_IPXIP4) when decapsulating + * a tunnel with an outer IPv4 header (IPv4-in-IPv4 or IPv6-in-IPv4). + * + * * **BPF_F_ADJ_ROOM_DECAP_IPXIP6**: + * Clear IPv6 encapsulation tunnel GSO flag (SKB_GSO_IPXIP6) when + * decapsulating a tunnel with an outer IPv6 header (IPv6-in-IPv6 + * or IPv4-in-IPv6). + * + * When using the decapsulation flags above, the skb->encapsulation + * flag is automatically cleared if all tunnel-specific GSO flags + * (SKB_GSO_UDP_TUNNEL, SKB_GSO_UDP_TUNNEL_CSUM, SKB_GSO_GRE, + * SKB_GSO_GRE_CSUM, SKB_GSO_IPXIP4, SKB_GSO_IPXIP6) have been + * removed from the packet. This handles cases where all tunnel + * layers have been decapsulated. * * A call to this helper is susceptible to change the underlying * packet buffer. Therefore, at load time, all checks on pointers @@ -3532,6 +3558,47 @@ union bpf_attr { * Use the mark present in *params*->mark for the fib lookup. * This option should not be used with BPF_FIB_LOOKUP_DIRECT, * as it only has meaning for full lookups. + * **BPF_FIB_LOOKUP_VLAN** + * If the fib lookup resolves to a VLAN device whose + * parent is a real (non-VLAN) device, set + * *params*->h_vlan_proto and *params*->h_vlan_TCI from + * the VLAN device and replace *params*->ifindex with the + * parent's ifindex. *params*->h_vlan_TCI carries the VID + * only, with PCP and DEI bits zero; a consumer wanting to + * set egress priority writes PCP itself. *params*->smac is + * the VLAN device's own address, which can differ from the + * parent's. Only the immediate parent is resolved; if it + * is itself a VLAN device (QinQ) or in another namespace, + * the egress cannot be reduced to a physical device plus + * one tag and the lookup returns + * **BPF_FIB_LKUP_RET_VLAN_FAILURE** with *params*->ifindex + * left at the input. To obtain the VLAN device's own + * ifindex, repeat the lookup without + * **BPF_FIB_LOOKUP_VLAN**, re-initializing *params* + * first: output fields overwrite the inputs they share + * storage with. The swap and the vlan fields + * are written only on success; other output fields keep + * the helper's existing behaviour, so a frag-needed result + * still reports the route mtu in *params*->mtu_result. + * This flag is only valid for XDP programs; tc programs + * receive -EINVAL since they can redirect to the VLAN + * device directly. + * **BPF_FIB_LOOKUP_VLAN_INPUT** + * Treat *params*->h_vlan_proto and *params*->h_vlan_TCI + * as an input VLAN tag and run the lookup as if ingress + * had happened on the VLAN subinterface carrying that tag + * on *params*->ifindex. The VID is the low 12 bits of + * *params*->h_vlan_TCI; *params*->h_vlan_proto must be + * ETH_P_8021Q or ETH_P_8021AD in network byte order, else + * **-EINVAL**. If *params*->ifindex is itself a VLAN + * device, its inner (QinQ) subinterface is matched; for a + * bond or team, pass the master's ifindex. An unmatched + * tag, a down device, or one in another namespace returns + * **BPF_FIB_LKUP_RET_NOT_FWDED**, mirroring real ingress. + * A VID of 0 is looked up literally, so do not set this + * flag for priority-tagged frames. Cannot be combined with + * **BPF_FIB_LOOKUP_TBID** or **BPF_FIB_LOOKUP_OUTPUT** + * (returns **-EINVAL**). * * *ctx* is either **struct xdp_md** for XDP programs or * **struct sk_buff** tc cls_act programs. @@ -4694,6 +4761,7 @@ union bpf_attr { * * **BPF_RB_RING_SIZE**: The size of ring buffer. * * **BPF_RB_CONS_POS**: Consumer position (can wrap around). * * **BPF_RB_PROD_POS**: Producer(s) position (can wrap around). + * * **BPF_RB_OVERWRITE_POS**: Overwrite position (can wrap around). * * Data returned is just a momentary snapshot of actual values * and could be inaccurate, so this facility should be used to @@ -5079,17 +5147,19 @@ union bpf_attr { * Description * Redirect the packet to another net device of index *ifindex*. * This helper is somewhat similar to **bpf_redirect**\ (), except - * that the redirection happens to the *ifindex*' peer device and - * the netns switch takes place from ingress to ingress without - * going through the CPU's backlog queue. + * that the redirection happens to the *ifindex*' peer device. If + * *flags* is 0, the netns switch takes place from ingress to + * ingress without going through the CPU's backlog queue. If the + * **BPF_F_EGRESS** flag is provided then redirection happens in + * the egress direction of the peer device. * * *skb*\ **->mark** and *skb*\ **->tstamp** are not cleared during * the netns switch. * - * The *flags* argument is reserved and must be 0. The helper is - * currently only supported for tc BPF program types at the - * ingress hook and for veth and netkit target device types. The - * peer device must reside in a different network namespace. + * If the *flags* argument is 0, the helper is currently only + * supported for tc BPF program types at the ingress hook and for + * veth and netkit target device types. The peer device must reside + * in a different network namespace. * Return * The helper returns **TC_ACT_REDIRECT** on success or * **TC_ACT_SHOT** on error. @@ -6239,7 +6309,7 @@ enum { }; /* BPF_FUNC_skb_adjust_room flags. */ -enum { +enum bpf_adj_room_flags { BPF_F_ADJ_ROOM_FIXED_GSO = (1ULL << 0), BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 = (1ULL << 1), BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 = (1ULL << 2), @@ -6249,6 +6319,10 @@ enum { BPF_F_ADJ_ROOM_ENCAP_L2_ETH = (1ULL << 6), BPF_F_ADJ_ROOM_DECAP_L3_IPV4 = (1ULL << 7), BPF_F_ADJ_ROOM_DECAP_L3_IPV6 = (1ULL << 8), + BPF_F_ADJ_ROOM_DECAP_L4_GRE = (1ULL << 9), + BPF_F_ADJ_ROOM_DECAP_L4_UDP = (1ULL << 10), + BPF_F_ADJ_ROOM_DECAP_IPXIP4 = (1ULL << 11), + BPF_F_ADJ_ROOM_DECAP_IPXIP6 = (1ULL << 12), }; enum { @@ -6336,9 +6410,10 @@ enum { /* Flags for bpf_redirect and bpf_redirect_map helpers */ enum { BPF_F_INGRESS = (1ULL << 0), /* used for skb path */ + BPF_F_EGRESS = (1ULL << 1), /* used for skb path */ BPF_F_BROADCAST = (1ULL << 3), /* used for XDP path */ BPF_F_EXCLUDE_INGRESS = (1ULL << 4), /* used for XDP path */ -#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS) +#define BPF_F_REDIRECT_FLAGS (BPF_F_INGRESS | BPF_F_EGRESS | BPF_F_BROADCAST | BPF_F_EXCLUDE_INGRESS) }; #define __bpf_md_ptr(type, name) \ @@ -6840,6 +6915,15 @@ struct bpf_link_info { __u32 pid; } uprobe_multi; struct { + __u32 attach_type; + __u32 count; /* in/out: tracing_multi target count */ + __u32 btf_obj_id; + __u32 :32; + __aligned_u64 ids; + __aligned_u64 addrs; + __aligned_u64 cookies; + } tracing_multi; + struct { __u32 type; /* enum bpf_perf_event_type */ __u32 :32; union { @@ -7327,6 +7411,8 @@ enum { BPF_FIB_LOOKUP_TBID = (1U << 3), BPF_FIB_LOOKUP_SRC = (1U << 4), BPF_FIB_LOOKUP_MARK = (1U << 5), + BPF_FIB_LOOKUP_VLAN = (1U << 6), + BPF_FIB_LOOKUP_VLAN_INPUT = (1U << 7), }; enum { @@ -7340,6 +7426,7 @@ enum { BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */ BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */ BPF_FIB_LKUP_RET_NO_SRC_ADDR, /* failed to derive IP src addr */ + BPF_FIB_LKUP_RET_VLAN_FAILURE, /* VLAN egress, parent unresolvable */ }; struct bpf_fib_lookup { @@ -7393,7 +7480,13 @@ struct bpf_fib_lookup { union { struct { - /* output */ + /* + * output with BPF_FIB_LOOKUP_VLAN: set from the + * resolved egress VLAN device (see the flag); zeroed + * on other successful lookups. input with + * BPF_FIB_LOOKUP_VLAN_INPUT: the VLAN tag to scope + * the lookup by. + */ __be16 h_vlan_proto; __be16 h_vlan_TCI; }; diff --git a/tools/lib/bpf/bpf_gen_internal.h b/tools/lib/bpf/bpf_gen_internal.h index 49af4260b8e6..6c5ad6c55e8a 100644 --- a/tools/lib/bpf/bpf_gen_internal.h +++ b/tools/lib/bpf/bpf_gen_internal.h @@ -51,7 +51,6 @@ struct bpf_gen { __u32 nr_ksyms; int fd_array; int nr_fd_array; - int hash_insn_offset[SHA256_DWORD_SIZE]; }; void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps); @@ -66,7 +65,8 @@ void bpf_gen__prog_load(struct bpf_gen *gen, enum bpf_prog_type prog_type, const char *prog_name, const char *license, struct bpf_insn *insns, size_t insn_cnt, struct bpf_prog_load_opts *load_attr, int prog_idx); -void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size); +void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size, + __u64 flags); void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 823bce895178..8417de92d028 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -589,7 +589,7 @@ static int btf_parse_type_sec(struct btf *btf) if (type_size < 0) return type_size; if (next_type + type_size > end_type) { - pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types); + pr_warn("BTF type [%u] is malformed\n", btf->start_id + btf->nr_types); return -EINVAL; } @@ -1424,7 +1424,7 @@ static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs continue; if (sh.sh_type != SHT_PROGBITS) { - pr_warn("unexpected section type (%d) of section(%d, %s) from %s\n", + pr_warn("unexpected section type (%u) of section(%d, %s) from %s\n", sh.sh_type, idx, name, path); goto err; } @@ -1506,9 +1506,6 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, dist_base_btf = NULL; } - if (dist_base_btf) - btf->owns_base = true; - switch (gelf_getclass(elf)) { case ELFCLASS32: btf__set_pointer_size(btf, 4); @@ -1523,13 +1520,16 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, if (btf_ext && secs.btf_ext_data) { *btf_ext = btf_ext__new(secs.btf_ext_data->d_buf, secs.btf_ext_data->d_size); - if (IS_ERR(*btf_ext)) { - err = PTR_ERR(*btf_ext); + if (!*btf_ext) { + err = -errno; goto done; } } else if (btf_ext) { *btf_ext = NULL; } + + if (dist_base_btf) + btf->owns_base = true; done: if (elf) elf_end(elf); @@ -4854,7 +4854,7 @@ recur: continue; if (!btf_dedup_identical_types(d, m1->type, m2->type, depth - 1)) { if (t1->name_off) { - pr_debug("%s '%s' size=%d vlen=%d id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n", + pr_debug("%s '%s' size=%u vlen=%u id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n", k1 == BTF_KIND_STRUCT ? "STRUCT" : "UNION", btf__name_by_offset(d->btf, t1->name_off), t1->size, btf_vlen(t1), id1, id2, i, @@ -5104,7 +5104,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type); if (eq <= 0) { if (cand_type->name_off) { - pr_debug("%s '%s' size=%d vlen=%d cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n", + pr_debug("%s '%s' size=%u vlen=%u cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n", cand_kind == BTF_KIND_STRUCT ? "STRUCT" : "UNION", btf__name_by_offset(d->btf, cand_type->name_off), cand_type->size, vlen, cand_id, canon_id, i, @@ -6069,7 +6069,7 @@ static int btf_add_distilled_types(struct btf_distill *dist) err = btf_add_type(&dist->pipe, t); break; default: - pr_warn("unexpected kind when adding base type '%s'[%u] of kind [%u] to distilled base BTF.\n", + pr_warn("unexpected kind when adding base type '%s'[%d] of kind [%d] to distilled base BTF.\n", name, i, kind); return -EINVAL; diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 1a31f2da947f..587172c0de08 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -172,6 +172,8 @@ LIBBPF_API __s32 btf__find_by_name(const struct btf *btf, const char *type_name); LIBBPF_API __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, __u32 kind); +LIBBPF_API __s32 btf__find_by_name_kind_own(const struct btf *btf, + const char *type_name, __u32 kind); LIBBPF_API __u32 btf__type_cnt(const struct btf *btf); LIBBPF_API const struct btf *btf__base_btf(const struct btf *btf); LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf, diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index cc1ba65bb6c5..123c448f20c7 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -1776,7 +1776,7 @@ static int btf_dump_get_bitfield_value(struct btf_dump *d, /* Maximum supported bitfield size is 64 bits */ if (t->size > 8) { - pr_warn("unexpected bitfield size %d\n", t->size); + pr_warn("unexpected bitfield size %u\n", t->size); return -EINVAL; } @@ -2251,7 +2251,7 @@ static int btf_dump_get_enum_value(struct btf_dump *d, *value = is_signed ? *(__s8 *)data : *(__u8 *)data; return 0; default: - pr_warn("unexpected size %d for enum, id:[%u]\n", t->size, id); + pr_warn("unexpected size %u for enum, id:[%u]\n", t->size, id); return -EINVAL; } } diff --git a/tools/lib/bpf/btf_relocate.c b/tools/lib/bpf/btf_relocate.c index 53d1f3541bce..df5fa4bd87d6 100644 --- a/tools/lib/bpf/btf_relocate.c +++ b/tools/lib/bpf/btf_relocate.c @@ -280,7 +280,7 @@ static int btf_relocate_map_distilled_base(struct btf_relocate *r) cmp_btf_name_size(&base_info, dist_info) == 0; dist_info++) { if (!dist_info->id || dist_info->id >= r->nr_dist_base_types) { - pr_warn("base BTF id [%d] maps to invalid distilled base BTF id [%d]\n", + pr_warn("base BTF id [%u] maps to invalid distilled base BTF id [%u]\n", id, dist_info->id); err = -EINVAL; goto done; @@ -368,7 +368,7 @@ static int btf_relocate_map_distilled_base(struct btf_relocate *r) continue; dist_t = btf_type_by_id(r->dist_base_btf, id); name = btf__name_by_offset(r->dist_base_btf, dist_t->name_off); - pr_warn("distilled base BTF type '%s' [%d] is not mapped to base BTF id\n", + pr_warn("distilled base BTF type '%s' [%u] is not mapped to base BTF id\n", name, id); err = -EINVAL; break; @@ -397,11 +397,11 @@ static int btf_relocate_validate_distilled_base(struct btf_relocate *r) case BTF_KIND_FWD: if (t->name_off) break; - pr_warn("type [%d], kind [%d] is invalid for distilled base BTF; it is anonymous\n", + pr_warn("type [%u], kind [%d] is invalid for distilled base BTF; it is anonymous\n", i, kind); return -EINVAL; default: - pr_warn("type [%d] in distilled based BTF has unexpected kind [%d]\n", + pr_warn("type [%u] in distilled based BTF has unexpected kind [%d]\n", i, kind); return -EINVAL; } diff --git a/tools/lib/bpf/elf.c b/tools/lib/bpf/elf.c index 295dbda24580..fe136d025967 100644 --- a/tools/lib/bpf/elf.c +++ b/tools/lib/bpf/elf.c @@ -354,7 +354,7 @@ long elf_find_func_offset(Elf *elf, const char *binary_path, const char *name) if (ret > 0) { pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path, - ret); + (unsigned long)ret); } else { if (ret == 0) { pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path, diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index e5641fa60163..9f589980beef 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -620,6 +620,38 @@ static int probe_bpf_syscall_common_attrs(int token_fd) return probe_sys_bpf_ext(); } +static int probe_kern_percpu_data(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "libbpf_percpu", sizeof(int), 8, 1, + &map_opts); + if (map < 0) { + pr_warn("Error in %s(): %s. Couldn't create simple percpu_array map.\n", + __func__, errstr(map)); + return map; + } + + insns[0].imm = map; + + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + typedef int (*feature_probe_fn)(int /* token_fd */); static struct kern_feature_cache feature_cache; @@ -707,6 +739,9 @@ static struct kern_feature_desc { [FEAT_BPF_SYSCALL_COMMON_ATTRS] = { "BPF syscall common attributes support", probe_bpf_syscall_common_attrs, }, + [FEAT_PERCPU_DATA] = { + "kernel supports percpu data", probe_kern_percpu_data, + }, }; bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index d79695f01c87..af3a04f161ac 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -111,7 +111,6 @@ static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn in static int add_data(struct bpf_gen *gen, const void *data, __u32 size); static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off); -static void emit_signature_match(struct bpf_gen *gen); void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps) { @@ -154,8 +153,6 @@ void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */ emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7)); emit(gen, BPF_EXIT_INSN()); - if (OPTS_GET(gen->opts, gen_hash, false)) - emit_signature_match(gen); } static int add_data(struct bpf_gen *gen, const void *data, __u32 size) @@ -377,14 +374,12 @@ static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off) __emit_sys_close(gen); } -static void compute_sha_update_offsets(struct bpf_gen *gen); - int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) { int i; if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) { - pr_warn("nr_progs %d/%d nr_maps %d/%d mismatch\n", + pr_warn("nr_progs %d/%u nr_maps %d/%u mismatch\n", nr_progs, gen->nr_progs, nr_maps, gen->nr_maps); gen->error = -EFAULT; return gen->error; @@ -408,9 +403,6 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) if (!gen->error) { struct gen_loader_opts *opts = gen->opts; - if (OPTS_GET(opts, gen_hash, false)) - compute_sha_update_offsets(gen); - opts->insns = gen->insn_start; opts->insns_sz = gen->insn_cur - gen->insn_start; opts->data = gen->data_start; @@ -460,22 +452,6 @@ void bpf_gen__free(struct bpf_gen *gen) _val; \ }) -static void compute_sha_update_offsets(struct bpf_gen *gen) -{ - __u64 sha[SHA256_DWORD_SIZE]; - __u64 sha_dw; - int i; - - libbpf_sha256(gen->data_start, gen->data_cur - gen->data_start, (__u8 *)sha); - for (i = 0; i < SHA256_DWORD_SIZE; i++) { - struct bpf_insn *insn = - (struct bpf_insn *)(gen->insn_start + gen->hash_insn_offset[i]); - sha_dw = tgt_endian(sha[i]); - insn[0].imm = (__u32)sha_dw; - insn[1].imm = sha_dw >> 32; - } -} - void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, __u32 btf_raw_size) { @@ -488,7 +464,7 @@ void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, attr.btf_size = tgt_endian(btf_raw_size); btf_load_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n", + pr_debug("gen: load_btf: off %d size %u, attr: off %d size %d\n", btf_data, btf_raw_size, btf_load_attr, attr_size); /* populate union bpf_attr with user provided log details */ @@ -534,7 +510,7 @@ void bpf_gen__map_create(struct bpf_gen *gen, attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id); map_create_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n", + pr_debug("gen: map_create: %s idx %d type %u value_type_id %u, attr: off %d size %d\n", map_name, map_idx, map_type, map_attr->btf_value_type_id, map_create_attr, attr_size); @@ -557,8 +533,9 @@ void bpf_gen__map_create(struct bpf_gen *gen, * Conditionally update max_entries from the host-supplied loader * ctx. This sizes the map at runtime, but for a signed loader * (gen_hash) it would let an untrusted host re-dimension the - * program's maps after emit_signature_match(), outside what the - * signature attests to. Keep the signer-provided max_entries + * program's maps, outside what the signature attests to: the + * metadata blob is covered by the program signature and verified + * by the kernel at load time. Keep the signer-provided max_entries * baked into the blob in that case. */ if (map_idx >= 0 && !OPTS_GET(gen->opts, gen_hash, false)) @@ -596,45 +573,6 @@ void bpf_gen__map_create(struct bpf_gen *gen, emit_sys_close_stack(gen, stack_off(inner_map_fd)); } -static void emit_signature_match(struct bpf_gen *gen) -{ - __s64 off; - int i; - - /* - * Reject if the metadata map is not exclusive. Without exclusivity - * the cached map->sha[] verified above can be stale: another BPF - * program with map access could have mutated the contents between - * BPF_OBJ_GET_INFO_BY_FD and loader execution. - */ - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX, - 0, 0, 0, 0)); - emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, SHA256_DIGEST_LENGTH)); - off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2; - if (is_simm16(off)) { - emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL)); - emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_2, 1, off)); - } else { - gen->error = -ERANGE; - } - - for (i = 0; i < SHA256_DWORD_SIZE; i++) { - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX, - 0, 0, 0, 0)); - emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, i * sizeof(__u64))); - gen->hash_insn_offset[i] = gen->insn_cur - gen->insn_start; - emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_3, 0, 0, 0, 0, 0)); - - off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 2; - if (is_simm16(off)) { - emit(gen, BPF_MOV64_IMM(BPF_REG_7, -EINVAL)); - emit(gen, BPF_JMP_REG(BPF_JNE, BPF_REG_2, BPF_REG_3, off)); - } else { - gen->error = -ERANGE; - } - } -} - void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, enum bpf_attach_type type) { @@ -1082,7 +1020,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, license_off = add_data(gen, license, strlen(license) + 1); /* add insns to blob of bytes */ insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); - pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n", + pr_debug("gen: prog_load: prog_idx %d type %u insn off %d insns_cnt %zu license off %d\n", prog_idx, prog_type, insns_off, insn_cnt, license_off); /* convert blob insns to target endianness */ @@ -1105,21 +1043,21 @@ void bpf_gen__prog_load(struct bpf_gen *gen, attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size); attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt); func_info = add_data(gen, load_attr->func_info, func_info_tot_sz); - pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n", + pr_debug("gen: prog_load: func_info: off %d cnt %u rec size %u\n", func_info, load_attr->func_info_cnt, load_attr->func_info_rec_size); attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size); attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt); line_info = add_data(gen, load_attr->line_info, line_info_tot_sz); - pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n", + pr_debug("gen: prog_load: line_info: off %d cnt %u rec size %u\n", line_info, load_attr->line_info_cnt, load_attr->line_info_rec_size); attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo)); attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt); core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz); - pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n", + pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zu\n", core_relos, gen->core_relo_cnt, sizeof(struct bpf_core_relo)); @@ -1190,7 +1128,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, } void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, - __u32 value_size) + __u32 value_size, __u64 flags) { int attr_size = offsetofend(union bpf_attr, flags); int map_update_attr, value, key; @@ -1198,6 +1136,7 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, int zero = 0; memset(&attr, 0, attr_size); + attr.flags = tgt_endian(flags); value = add_data(gen, pvalue, value_size); key = add_data(gen, &zero, sizeof(zero)); @@ -1211,10 +1150,10 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, * } * * The runtime initial_value comes from the host-supplied loader - * ctx and would overwrite the blob value after emit_signature_match() - * has already validated map->sha[]. For a signed loader (gen_hash) - * the attested blob value must be authoritative, so skip the override - * and leave the hashed value in place. + * ctx and would overwrite the blob value that the program signature + * covers and the kernel verifies at load time. For a signed loader + * (gen_hash) the attested blob value must be authoritative, so skip + * the override and leave the signed value in place. */ if (!OPTS_GET(gen->opts, gen_hash, false)) { emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6, @@ -1234,7 +1173,7 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, } map_update_attr = add_data(gen, &attr, attr_size); - pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n", + pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n", map_idx, value, value_size, map_update_attr, attr_size); move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4, blob_fd_array_off(gen, map_idx)); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 1368752aa13c..b749c01742ee 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -541,6 +541,7 @@ struct bpf_struct_ops { }; #define DATA_SEC ".data" +#define PERCPU_SEC ".percpu" #define BSS_SEC ".bss" #define RODATA_SEC ".rodata" #define KCONFIG_SEC ".kconfig" @@ -555,6 +556,7 @@ enum libbpf_map_type { LIBBPF_MAP_BSS, LIBBPF_MAP_RODATA, LIBBPF_MAP_KCONFIG, + LIBBPF_MAP_PERCPU, }; struct bpf_map_def { @@ -666,6 +668,7 @@ enum sec_type { SEC_DATA, SEC_RODATA, SEC_ST_OPS, + SEC_PERCPU, }; struct elf_sec_desc { @@ -1486,7 +1489,7 @@ static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name, type->size); st_ops->type_id = type_id; - pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", + pr_debug("struct_ops init: struct %s(type_id=%d) %s found at offset %u\n", tname, type_id, var_name, vsi->offset); } @@ -1838,6 +1841,7 @@ static size_t bpf_map_mmap_sz(const struct bpf_map *map) switch (map->def.type) { case BPF_MAP_TYPE_ARRAY: + case BPF_MAP_TYPE_PERCPU_ARRAY: return array_map_mmap_sz(map->def.value_size, map->def.max_entries); case BPF_MAP_TYPE_ARENA: return page_sz * map->def.max_entries; @@ -1866,7 +1870,8 @@ static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz return 0; } -static char *internal_map_name(struct bpf_object *obj, const char *real_name) +static char *internal_map_name(struct bpf_object *obj, const char *real_name, + enum libbpf_map_type type) { char map_name[BPF_OBJ_NAME_LEN], *p; int pfx_len, sfx_len = max((size_t)7, strlen(real_name)); @@ -1907,8 +1912,11 @@ static char *internal_map_name(struct bpf_object *obj, const char *real_name) if (sfx_len >= BPF_OBJ_NAME_LEN) sfx_len = BPF_OBJ_NAME_LEN - 1; - /* if there are two or more dots in map name, it's a custom dot map */ - if (strchr(real_name + 1, '.') != NULL) + /* + * Don't prefix the bpf_object name if this is a custom dot map + * (containing two or more dots) or a percpu data map. + */ + if (strchr(real_name + 1, '.') != NULL || type == LIBBPF_MAP_PERCPU) pfx_len = 0; else pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); @@ -1941,6 +1949,13 @@ static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map) if (!map->btf_value_type_id) return false; + /* + * The internal PERCPU maps are not mmapable because the underlying + * percpu_array maps do not have mmap support. + */ + if (map->libbpf_type == LIBBPF_MAP_PERCPU) + return false; + t = btf__type_by_id(obj->btf, map->btf_value_type_id); if (!btf_is_datasec(t)) return false; @@ -1962,6 +1977,7 @@ static int bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, const char *real_name, int sec_idx, void *data, size_t data_sz) { + bool is_percpu = type == LIBBPF_MAP_PERCPU; struct bpf_map_def *def; struct bpf_map *map; size_t mmap_sz; @@ -1975,7 +1991,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, map->sec_idx = sec_idx; map->sec_offset = 0; map->real_name = strdup(real_name); - map->name = internal_map_name(obj, real_name); + map->name = internal_map_name(obj, real_name, type); if (!map->real_name || !map->name) { zfree(&map->real_name); zfree(&map->name); @@ -1983,7 +1999,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, } def = &map->def; - def->type = BPF_MAP_TYPE_ARRAY; + def->type = is_percpu ? BPF_MAP_TYPE_PERCPU_ARRAY : BPF_MAP_TYPE_ARRAY; def->key_size = sizeof(int); def->value_size = data_sz; def->max_entries = 1; @@ -1996,8 +2012,9 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, if (map_is_mmapable(obj, map)) def->map_flags |= BPF_F_MMAPABLE; - pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", - map->name, map->sec_idx, map->sec_offset, def->map_flags); + pr_debug("map '%s' (global %sdata): at sec_idx %d, offset %zu, flags %x.\n", + map->name, is_percpu ? "percpu " : "", map->sec_idx, + map->sec_offset, def->map_flags); mmap_sz = bpf_map_mmap_sz(map); map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, @@ -2057,6 +2074,13 @@ static int bpf_object__init_global_data_maps(struct bpf_object *obj) NULL, sec_desc->data->d_size); break; + case SEC_PERCPU: + sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); + err = bpf_object__init_internal_map(obj, LIBBPF_MAP_PERCPU, + sec_name, sec_idx, + sec_desc->data->d_buf, + sec_desc->data->d_size); + break; default: /* skip */ break; @@ -2626,7 +2650,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': key type [%d] not found.\n", + pr_warn("map '%s': key type [%u] not found.\n", map_name, m->type); return -EINVAL; } @@ -2666,7 +2690,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': value type [%d] not found.\n", + pr_warn("map '%s': value type [%u] not found.\n", map_name, m->type); return -EINVAL; } @@ -2720,7 +2744,7 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, map_def->value_size = 4; t = btf__type_by_id(btf, m->type); if (!t) { - pr_warn("map '%s': %s type [%d] not found.\n", + pr_warn("map '%s': %s type [%u] not found.\n", map_name, desc, m->type); return -EINVAL; } @@ -2806,7 +2830,7 @@ static size_t adjust_ringbuf_sz(size_t sz) return 0; /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be * a power-of-2 multiple of kernel's page size. If user diligently - * satisified these conditions, pass the size through. + * satisfied these conditions, pass the size through. */ if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) return sz; @@ -3476,7 +3500,7 @@ static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, var_name = btf__name_by_offset(btf, t_var->name_off); if (!var_name) { - pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n", + pr_debug("sec '%s': failed to find name of DATASEC's member #%u\n", sec_name, i); return -ENOENT; } @@ -3971,7 +3995,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (!data) return -LIBBPF_ERRNO__FORMAT; - pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", + pr_debug("elf: section(%d) %s, size %lu, link %d, flags %lx, type=%d\n", idx, name, (unsigned long)data->d_size, (int)sh->sh_link, (unsigned long)sh->sh_flags, (int)sh->sh_type); @@ -4016,6 +4040,11 @@ static int bpf_object__elf_collect(struct bpf_object *obj) sec_desc->sec_type = SEC_RODATA; sec_desc->shdr = sh; sec_desc->data = data; + } else if (strcmp(name, PERCPU_SEC) == 0 || + str_has_pfx(name, PERCPU_SEC ".")) { + sec_desc->sec_type = SEC_PERCPU; + sec_desc->shdr = sh; + sec_desc->data = data; } else if (strcmp(name, STRUCT_OPS_SEC) == 0 || strcmp(name, STRUCT_OPS_LINK_SEC) == 0 || strcmp(name, "?" STRUCT_OPS_SEC) == 0 || @@ -4494,7 +4523,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj) ext->kcfg.data_off = roundup(off, ext->kcfg.align); off = ext->kcfg.data_off + ext->kcfg.sz; - pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", + pr_debug("extern (kcfg) #%d: symbol %d, off %d, name %s\n", i, ext->sym_idx, ext->kcfg.data_off, ext->name); } sec->size = off; @@ -4544,6 +4573,7 @@ static bool bpf_object__shndx_is_data(const struct bpf_object *obj, case SEC_BSS: case SEC_DATA: case SEC_RODATA: + case SEC_PERCPU: return true; default: return false; @@ -4569,6 +4599,8 @@ bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) return LIBBPF_MAP_DATA; case SEC_RODATA: return LIBBPF_MAP_RODATA; + case SEC_PERCPU: + return LIBBPF_MAP_PERCPU; default: return LIBBPF_MAP_UNSPEC; } @@ -4626,7 +4658,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, struct bpf_map *map; if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { - pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", + pr_warn("prog '%s': invalid relo against '%s' for insns[%u].code 0x%x\n", prog->name, sym_name, insn_idx, insn->code); return -LIBBPF_ERRNO__RELOC; } @@ -4749,7 +4781,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, map->sec_idx != sym->st_shndx || map->sec_offset != sym->st_value) continue; - pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", + pr_debug("prog '%s': found map %zu (%s, sec %d, off %zu) for insn #%u\n", prog->name, map_idx, map->name, map->sec_idx, map->sec_offset, insn_idx); break; @@ -4776,7 +4808,7 @@ static int bpf_program__record_reloc(struct bpf_program *prog, map = &obj->maps[map_idx]; if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) continue; - pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", + pr_debug("prog '%s': found data map %zu (%s, sec %d, off %zu) for insn %u\n", prog->name, map_idx, map->name, map->sec_idx, map->sec_offset, insn_idx); break; @@ -4944,7 +4976,7 @@ static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map) /* * LLVM annotates global data differently in BTF, that is, - * only as '.data', '.bss' or '.rodata'. + * only as '.data', '.bss', '.percpu' or '.rodata'. */ if (!bpf_map__is_internal(map)) return -ENOENT; @@ -4985,7 +5017,7 @@ static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) info->value_size = val; else if (sscanf(buff, "max_entries:\t%u", &val) == 1) info->max_entries = val; - else if (sscanf(buff, "map_flags:\t%i", &val) == 1) + else if (sscanf(buff, "map_flags:\t%x", &val) == 1) info->map_flags = val; } @@ -5172,12 +5204,8 @@ bpf_object__probe_loading(struct bpf_object *obj) BPF_EXIT_INSN(), }; int ret, insn_cnt = ARRAY_SIZE(insns); - LIBBPF_OPTS(bpf_prog_load_opts, opts, - .token_fd = obj->token_fd, - .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0, - ); - if (obj->gen_loader) + if (obj->gen_loader || obj->token_fd) return 0; ret = bump_rlimit_memlock(); @@ -5186,9 +5214,9 @@ bpf_object__probe_loading(struct bpf_object *obj) errstr(ret)); /* make sure basic loading works */ - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts); + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) - ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); + ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) { ret = errno; pr_warn("Error in %s(): %s. Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.\n", @@ -5297,18 +5325,20 @@ static int bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) { enum libbpf_map_type map_type = map->libbpf_type; + bool is_percpu = map_type == LIBBPF_MAP_PERCPU; + const __u64 update_flags = is_percpu ? BPF_F_ALL_CPUS : 0; int err, zero = 0; size_t mmap_sz; if (obj->gen_loader) { bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, - map->mmaped, map->def.value_size); + map->mmaped, map->def.value_size, update_flags); if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); return 0; } - err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); + err = bpf_map_update_elem(map->fd, &zero, map->mmaped, update_flags); if (err) { err = -errno; pr_warn("map '%s': failed to set initial contents: %s\n", @@ -5353,6 +5383,13 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) return err; } map->mmaped = mmaped; + } else if (is_percpu) { + if (mprotect(map->mmaped, mmap_sz, PROT_READ)) { + err = -errno; + pr_warn("map '%s': failed to mprotect() contents: %s\n", + bpf_map__name(map), errstr(err)); + return err; + } } else if (map->mmaped) { munmap(map->mmaped, mmap_sz); map->mmaped = NULL; @@ -5521,11 +5558,11 @@ static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) } if (err) { err = -errno; - pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %s\n", + pr_warn("map '%s': failed to initialize slot [%u] to map '%s' fd=%d: %s\n", map->name, i, targ_map->name, fd, errstr(err)); return err; } - pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", + pr_debug("map '%s': slot [%u] set to map '%s' fd=%d\n", map->name, i, targ_map->name, fd); } @@ -5554,11 +5591,11 @@ static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) err = bpf_map_update_elem(map->fd, &i, &fd, 0); if (err) { err = -errno; - pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %s\n", + pr_warn("map '%s': failed to initialize slot [%u] to prog '%s' fd=%d: %s\n", map->name, i, targ_prog->name, fd, errstr(err)); return err; } - pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n", + pr_debug("map '%s': slot [%u] set to prog '%s' fd=%d\n", map->name, i, targ_prog->name, fd); } @@ -5628,9 +5665,16 @@ bpf_object__create_maps(struct bpf_object *obj) * runtime due to bpf_program__set_autoload(prog, false), * bpf_object loading will succeed just fine even on old * kernels. + * Same skipping applies to percpu data. */ - if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA)) - map->autocreate = false; + if (bpf_map__is_internal(map)) { + bool is_percpu = map->libbpf_type == LIBBPF_MAP_PERCPU; + enum kern_feature_id feat_id; + + feat_id = is_percpu ? FEAT_PERCPU_DATA : FEAT_GLOBAL_DATA; + if (!kernel_supports(obj, feat_id)) + map->autocreate = false; + } if (!map->autocreate) { pr_debug("map '%s': skipped auto-creating...\n", map->name); @@ -5788,7 +5832,7 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand, if (strncmp(local_name, targ_name, local_essent_len) != 0) continue; - pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", + pr_debug("CO-RE relocating [%u] %s %s: found target candidate [%d] %s %s in [%s]\n", local_cand->id, btf_kind_str(local_t), local_name, i, btf_kind_str(t), targ_name, targ_btf_name); @@ -5848,7 +5892,7 @@ static int load_module_btfs(struct bpf_object *obj) if (errno == ENOENT) continue; /* expected race: BTF was unloaded */ err = -errno; - pr_warn("failed to get BTF object #%d FD: %s\n", id, errstr(err)); + pr_warn("failed to get BTF object #%u FD: %s\n", id, errstr(err)); return err; } @@ -5861,7 +5905,7 @@ static int load_module_btfs(struct bpf_object *obj) err = bpf_btf_get_info_by_fd(fd, &info, &len); if (err) { err = -errno; - pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err)); + pr_warn("failed to get BTF object #%u info: %s\n", id, errstr(err)); break; } @@ -5874,7 +5918,7 @@ static int load_module_btfs(struct bpf_object *obj) btf = btf_get_from_fd(fd, obj->btf_vmlinux); err = libbpf_get_error(btf); if (err) { - pr_warn("failed to load module [%s]'s BTF object #%d: %s\n", + pr_warn("failed to load module [%s]'s BTF object #%u: %s\n", name, id, errstr(err)); break; } @@ -6067,7 +6111,7 @@ static int bpf_core_resolve_relo(struct bpf_program *prog, !hashmap__find(cand_cache, local_id, &cands)) { cands = bpf_core_find_cands(prog->obj, local_btf, local_id); if (IS_ERR(cands)) { - pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", + pr_warn("prog '%s': relo #%d: target candidate search failed for [%u] %s %s: %ld\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), local_name, PTR_ERR(cands)); return PTR_ERR(cands); @@ -6127,7 +6171,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) goto out; } - pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); + pr_debug("sec '%s': found %u CO-RE relocations\n", sec_name, sec->num_info); for_each_btf_ext_rec(seg, sec, i, rec) { if (rec->insn_off % BPF_INSN_SZ) @@ -6181,7 +6225,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); if (err) { - pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %s\n", + pr_warn("prog '%s': relo #%d: failed to patch insn #%d: %s\n", prog->name, i, insn_idx, errstr(err)); goto out; } @@ -6346,7 +6390,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc goto err_close; } if (sym_off + jt_size > obj->jumptables_data_sz) { - pr_warn("map '.jumptables': jumptables_data size is %zd, trying to access %d\n", + pr_warn("map '.jumptables': jumptables_data size is %zu, trying to access %u\n", obj->jumptables_data_sz, sym_off + jt_size); err = -EINVAL; goto err_close; @@ -6381,7 +6425,7 @@ static int create_jt_map(struct bpf_object *obj, struct bpf_program *prog, struc */ if (insn_off > UINT32_MAX) { pr_warn("map '.jumptables': invalid jump table value 0x%llx at offset %u\n", - (long long)jt[i], sym_off + i * jt_entry_size); + (unsigned long long)jt[i], sym_off + i * jt_entry_size); err = -EINVAL; goto err_close; } @@ -6517,7 +6561,7 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) } break; default: - pr_warn("prog '%s': relo #%d: bad relo type %d\n", + pr_warn("prog '%s': relo #%d: bad relo type %u\n", prog->name, i, relo->type); return -EINVAL; } @@ -6797,7 +6841,7 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, */ continue; if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { - pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", + pr_warn("prog '%s': unexpected relo for insn #%zu, type %u\n", prog->name, insn_idx, relo->type); return -LIBBPF_ERRNO__RELOC; } @@ -6944,7 +6988,7 @@ bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, * +-----------+------+------+ * * At this point, we relocate subA calls, then go one level up and finish with - * relocatin mainA calls. mainA is done. + * relocation mainA calls. mainA is done. * * For mainB process is similar but results in different order. We start with * mainB and skip subA and subB, as mainB never calls them (at least @@ -7587,7 +7631,7 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, } name = elf_sym_str(obj, sym->st_name) ?: "<?>"; - pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n", + pr_debug(".maps relo #%d: for %zd value %zu rel->r_offset %zu name %u ('%s')\n", i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, (size_t)rel->r_offset, sym->st_name, name); @@ -7678,7 +7722,7 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, } map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; - pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n", + pr_debug(".maps relo #%d: map '%s' slot [%u] points to %s '%s'\n", i, map->name, moff, type, name); } @@ -7891,7 +7935,7 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, prog->attach_btf_id = btf_type_id; /* but by now libbpf common logic is not utilizing - * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because + * prog->attach_btf_obj_fd/prog->attach_btf_id anymore because * this callback is called after opts were populated by * libbpf, so this callback has to update opts explicitly here */ @@ -8738,7 +8782,7 @@ static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, local_name = btf__name_by_offset(obj->btf, local_type->name_off); targ_name = btf__name_by_offset(btf, targ_type->name_off); - pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", + pr_warn("extern (var ksym) '%s': incompatible types, expected [%u] %s %s, but kernel has [%u] %s %s\n", ext->name, local_type_id, btf_kind_str(local_type), local_name, targ_type_id, btf_kind_str(targ_type), targ_name); @@ -8915,7 +8959,7 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, if (err) return err; pr_debug("extern (kcfg) '%s': set to 0x%llx\n", - ext->name, (long long)value); + ext->name, (unsigned long long)value); } else { pr_warn("extern '%s': unrecognized extern kind\n", ext->name); return -EINVAL; @@ -10494,7 +10538,7 @@ static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, moff = rel->r_offset - map->sec_offset; shdr_idx = sym->st_shndx; st_ops = map->st_ops; - pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", + pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %u (\'%s\')\n", map->name, (long long)(rel->r_info >> 32), (long long)sym->st_value, @@ -10643,7 +10687,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int t memset(&info, 0, info_len); err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len); if (err) { - pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %s\n", + pr_warn("failed bpf_prog_get_info_by_fd for FD %u: %s\n", attach_prog_fd, errstr(err)); return err; } @@ -10656,7 +10700,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int t btf = btf_load_from_kernel(info.btf_id, NULL, token_fd); err = libbpf_get_error(btf); if (err) { - pr_warn("Failed to get BTF %d of the program: %s\n", info.btf_id, errstr(err)); + pr_warn("Failed to get BTF %u of the program: %s\n", info.btf_id, errstr(err)); goto out; } err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); @@ -10738,7 +10782,7 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac } err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd); if (err < 0) { - pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n", + pr_warn("prog '%s': failed to find BPF program (FD %u) BTF ID for '%s': %s\n", prog->name, attach_prog_fd, attach_name, errstr(err)); return err; } @@ -10811,11 +10855,16 @@ static bool map_uses_real_name(const struct bpf_map *map) * such map's corresponding ELF section name as a map name. * This check distinguishes .data/.rodata from .data.* and .rodata.* * maps to know which name has to be returned to the user. + * Map name of the custom .percpu.* maps might be truncated to + * BPF_OBJ_NAME_LEN-1 chars in internal_map_name(). Hence, percpu data + * maps must use real name for their user-visible name. */ if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) return true; if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) return true; + if (map->libbpf_type == LIBBPF_MAP_PERCPU) + return true; return false; } @@ -10980,7 +11029,8 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) size_t mmap_old_sz, mmap_new_sz; int err; - if (map->def.type != BPF_MAP_TYPE_ARRAY) + if (map->def.type != BPF_MAP_TYPE_ARRAY && + map->def.type != BPF_MAP_TYPE_PERCPU_ARRAY) return libbpf_err(-EOPNOTSUPP); mmap_old_sz = bpf_map_mmap_sz(map); @@ -11233,7 +11283,7 @@ static int validate_map_op(const struct bpf_map *map, size_t key_sz, } if (value_sz != num_cpu * elem_sz) { - pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", + pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zu\n", map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); return -EINVAL; } @@ -11774,7 +11824,7 @@ static void gen_probe_legacy_event_name(char *buf, size_t buf_sz, static int index = 0; int i; - snprintf(buf, buf_sz, "libbpf_%u_%d_%s_0x%zx", getpid(), + snprintf(buf, buf_sz, "libbpf_%d_%d_%s_0x%zx", getpid(), __sync_fetch_and_add(&index, 1), name, offset); /* sanitize name in the probe name */ @@ -12924,8 +12974,8 @@ static long elf_find_func_offset_from_archive(const char *archive_path, const ch ret = elf_find_func_offset(elf, file_name, func_name); if (ret > 0) { pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n", - func_name, file_name, archive_path, entry.data_offset, ret, - ret + entry.data_offset); + func_name, file_name, archive_path, entry.data_offset, (unsigned long)ret, + (unsigned long)(ret + entry.data_offset)); ret += entry.data_offset; } elf_end(elf); @@ -12973,13 +13023,14 @@ static const char *arch_specific_lib_paths(void) /* Get full path to program/shared library. */ static int resolve_full_path(const char *file, char *result, size_t result_sz) { - const char *search_paths[3] = {}; + const char *search_paths[4] = {}; int i, perm; if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { search_paths[0] = getenv("LD_LIBRARY_PATH"); search_paths[1] = "/usr/lib64:/usr/lib"; search_paths[2] = arch_specific_lib_paths(); + search_paths[3] = "/lib64:/lib"; perm = R_OK; } else { search_paths[0] = getenv("PATH"); @@ -14167,7 +14218,7 @@ perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, if (((void *)ehdr) + ehdr_size > base + mmap_size) { void *copy_start = ehdr; size_t len_first = base + mmap_size - copy_start; - size_t len_secnd = ehdr_size - len_first; + size_t len_second = ehdr_size - len_first; if (*copy_size < ehdr_size) { free(*copy_mem); @@ -14181,7 +14232,7 @@ perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, } memcpy(*copy_mem, copy_start, len_first); - memcpy(*copy_mem + len_first, base, len_secnd); + memcpy(*copy_mem + len_first, base, len_second); ehdr = *copy_mem; } @@ -14199,7 +14250,7 @@ struct perf_buffer; struct perf_buffer_params { struct perf_event_attr *attr; - /* if event_cb is specified, it takes precendence */ + /* if event_cb is specified, it takes precedence */ perf_buffer_event_fn event_cb; /* sample_cb and lost_cb are higher-level common-case callbacks */ perf_buffer_sample_fn sample_cb; @@ -14570,7 +14621,7 @@ perf_buffer__process_record(struct perf_event_header *e, void *ctx) break; } default: - pr_warn("unknown perf sample type %d\n", e->type); + pr_warn("unknown perf sample type %u\n", e->type); return LIBBPF_PERF_EVENT_ERROR; } return LIBBPF_PERF_EVENT_CONT; diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index b731df19ae69..08ab2ea881fb 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -460,5 +460,6 @@ LIBBPF_1.8.0 { global: bpf_program__attach_tracing_multi; bpf_program__clone; + btf__find_by_name_kind_own; btf__new_empty_opts; } LIBBPF_1.7.0; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 04cd303fb5a8..4c46d34fc055 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -401,6 +401,8 @@ enum kern_feature_id { FEAT_BTF_LAYOUT, /* Kernel supports BPF syscall common attributes */ FEAT_BPF_SYSCALL_COMMON_ATTRS, + /* Kernel supports percpu data */ + FEAT_PERCPU_DATA, __FEAT_CNT, }; @@ -596,8 +598,6 @@ typedef int (*type_id_visit_fn)(__u32 *type_id, void *ctx); typedef int (*str_off_visit_fn)(__u32 *str_off, void *ctx); int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx); int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx); -__s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, - __u32 kind); /* handle direct returned errors */ static inline int libbpf_err(int ret) @@ -768,7 +768,6 @@ int elf_resolve_pattern_offsets(const char *binary_path, const char *pattern, int probe_fd(int fd); #define SHA256_DIGEST_LENGTH 32 -#define SHA256_DWORD_SIZE SHA256_DIGEST_LENGTH / sizeof(__u64) void libbpf_sha256(const void *data, size_t len, __u8 out[SHA256_DIGEST_LENGTH]); int probe_sys_bpf_ext(void); diff --git a/tools/lib/bpf/nlattr.c b/tools/lib/bpf/nlattr.c index 06663f9ea581..007fe17d17b4 100644 --- a/tools/lib/bpf/nlattr.c +++ b/tools/lib/bpf/nlattr.c @@ -123,7 +123,7 @@ int libbpf_nla_parse(struct nlattr *tb[], int maxtype, struct nlattr *head, if (tb[type]) { pr_warn("Attribute of type %#x found multiple times in message, " - "previous attribute is being ignored.\n", type); + "previous attribute is being ignored.\n", (unsigned)type); } tb[type] = nla; diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index 6ae3f2a15ad0..8ad2715721cf 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -216,7 +216,7 @@ recur: goto recur; } default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", + pr_warn("unexpected kind %s relocated, local [%u], target [%u]\n", btf_kind_str(local_type), local_id, targ_id); return 0; } @@ -384,7 +384,7 @@ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, return sz; spec->bit_offset += access_idx * sz * 8; } else { - pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", + pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%u] of unexpected kind %s\n", prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t)); return -EINVAL; } @@ -725,7 +725,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, return -EINVAL; *val = sz; } else { - pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", + pr_warn("prog '%s': relo %u at insn #%u can't be applied to array access\n", prog_name, relo->kind, relo->insn_off / 8); return -EINVAL; } @@ -747,7 +747,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { if (byte_sz >= 8) { /* bitfield can't be read with 64-bit read */ - pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", + pr_warn("prog '%s': relo %u at insn #%u can't be satisfied for bitfield\n", prog_name, relo->kind, relo->insn_off / 8); return -E2BIG; } @@ -971,7 +971,7 @@ done: err = 0; } else if (err == -EOPNOTSUPP) { /* EOPNOTSUPP means unknown/unsupported relocation */ - pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n", + pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%u) at insn #%u\n", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind, relo->insn_off / 8); } @@ -1067,7 +1067,7 @@ poison: if (BPF_SRC(insn->code) != BPF_K) return -EINVAL; if (res->validate && insn->imm != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n", + pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %d, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, insn->imm, (unsigned long long)orig_val, (unsigned long long)new_val); @@ -1083,7 +1083,7 @@ poison: case BPF_ST: case BPF_STX: if (res->validate && insn->off != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n", + pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %d, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val, (unsigned long long)new_val); return -EINVAL; @@ -1159,7 +1159,7 @@ poison: default: pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n", prog_name, relo_idx, insn_idx, insn->code, - insn->src_reg, insn->dst_reg, insn->off, insn->imm); + (unsigned)insn->src_reg, (unsigned)insn->dst_reg, (unsigned)insn->off, (unsigned)insn->imm); return -EINVAL; } @@ -1323,7 +1323,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, const char *spec_str; spec_str = btf__name_by_offset(local_btf, relo->access_str_off); - pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", + pr_warn("prog '%s': relo #%d: parsing [%u] %s %s + %s failed: %d\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), str_is_empty(local_name) ? "<anon>" : local_name, spec_str ?: "<?>", err); @@ -1346,7 +1346,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, /* libbpf doesn't support candidate search for anonymous types */ if (str_is_empty(local_name)) { - pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n", + pr_warn("prog '%s': relo #%d: <%s> (%u) relocation doesn't support anonymous types\n", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); return -EOPNOTSUPP; } @@ -1697,7 +1697,7 @@ recur: goto recur; } default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", + pr_warn("unexpected kind %s relocated, local [%u], target [%u]\n", btf_kind_str(local_t), local_id, targ_id); return 0; } diff --git a/tools/lib/bpf/ringbuf.c b/tools/lib/bpf/ringbuf.c index 00ec4837a06d..c8fe57401a8b 100644 --- a/tools/lib/bpf/ringbuf.c +++ b/tools/lib/bpf/ringbuf.c @@ -244,7 +244,7 @@ static int64_t ringbuf_process_ring(struct ring *r, size_t n) do { got_new_data = false; prod_pos = smp_load_acquire(r->producer_pos); - while (cons_pos < prod_pos) { + while (prod_pos - cons_pos > 0) { len_ptr = r->data + (cons_pos & r->mask); len = smp_load_acquire(len_ptr); diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h index 74503d358bc8..1f3f332dffbe 100644 --- a/tools/lib/bpf/skel_internal.h +++ b/tools/lib/bpf/skel_internal.h @@ -18,10 +18,6 @@ #include "bpf.h" #endif -#ifndef SHA256_DIGEST_LENGTH -#define SHA256_DIGEST_LENGTH 32 -#endif - #ifndef __NR_bpf # if defined(__mips__) && defined(_ABIO32) # define __NR_bpf 4355 @@ -135,8 +131,10 @@ static inline void skel_free_map_data(void *p, __u64 addr, size_t sz) { if (addr != ~0ULL) kvfree(p); - /* When addr == ~0ULL the 'p' points to - * ((struct bpf_array *)map)->value. See skel_finalize_map_data. + /* + * When addr == ~0ULL the init buffer has already been released. + * For skel_finalize_map_data(), 'p' points to + * ((struct bpf_array *)map)->value. */ } @@ -174,6 +172,15 @@ out: return addr; } +static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) +{ + (void)sz; + + kvfree(p); + *init_val = ~0ULL; + return 0; +} + #else static inline void *skel_alloc(size_t size) @@ -212,6 +219,15 @@ static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int return NULL; return addr; } + +static inline int skel_protect_map_data(void *p, __u64 *init_val, size_t sz) +{ + (void)init_val; + + if (mprotect(p, sz, PROT_READ)) + return -errno; + return 0; +} #endif static inline int skel_closenz(int fd) @@ -320,25 +336,6 @@ static inline int skel_link_create(int prog_fd, int target_fd, return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz); } -static inline int skel_obj_get_info_by_fd(int fd) -{ - const size_t attr_sz = offsetofend(union bpf_attr, info); - __u8 sha[SHA256_DIGEST_LENGTH]; - struct bpf_map_info info; - __u32 info_len = sizeof(info); - union bpf_attr attr; - - memset(&info, 0, sizeof(info)); - info.hash = (long) &sha; - info.hash_size = SHA256_DIGEST_LENGTH; - - memset(&attr, 0, attr_sz); - attr.info.bpf_fd = fd; - attr.info.info = (long) &info; - attr.info.info_len = info_len; - return skel_sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz); -} - static inline int skel_map_freeze(int fd) { const size_t attr_sz = offsetofend(union bpf_attr, map_fd); @@ -384,12 +381,6 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) set_err; goto out; } - err = skel_obj_get_info_by_fd(map_fd); - if (err < 0) { - opts->errstr = "failed to fetch obj info"; - set_err; - goto out; - } #endif memset(&attr, 0, prog_load_attr_sz); @@ -400,6 +391,8 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) #ifndef __KERNEL__ attr.signature = (long) opts->signature; attr.signature_size = opts->signature_sz; + if (opts->signature) + attr.fd_array_cnt = 1; #else if (opts->signature || opts->signature_sz) pr_warn("signatures are not supported from bpf_preload\n"); diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index d2ecd3daab96..2e56e3ab5b6c 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -327,7 +327,7 @@ static int sanity_check_usdt_elf(Elf *elf, const char *path) int endianness; if (elf_kind(elf) != ELF_K_ELF) { - pr_warn("usdt: unrecognized ELF kind %d for '%s'\n", elf_kind(elf), path); + pr_warn("usdt: unrecognized ELF kind %u for '%s'\n", elf_kind(elf), path); return -EBADF; } @@ -438,8 +438,9 @@ static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, siz } pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n", - i, path, (long)phdr.p_vaddr, (long)phdr.p_memsz, (long)phdr.p_offset, - (long)phdr.p_type, (long)phdr.p_flags); + i, path, + (unsigned long)phdr.p_vaddr, (unsigned long)phdr.p_memsz, (unsigned long)phdr.p_offset, + (unsigned long)phdr.p_type, (unsigned long)phdr.p_flags); if (phdr.p_type != PT_LOAD) continue; @@ -719,14 +720,14 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n", - usdt_provider, usdt_name, path, usdt_abs_ip); + usdt_provider, usdt_name, path, (unsigned long)usdt_abs_ip); goto err_out; } if (!seg->is_exec) { err = -ESRCH; pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n", - path, seg->start, seg->end, usdt_provider, usdt_name, - usdt_abs_ip); + path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name, + (unsigned long)usdt_abs_ip); goto err_out; } /* translate from virtual address to file offset */ @@ -766,7 +767,7 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n", - usdt_provider, usdt_name, path, usdt_rel_ip); + usdt_provider, usdt_name, path, (unsigned long)usdt_rel_ip); goto err_out; } @@ -775,8 +776,10 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n", usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path, - note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args, - seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0); + (unsigned long)note.loc_addr, (unsigned long)note.base_addr, + (unsigned long)usdt_abs_ip, (unsigned long)usdt_rel_ip, note.args, + (unsigned long)(seg ? seg->start : 0), (unsigned long)(seg ? seg->end : 0), + (unsigned long)(seg ? seg->offset : 0)); /* Adjust semaphore address to be a file offset */ if (note.sema_addr) { @@ -791,14 +794,14 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, if (!seg) { err = -ESRCH; pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n", - usdt_provider, usdt_name, path, note.sema_addr); + usdt_provider, usdt_name, path, (unsigned long)note.sema_addr); goto err_out; } if (seg->is_exec) { err = -ESRCH; pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n", - path, seg->start, seg->end, usdt_provider, usdt_name, - note.sema_addr); + path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name, + (unsigned long)note.sema_addr); goto err_out; } @@ -806,8 +809,8 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n", usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", - path, note.sema_addr, note.base_addr, usdt_sema_off, - seg->start, seg->end, seg->offset); + path, (unsigned long)note.sema_addr, (unsigned long)note.base_addr, (unsigned long)usdt_sema_off, + (unsigned long)seg->start, (unsigned long)seg->end, (unsigned long)seg->offset); } /* Record adjusted addresses and offsets and parse USDT spec */ @@ -1117,7 +1120,7 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct spec_id, usdt_provider, usdt_name, path); } else { pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %s\n", - target->abs_ip, spec_id, usdt_provider, usdt_name, + (unsigned long)target->abs_ip, spec_id, usdt_provider, usdt_name, path, errstr(err)); } goto err_out; diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 87f15f296234..3f74d522f7e7 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -84,7 +84,7 @@ bool scx_bpf_dispatch_vtime_from_dsq___old(struct bpf_iter_scx_dsq *it__iter, st * * Compat macro will be dropped on v6.19 release. */ -int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak; +int bpf_cpumask_populate(struct bpf_cpumask *dst, void *src, size_t src__sz) __ksym __weak; #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz) \ (bpf_ksym_exists(bpf_cpumask_populate) ? \ diff --git a/tools/testing/selftests/bpf/DENYLIST.riscv64 b/tools/testing/selftests/bpf/DENYLIST.riscv64 index 4fc4dfdde293..ca1beae7fe8f 100644 --- a/tools/testing/selftests/bpf/DENYLIST.riscv64 +++ b/tools/testing/selftests/bpf/DENYLIST.riscv64 @@ -1,3 +1,2 @@ # riscv64 deny list for BPF CI and local vmtest exceptions # JIT does not support exceptions -tailcalls/tailcall_bpf2bpf* # JIT does not support mixing bpf2bpf and tailcalls diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index b642ee489ea6..2b2f93dec474 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -324,8 +324,6 @@ TRUNNER_BPFTOOL := $(DEFAULT_BPFTOOL) USE_BOOTSTRAP := "bootstrap/" endif -TEST_GEN_PROGS_EXTENDED += $(TRUNNER_BPFTOOL) - $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(BPFOBJ) TESTING_HELPERS := $(OUTPUT)/testing_helpers.o @@ -466,7 +464,7 @@ endif CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH)) BPF_CFLAGS = -g -Wall -Werror -D__TARGET_ARCH_$(SRCARCH) $(MENDIAN) \ -I$(INCLUDE_DIR) -I$(CURDIR) -I$(APIDIR) \ - -I$(CURDIR)/libarena/include \ + -I$(TOOLSINCDIR) -I$(CURDIR)/libarena/include \ -I$(abspath $(OUTPUT)/../usr/include) \ -std=gnu11 \ -fno-strict-aliasing \ @@ -533,7 +531,7 @@ LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c # Generate both light skeleton and libbpf skeleton for these LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ - kfunc_call_test_subprog.c + kfunc_call_test_subprog.c test_global_percpu_data.c SKEL_BLACKLIST += $$(LSKELS) $$(LSKELS_SIGNED) test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o @@ -934,17 +932,26 @@ $(OUTPUT)/test_verifier: test_verifier.c verifier/tests.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ -# Include find_bit.c to compile xskxceiver. -EXTRA_SRC := $(TOOLSDIR)/lib/find_bit.c prog_tests/test_xsk.c prog_tests/test_xsk.h -$(OUTPUT)/xskxceiver: $(EXTRA_SRC) xskxceiver.c xskxceiver.h $(OUTPUT)/network_helpers.o $(OUTPUT)/xsk.o $(OUTPUT)/xsk_xdp_progs.skel.h $(BPFOBJ) | $(OUTPUT) +# Keep xskxceiver independent from test_progs object dependencies. +$(OUTPUT)/xskxceiver: xskxceiver.c xsk.c network_helpers.c \ + $(TOOLSDIR)/lib/find_bit.c prog_tests/test_xsk.c \ + xskxceiver.h xsk.h network_helpers.h \ + prog_tests/test_xsk.h test_progs.h bpf_util.h \ + $(OUTPUT)/xsk_xdp_progs.skel.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ -$(OUTPUT)/xdp_hw_metadata: xdp_hw_metadata.c $(OUTPUT)/network_helpers.o $(OUTPUT)/xsk.o $(OUTPUT)/xdp_hw_metadata.skel.h | $(OUTPUT) +$(OUTPUT)/xdp_hw_metadata: xdp_hw_metadata.c xsk.c network_helpers.c \ + $(TOOLSDIR)/lib/find_bit.c xdp_metadata.h \ + xsk.h network_helpers.h test_progs.h bpf_util.h \ + $(OUTPUT)/xdp_hw_metadata.skel.h $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ -$(OUTPUT)/xdp_features: xdp_features.c $(OUTPUT)/network_helpers.o $(OUTPUT)/xdp_features.skel.h | $(OUTPUT) +$(OUTPUT)/xdp_features: xdp_features.c network_helpers.c xdp_features.h \ + network_helpers.h \ + test_progs.h bpf_util.h $(OUTPUT)/xdp_features.skel.h \ + $(BPFOBJ) | $(OUTPUT) $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@ @@ -967,6 +974,7 @@ $(OUTPUT)/bench_ringbufs.o: $(OUTPUT)/ringbuf_bench.skel.h \ $(OUTPUT)/perfbuf_bench.skel.h $(OUTPUT)/bench_bloom_filter_map.o: $(OUTPUT)/bloom_filter_bench.skel.h $(OUTPUT)/bench_bpf_loop.o: $(OUTPUT)/bpf_loop_bench.skel.h +$(OUTPUT)/bench_bpf_for.o: $(OUTPUT)/bpf_for_bench.skel.h $(OUTPUT)/bench_strncmp.o: $(OUTPUT)/strncmp_bench.skel.h $(OUTPUT)/bench_bpf_hashmap_full_update.o: $(OUTPUT)/bpf_hashmap_full_update_bench.skel.h $(OUTPUT)/bench_local_storage.o: $(OUTPUT)/local_storage_bench.skel.h @@ -992,6 +1000,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/bench_ringbufs.o \ $(OUTPUT)/bench_bloom_filter_map.o \ $(OUTPUT)/bench_bpf_loop.o \ + $(OUTPUT)/bench_bpf_for.o \ $(OUTPUT)/bench_strncmp.o \ $(OUTPUT)/bench_bpf_hashmap_full_update.o \ $(OUTPUT)/bench_local_storage.o \ @@ -1055,10 +1064,13 @@ endif DEFAULT_INSTALL_RULE := $(INSTALL_RULE) override define INSTALL_RULE $(DEFAULT_INSTALL_RULE) + @mkdir -p $(INSTALL_PATH)/tools/sbin + @rsync -a $(if $(PERMISSIVE),--ignore-missing-args) $(TRUNNER_BPFTOOL) $(INSTALL_PATH)/tools/sbin/ + @rsync -a $(if $(PERMISSIVE),--ignore-missing-args) $(OUTPUT)/*.BTF $(INSTALL_PATH)/ @for DIR in $(TEST_INST_SUBDIRS); do \ mkdir -p $(INSTALL_PATH)/$$DIR; \ rsync -a $(if $(PERMISSIVE),--ignore-missing-args) \ - $(OUTPUT)/$$DIR/*.bpf.o \ + $(OUTPUT)/$$DIR/*.bpf.o $(OUTPUT)/$$DIR/*.BTF \ $(INSTALL_PATH)/$$DIR; \ done endef diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 37164322a102..07c834433b38 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -107,12 +107,12 @@ Docker container and local rootfs image. The overall steps are as follows: tools/testing/selftests/bpf/vmtest.sh \ -l <path of local rootfs image> -- \ ./test_progs -d \ - \"$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ + "$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ | cut -d'#' -f1 \ | sed -e 's/^[[:space:]]*//' \ -e 's/[[:space:]]*$//' \ | tr -s '\n' ',' \ - )\" + )" Link: https://github.com/pulehui/riscv-bpf-vmtest.git [0] Link: https://github.com/libbpf/ci/blob/main/rootfs/mkrootfs_debian.sh [1] diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index c4a3a6b3eb83..465233ea5f79 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -276,6 +276,7 @@ static const struct argp_option opts[] = { extern struct argp bench_ringbufs_argp; extern struct argp bench_bloom_map_argp; extern struct argp bench_bpf_loop_argp; +extern struct argp bench_bpf_for_argp; extern struct argp bench_local_storage_argp; extern struct argp bench_local_storage_rcu_tasks_trace_argp; extern struct argp bench_strncmp_argp; @@ -292,6 +293,7 @@ static const struct argp_child bench_parsers[] = { { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 }, { &bench_bloom_map_argp, 0, "Bloom filter map benchmark", 0 }, { &bench_bpf_loop_argp, 0, "bpf_loop helper benchmark", 0 }, + { &bench_bpf_for_argp, 0, "bpf_for loop benchmark", 0 }, { &bench_local_storage_argp, 0, "local_storage benchmark", 0 }, { &bench_strncmp_argp, 0, "bpf_strncmp helper benchmark", 0 }, { &bench_local_storage_rcu_tasks_trace_argp, 0, @@ -557,6 +559,7 @@ extern const struct bench bench_bloom_false_positive; extern const struct bench bench_hashmap_without_bloom; extern const struct bench bench_hashmap_with_bloom; extern const struct bench bench_bpf_loop; +extern const struct bench bench_bpf_for; extern const struct bench bench_strncmp_no_helper; extern const struct bench bench_strncmp_helper; extern const struct bench bench_bpf_hashmap_full_update; @@ -640,6 +643,7 @@ static const struct bench *benchs[] = { &bench_hashmap_without_bloom, &bench_hashmap_with_bloom, &bench_bpf_loop, + &bench_bpf_for, &bench_strncmp_no_helper, &bench_strncmp_helper, &bench_bpf_hashmap_full_update, diff --git a/tools/testing/selftests/bpf/benchs/bench_bpf_for.c b/tools/testing/selftests/bpf/benchs/bench_bpf_for.c new file mode 100644 index 000000000000..730c51ad2dec --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_bpf_for.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <argp.h> +#include "bench.h" +#include "bpf_for_bench.skel.h" + +/* BPF triggering benchmarks */ +static struct ctx { + struct bpf_for_bench *skel; +} ctx; + +static struct { + __u32 nr_loops; +} args = { + /* + * Default to a large loop count so the per-iteration bpf_iter_num_next() cost dominates + * the one-time bpf_iter_num_new()/destroy() setup and teardown. + */ + .nr_loops = 1000, +}; + +enum { + ARG_NR_LOOPS = 4000, +}; + +static const struct argp_option opts[] = { + { "nr_loops", ARG_NR_LOOPS, "nr_loops", 0, + "Set number of iterations for the bpf_for() loop"}, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + switch (key) { + case ARG_NR_LOOPS: + args.nr_loops = strtol(arg, NULL, 10); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +/* exported into benchmark runner */ +const struct argp bench_bpf_for_argp = { + .options = opts, + .parser = parse_arg, +}; + +static void validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumer!\n"); + exit(1); + } +} + +static void *producer(void *input) +{ + while (true) + /* trigger the bpf program */ + syscall(__NR_getpgid); + + return NULL; +} + +static void measure(struct bench_res *res) +{ + res->hits = atomic_swap(&ctx.skel->bss->hits, 0); +} + +static void setup(void) +{ + struct bpf_link *link; + + setup_libbpf(); + + ctx.skel = bpf_for_bench__open_and_load(); + if (!ctx.skel) { + fprintf(stderr, "failed to open skeleton\n"); + exit(1); + } + + link = bpf_program__attach(ctx.skel->progs.benchmark); + if (!link) { + fprintf(stderr, "failed to attach program!\n"); + exit(1); + } + + ctx.skel->bss->nr_loops = args.nr_loops; +} + +const struct bench bench_bpf_for = { + .name = "bpf-for", + .argp = &bench_bpf_for_argp, + .validate = validate, + .setup = setup, + .producer_thread = producer, + .measure = measure, + .report_progress = ops_report_progress, + .report_final = ops_report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh b/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh new file mode 100755 index 000000000000..7da6453920da --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_bpf_for.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +for t in 1 4 8 12 16; do +for i in 10 100 500 1000 5000 10000 50000 100000 500000 1000000; do +subtitle "nr_loops: $i, nr_threads: $t" + summarize_ops "bpf_for: " \ + "$($RUN_BENCH -p $t --nr_loops $i bpf-for)" + printf "\n" +done +done diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h index f30a4ac939db..2893bf06ff25 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -431,6 +431,8 @@ static inline int get_preempt_count(void) return bpf_get_lowcore()->preempt_count; #elif defined(bpf_target_loongarch) return bpf_get_current_task_btf()->thread_info.preempt_count; +#elif defined(bpf_target_riscv) + return bpf_get_current_task_btf()->thread_info.preempt_count; #endif return 0; } @@ -442,6 +444,7 @@ static inline int get_preempt_count(void) * * powerpc64 * * s390x * * loongarch + * * riscv */ static inline int bpf_in_interrupt(void) { @@ -464,6 +467,7 @@ static inline int bpf_in_interrupt(void) * * powerpc64 * * s390x * * loongarch + * * riscv */ static inline int bpf_in_nmi(void) { @@ -477,6 +481,7 @@ static inline int bpf_in_nmi(void) * * powerpc64 * * s390x * * loongarch + * * riscv */ static inline int bpf_in_hardirq(void) { @@ -490,6 +495,7 @@ static inline int bpf_in_hardirq(void) * * powerpc64 * * s390x * * loongarch + * * riscv */ static inline int bpf_in_serving_softirq(void) { @@ -511,6 +517,7 @@ static inline int bpf_in_serving_softirq(void) * * powerpc64 * * s390x * * loongarch + * * riscv */ static inline int bpf_in_task(void) { diff --git a/tools/testing/selftests/bpf/config.aarch64 b/tools/testing/selftests/bpf/config.aarch64 index 7efad36ceb26..fc85257701dc 100644 --- a/tools/testing/selftests/bpf/config.aarch64 +++ b/tools/testing/selftests/bpf/config.aarch64 @@ -71,7 +71,6 @@ CONFIG_INPUT_EVDEV=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPVLAN=y CONFIG_JUMP_LABEL=y CONFIG_KERNEL_UNCOMPRESSED=y diff --git a/tools/testing/selftests/bpf/config.ppc64el b/tools/testing/selftests/bpf/config.ppc64el index b53afb5e0b71..5685fa4ee82b 100644 --- a/tools/testing/selftests/bpf/config.ppc64el +++ b/tools/testing/selftests/bpf/config.ppc64el @@ -39,7 +39,6 @@ CONFIG_INET=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_JUMP_LABEL=y CONFIG_KALLSYMS_ALL=y CONFIG_KPROBES=y diff --git a/tools/testing/selftests/bpf/config.riscv64 b/tools/testing/selftests/bpf/config.riscv64 index 7bee24a79a71..655cb05a7689 100644 --- a/tools/testing/selftests/bpf/config.riscv64 +++ b/tools/testing/selftests/bpf/config.riscv64 @@ -30,7 +30,6 @@ CONFIG_HARDLOCKUP_DETECTOR=y CONFIG_HIGH_RES_TIMERS=y CONFIG_HUGETLBFS=y CONFIG_INET=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y diff --git a/tools/testing/selftests/bpf/config.s390x b/tools/testing/selftests/bpf/config.s390x index db61878148e4..755d1cfcd9e0 100644 --- a/tools/testing/selftests/bpf/config.s390x +++ b/tools/testing/selftests/bpf/config.s390x @@ -56,7 +56,6 @@ CONFIG_INET=y CONFIG_IP_ADVANCED_ROUTER=y CONFIG_IP_MULTICAST=y CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPVLAN=y CONFIG_JUMP_LABEL=y CONFIG_KERNEL_UNCOMPRESSED=y diff --git a/tools/testing/selftests/bpf/config.x86_64 b/tools/testing/selftests/bpf/config.x86_64 index 42ad817b00ae..523e0d29bbd4 100644 --- a/tools/testing/selftests/bpf/config.x86_64 +++ b/tools/testing/selftests/bpf/config.x86_64 @@ -114,7 +114,6 @@ CONFIG_IP_ROUTE_VERBOSE=y CONFIG_IPV6_MIP6=y CONFIG_IPV6_ROUTE_INFO=y CONFIG_IPV6_ROUTER_PREF=y -CONFIG_IPV6_SEG6_LWTUNNEL=y CONFIG_IPV6_SUBTREES=y CONFIG_IRQ_POLL=y CONFIG_JUMP_LABEL=y diff --git a/tools/testing/selftests/bpf/disasm_helpers.c b/tools/testing/selftests/bpf/disasm_helpers.c index f529f1c8c171..30221352568d 100644 --- a/tools/testing/selftests/bpf/disasm_helpers.c +++ b/tools/testing/selftests/bpf/disasm_helpers.c @@ -55,10 +55,9 @@ struct bpf_insn *disasm_insn(struct bpf_insn *insn, char *buf, size_t buf_sz) * for each instruction (FF stands for instruction `code` byte). * Remove the prefix inplace, and also simplify call instructions. * E.g.: "(85) call foo#10" -> "call foo". - * Also remove newline in the end (the 'max(strlen(buf) - 1, 0)' thing). */ pfx_end = buf + 5; - sfx_start = buf + max((int)strlen(buf) - 1, 0); + sfx_start = buf + (int)strlen(buf); if (strncmp(pfx_end, "call ", 5) == 0 && (tmp = strrchr(buf, '#'))) sfx_start = tmp; len = sfx_start - pfx_end; diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h index ae6b72d15bb6..71d9db610263 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h @@ -103,12 +103,7 @@ struct arena_qnode { #define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET) #define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET) -/* - * The qnodes are marked __weak so we can define them in the header - * while still ensuring all compilation units use the same struct - * instance. - */ -struct arena_qnode __weak __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; +extern struct arena_qnode __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; static inline u32 encode_tail(int cpu, int idx) { diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h index b7b230431929..43c306e17f19 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h @@ -86,6 +86,25 @@ extern bool CONFIG_X86_64 __kconfig __weak; /* Control dependency provides LOAD->STORE, provide LOAD->LOAD */ #define smp_acquire__after_ctrl_dep() ({ smp_rmb(); }) +#if defined(__BPF_FEATURE_LOAD_ACQ_STORE_REL) +/* + * Clang advertises this feature when it can lower acquire/release atomic + * builtins to BPF_LOAD_ACQ/BPF_STORE_REL. Older compilers keep using the + * barrier-based fallback below. The generated instructions require kernel + * verifier/JIT support added in Linux 6.15; compile for an older BPF CPU to + * keep using the fallback when targeting older kernels. + */ +#define smp_load_acquire(p) \ + ({ \ + __unqual_typeof(*(p)) ___p1 = __atomic_load_n((p), __ATOMIC_ACQUIRE); \ + (typeof(*(p)))___p1; \ + }) + +#define smp_store_release(p, val) \ + ({ \ + __atomic_store_n((p), (val), __ATOMIC_RELEASE); \ + }) +#else #define smp_load_acquire(p) \ ({ \ __unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \ @@ -102,6 +121,7 @@ extern bool CONFIG_X86_64 __kconfig __weak; barrier(); \ WRITE_ONCE(*(p), val); \ }) +#endif #define smp_cond_load_relaxed_label(p, cond_expr, label) \ ({ \ diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h new file mode 100644 index 000000000000..e2431ea6fdd6 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h @@ -0,0 +1,34 @@ +#pragma once + +#define BITS_PER_BYTE 8 +#define BYTES_TO_BITS(nb) ((nb) * BITS_PER_BYTE) + +#define BITS_PER_LONG_LONG (sizeof(long long) * BITS_PER_BYTE) +#define BITS_TO_LONG_LONGS(nr) (((nr) + BITS_PER_LONG_LONG - 1) / BITS_PER_LONG_LONG) +#define BIT_MASK(nr) (1ULL << ((nr) % BITS_PER_LONG_LONG)) +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG_LONG) + +struct arena_bitmap { + u64 bits[0]; +}; + +struct arena_bitmap __arena *bmp_alloc(size_t bits); +void bmp_free(struct arena_bitmap __arena *bmp); + +void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); +void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); +void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); +void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); +bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp); +bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); +bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp); + +void bmp_clear(size_t bits, struct arena_bitmap __arena *bmp); +void bmp_and(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2); +void bmp_or(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2); +bool bmp_empty(size_t bits, struct arena_bitmap __arena *bmp); +void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src); + +bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2); +bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small); +void bmp_print(size_t bits, struct arena_bitmap __arena *bmp); diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/common.h b/tools/testing/selftests/bpf/libarena/include/libarena/common.h index a3eb1641ac36..931ace9a49e2 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/common.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/common.h @@ -43,7 +43,7 @@ struct { * imprecise. To force the variable to be imprecise, initialize it with * the opaque volatile variable 0 instead of the constant 0. */ -extern const volatile u32 zero; +volatile u32 zero __weak; extern volatile u64 asan_violated; int arena_fls(__u64 word); diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h b/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h index fc27a4bcf5d7..b6676dd67bc0 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/userspace.h @@ -115,7 +115,7 @@ static inline int libarena_asan_init(int arena_asan_init_fd, { LIBBPF_OPTS(bpf_test_run_opts, opts); struct asan_init_args args; - u64 globals_pages; + u64 globals_pages = 0; int ret; ret = libarena_get_globals_pages(arena_asan_init_fd, diff --git a/tools/testing/selftests/bpf/libarena/selftests/st_asan_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_asan_buddy.bpf.c index 686caba2c643..3266a28f53d7 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/st_asan_buddy.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_asan_buddy.bpf.c @@ -12,7 +12,7 @@ extern struct buddy __arena buddy; #ifdef BPF_ARENA_ASAN -#include "st_asan_common.h" +#include "test_asan_common.h" static __always_inline int asan_test_buddy_oob_single(size_t alloc_size) { @@ -154,7 +154,8 @@ __weak int asan_test_buddy_oob(void) size_t sizes[] = { 7, 8, 17, 18, 64, 256, 317, 512, 1024, }; - int ret, i; + int ret; + u32 i; ret = buddy_init(&buddy); if (ret) { @@ -163,6 +164,7 @@ __weak int asan_test_buddy_oob(void) } for (i = zero; i < sizeof(sizes) / sizeof(sizes[0]) && can_loop; i++) { + barrier_var(i); ret = asan_test_buddy_oob_single(sizes[i]); if (ret) { arena_stdout("%s:%d Failed for size %lu", __func__, @@ -190,7 +192,8 @@ __stderr("Call trace:\n" __weak int asan_test_buddy_uaf(void) { size_t sizes[] = { 16, 32, 64, 128, 256, 512, 1024, 16384 }; - int ret, i; + int ret; + u32 i; ret = buddy_init(&buddy); if (ret) { @@ -199,6 +202,7 @@ __weak int asan_test_buddy_uaf(void) } for (i = zero; i < sizeof(sizes) / sizeof(sizes[0]) && can_loop; i++) { + barrier_var(i); ret = asan_test_buddy_uaf_single(sizes[i]); if (ret) { arena_stdout("%s:%d Failed for size %lu", __func__, diff --git a/tools/testing/selftests/bpf/libarena/selftests/st_asan_common.h b/tools/testing/selftests/bpf/libarena/selftests/test_asan_common.h index 34a7918cb4cf..34a7918cb4cf 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/st_asan_common.h +++ b/tools/testing/selftests/bpf/libarena/selftests/test_asan_common.h diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c new file mode 100644 index 000000000000..76319a529f02 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c @@ -0,0 +1,394 @@ +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/bitmap.h> + +#define TEST_BITS (2 * BITS_PER_LONG_LONG) +#define TEST_WORDS BITS_TO_LONG_LONGS(TEST_BITS) +#define MID_BIT (BITS_PER_LONG_LONG + 1) +#define LAST_BIT (TEST_BITS - 1) + +static void test_bmp_setall(struct arena_bitmap __arena *bmp) +{ + volatile u32 i; + + for (i = zero; i < TEST_WORDS && can_loop; i++) + bmp->bits[i] = ~0ULL; +} + +SEC("syscall") +__weak int test_bitmap_alloc_free(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + if (!bmp_empty(TEST_BITS, bmp)) + goto err; + + __bmp_set_bit(LAST_BIT, bmp); + if (!bmp_test_bit(LAST_BIT, bmp)) + goto err; + + __bmp_clear_bit(LAST_BIT, bmp); + if (bmp_test_bit(LAST_BIT, bmp)) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_bit_ops(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + __bmp_set_bit(0, bmp); + if (!bmp_test_bit(0, bmp)) + goto err; + + __bmp_set_bit(MID_BIT, bmp); + if (!bmp_test_bit(MID_BIT, bmp)) + goto err; + + __bmp_set_bit(LAST_BIT, bmp); + if (!bmp_test_bit(LAST_BIT, bmp)) + goto err; + + if (bmp_test_bit(MID_BIT - 1, bmp)) + goto err; + + __bmp_clear_bit(MID_BIT, bmp); + if (bmp_test_bit(MID_BIT, bmp)) + goto err; + + if (!bmp_test_bit(0, bmp)) + goto err; + + if (!bmp_test_bit(LAST_BIT, bmp)) + goto err; + + __bmp_clear_bit(0, bmp); + __bmp_clear_bit(LAST_BIT, bmp); + if (!bmp_empty(TEST_BITS, bmp)) + goto err; + + if (bmp->bits[0]) + goto err; + + if (bmp->bits[1]) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + +static bool test_bitmap_test_and_clear_single(struct arena_bitmap __arena *bmp, size_t ind) +{ + if (bmp_test_and_clear_bit(ind, bmp)) + return false; + + __bmp_set_bit(ind, bmp); + + if (!bmp_test_and_clear_bit(ind, bmp)) + return false; + + if (bmp_test_bit(ind, bmp)) + return false; + + if (bmp_test_and_clear_bit(ind, bmp)) + return false; + + return true; +} + +static bool test_bitmap_test_and_set_single(struct arena_bitmap __arena *bmp, size_t ind) +{ + if (bmp_test_and_set_bit(ind, bmp)) + return false; + + if (!bmp_test_and_set_bit(ind, bmp)) + return false; + + if (!bmp_test_bit(ind, bmp)) + return false; + + __bmp_clear_bit(ind, bmp); + + if (bmp_test_and_set_bit(ind, bmp)) + return false; + + return true; +} + +SEC("syscall") +__weak int test_bitmap_test_and_clear_bit(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + if (!test_bitmap_test_and_clear_single(bmp, 0)) + goto err; + + if (!test_bitmap_test_and_clear_single(bmp, MID_BIT)) + goto err; + + if (!test_bitmap_test_and_clear_single(bmp, LAST_BIT)) + goto err; + + if (!bmp_empty(TEST_BITS, bmp)) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_test_and_set_bit(void) +{ + struct arena_bitmap __arena *bmp; + + bmp = bmp_alloc(TEST_BITS); + if (!bmp) + return -ENOMEM; + + if (!test_bitmap_test_and_set_single(bmp, 0)) + goto err; + + if (!test_bitmap_test_and_set_single(bmp, MID_BIT)) + goto err; + + if (!test_bitmap_test_and_set_single(bmp, LAST_BIT)) + goto err; + + bmp_free(bmp); + return 0; + +err: + bmp_free(bmp); + return -EINVAL; +} + + +SEC("syscall") +__weak int test_bitmap_and(void) +{ + struct arena_bitmap __arena *src1 = NULL, *src2 = NULL, *dst = NULL; + + src1 = bmp_alloc(TEST_BITS); + src2 = bmp_alloc(TEST_BITS); + dst = bmp_alloc(TEST_BITS); + if (!src1 || !src2 || !dst) + goto err; + + test_bmp_setall(dst); + + __bmp_set_bit(0, src1); + __bmp_set_bit(MID_BIT, src1); + __bmp_set_bit(LAST_BIT, src1); + + __bmp_set_bit(MID_BIT, src2); + __bmp_set_bit(LAST_BIT, src2); + + bmp_and(TEST_BITS, dst, src1, src2); + + if (bmp_test_bit(0, dst)) + goto err; + if (!bmp_test_bit(MID_BIT, dst)) + goto err; + if (!bmp_test_bit(LAST_BIT, dst)) + goto err; + + if (dst->bits[0]) + goto err; + if (dst->bits[1] != (BIT_MASK(MID_BIT) | BIT_MASK(LAST_BIT))) + goto err; + + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return 0; + +err: + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_or(void) +{ + struct arena_bitmap __arena *src1 = NULL, *src2 = NULL, *dst = NULL; + + src1 = bmp_alloc(TEST_BITS); + src2 = bmp_alloc(TEST_BITS); + dst = bmp_alloc(TEST_BITS); + if (!src1 || !src2 || !dst) + goto err; + + test_bmp_setall(dst); + + __bmp_set_bit(0, src1); + __bmp_set_bit(LAST_BIT, src1); + + __bmp_set_bit(MID_BIT, src2); + __bmp_set_bit(LAST_BIT, src2); + + bmp_or(TEST_BITS, dst, src1, src2); + + if (!bmp_test_bit(0, dst)) + goto err; + if (!bmp_test_bit(MID_BIT, dst)) + goto err; + if (!bmp_test_bit(LAST_BIT, dst)) + goto err; + + if (dst->bits[0] != BIT_MASK(0)) + goto err; + if (dst->bits[1] != (BIT_MASK(MID_BIT) | BIT_MASK(LAST_BIT))) + goto err; + + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return 0; + +err: + bmp_free(src1); + bmp_free(src2); + bmp_free(dst); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_subset(void) +{ + struct arena_bitmap __arena *big = NULL, *small = NULL; + + big = bmp_alloc(TEST_BITS); + small = bmp_alloc(TEST_BITS); + if (!big || !small) + goto err; + + if (!bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(0, small); + if (bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(0, big); + if (!bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(LAST_BIT, small); + if (bmp_subset(TEST_BITS, big, small)) + goto err; + + __bmp_set_bit(LAST_BIT, big); + __bmp_set_bit(MID_BIT, big); + if (!bmp_subset(TEST_BITS, big, small)) + goto err; + + if (bmp_subset(TEST_BITS, small, big)) + goto err; + + bmp_free(big); + bmp_free(small); + return 0; + +err: + bmp_free(big); + bmp_free(small); + return -EINVAL; + +} + +SEC("syscall") +__weak int test_bitmap_intersects(void) +{ + struct arena_bitmap __arena *arg1 = NULL, *arg2 = NULL; + + arg1 = bmp_alloc(TEST_BITS); + arg2 = bmp_alloc(TEST_BITS); + if (!arg1 || !arg2) + goto err; + + if (bmp_intersects(TEST_BITS, arg1, arg2)) + goto err; + + __bmp_set_bit(0, arg1); + __bmp_set_bit(MID_BIT, arg2); + if (bmp_intersects(TEST_BITS, arg1, arg2)) + goto err; + + __bmp_set_bit(LAST_BIT, arg1); + __bmp_set_bit(LAST_BIT, arg2); + if (!bmp_intersects(TEST_BITS, arg1, arg2)) + goto err; + + bmp_free(arg1); + bmp_free(arg2); + return 0; + +err: + bmp_free(arg1); + bmp_free(arg2); + return -EINVAL; +} + +SEC("syscall") +__weak int test_bitmap_copy(void) +{ + struct arena_bitmap __arena *arg1 = NULL, *arg2 = NULL; + + arg1 = bmp_alloc(TEST_BITS); + arg2 = bmp_alloc(TEST_BITS); + if (!arg1 || !arg2) + goto err; + + __bmp_set_bit(0, arg1); + __bmp_set_bit(MID_BIT, arg1); + + /* Make sure those get overwritten. */ + __bmp_set_bit(1, arg2); + __bmp_set_bit(MID_BIT + 2, arg2); + + bmp_copy(TEST_BITS, arg2, arg1); + + /* Bitmaps are equal if a subset of each other. */ + if (!bmp_subset(TEST_BITS, arg1, arg2) || + !bmp_subset(TEST_BITS, arg2, arg1)) + goto err; + + bmp_free(arg1); + bmp_free(arg2); + return 0; + +err: + bmp_free(arg1); + bmp_free(arg2); + return -EINVAL; +} diff --git a/tools/testing/selftests/bpf/libarena/selftests/st_buddy.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c index b45a306816c0..5628f0987012 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/st_buddy.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_buddy.bpf.c @@ -171,7 +171,8 @@ __weak int test_buddy_alloc_multiple(void) SEC("syscall") __weak int test_buddy_alignment(void) { - int ret, i; + int ret; + u32 i; ret = buddy_init(&buddy); if (ret) @@ -179,6 +180,7 @@ __weak int test_buddy_alignment(void) /* Allocate various sizes and check alignment */ for (i = zero; i < 17 && can_loop; i++) { + barrier_var(i); ptrs[i] = buddy_alloc(&buddy, alignment_sizes[i]); if (!ptrs[i]) { arena_stdout("alignment test: alloc failed for size %lu", @@ -198,8 +200,10 @@ __weak int test_buddy_alignment(void) } /* Free all allocations */ - for (i = zero; i < 17 && can_loop; i++) + for (i = zero; i < 17 && can_loop; i++) { + barrier_var(i); buddy_free(&buddy, ptrs[i]); + } buddy_destroy(&buddy); diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c new file mode 100644 index 000000000000..ea1fac95b461 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause + +#include <bpf_atomic.h> + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/bitmap.h> + +#define TEST_BITMAP_THREADS 2 +#define TEST_BITMAP_BITS (2 * BITS_PER_LONG_LONG) +#define TEST_BITMAP_SYNC_SPINS BPF_MAX_LOOPS +#define TEST_BITMAP_ITERS 10 * 1000 * 1000 + +static struct arena_bitmap __arena *bitmap; +static volatile u64 started; +static volatile bool test_abort; + +/* + * The test needs cmpxchg atomics on arena memory. + */ +#if defined(ENABLE_ATOMICS_TESTS) && \ + (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ + defined(__TARGET_ARCH_s390) || \ + defined(__TARGET_ARCH_powerpc) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) +static bool bitmap_tests_enabled(void) +{ + return true; +} +#else +static bool bitmap_tests_enabled(void) +{ + return false; +} +#endif + +__weak +int bitmap_wait_for_start(void) +{ + u64 i; + + __sync_fetch_and_add(&started, 1); + + for (i = zero; i < TEST_BITMAP_SYNC_SPINS && can_loop; i++) { + if (test_abort) + return -EINTR; + if (smp_load_acquire(&started) >= TEST_BITMAP_THREADS) + return 0; + } + + test_abort = true; + return -ETIMEDOUT; +} + +/* + * The test makes sure writes don't clobber each other by overwriting + * the same word. One thread always writes on even bits, the other on + * odds. Both should be able to operate on the bitmap oblivious of the + * other's operations. + */ +__weak +int bitmap_test_bit_sequence(u32 bit) +{ + if (bmp_test_and_clear_bit(bit, bitmap)) + return -EINVAL; + + if (bmp_test_and_set_bit(bit, bitmap)) + return -EINVAL; + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + if (!bmp_test_and_set_bit(bit, bitmap)) + return -EINVAL; + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + if (!bmp_test_and_clear_bit(bit, bitmap)) + return -EINVAL; + if (bmp_test_bit(bit, bitmap)) + return -EINVAL; + + if (bmp_test_and_clear_bit(bit, bitmap)) + return -EINVAL; + + bmp_set_bit(bit, bitmap); + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + bmp_clear_bit(bit, bitmap); + if (bmp_test_bit(bit, bitmap)) + return -EINVAL; + + bmp_set_bit(bit, bitmap); + if (!bmp_test_bit(bit, bitmap)) + return -EINVAL; + + return 0; + +} + +static void bitmap_test_reset_single(int parity) +{ + u32 bit; + + for (bit = parity; bit < TEST_BITMAP_BITS && can_loop; bit += 2) + bmp_clear_bit(bit, bitmap); + +} + +static int bitmap_test_common_single(int parity) +{ + u32 bit; + int ret; + + for (bit = parity; bit < TEST_BITMAP_BITS && can_loop; bit += 2) { + if (test_abort) + return -EINTR; + + ret = bitmap_test_bit_sequence(bit); + if (ret) { + test_abort = true; + return ret; + } + } + + return 0; +} + +static int bitmap_test_common(int parity) +{ + int ret; + u32 i; + + arena_subprog_init(); + + ret = bitmap_wait_for_start(); + if (ret) + return ret; + + for (i = zero; i < TEST_BITMAP_ITERS && can_loop; i++) { + ret = bitmap_test_common_single(parity); + if (ret) + return ret; + + if (test_abort) + break; + + bitmap_test_reset_single(parity); + } + + return 0; +} + +SEC("syscall") int parallel_test_bitmap__enabled(void) +{ + return bitmap_tests_enabled() ? 0 : -EOPNOTSUPP; +} + +SEC("syscall") int parallel_test_bitmap__init(void) +{ + bitmap = bmp_alloc(TEST_BITMAP_BITS); + if (!bitmap) + return -ENOMEM; + + return 0; +} + +SEC("syscall") int parallel_test_bitmap__fini(void) +{ + int ret = 0; + + if (!bitmap) + return -EINVAL; + + bmp_free(bitmap); + bitmap = NULL; + + return ret; +} + +SEC("syscall") int parallel_test_bitmap__0(void) +{ + return bitmap_test_common(0); +} + +SEC("syscall") int parallel_test_bitmap__1(void) +{ + return bitmap_test_common(1); +} diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c index f08f2a92e194..5fa96eb74095 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_spmc.bpf.c @@ -155,7 +155,7 @@ int spmc_quiesce_on_owner(u64 epoch) { u64 i; - bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) { + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { if (test_abort) return -EINTR; if (smp_load_acquire(&owner_epoch) >= epoch) @@ -175,8 +175,7 @@ int spmc_quiesce_on_stealer(u64 epoch) int err = -ETIMEDOUT; target = STEALER_EPOCH(epoch); - bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) { - + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { if (test_abort) { err = -EINTR; break; @@ -391,7 +390,7 @@ int spmc_wait_for_stealers_to_start(u64 target) { u64 i; - bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) { + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { if (test_abort) return -EINTR; if (READ_ONCE(stealers_started) >= target) @@ -537,7 +536,7 @@ static int spmc_wait_for_round_steals(u64 target) arena_subprog_init(); - bpf_for(i, 0, TEST_SPMC_SYNC_SPINS) { + for (i = zero; i < TEST_SPMC_SYNC_SPINS && can_loop; i++) { if (test_abort) return -EINTR; if (round_steals >= target) diff --git a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c new file mode 100644 index 000000000000..5ff8e688ddc7 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* + * Copyright (c) 2025-2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2025-2026 Emil Tsalapatis <emil@etsalapatis.com> + */ + +#include <libarena/common.h> + +#include <libarena/asan.h> +#include <libarena/bitmap.h> + +__weak +struct arena_bitmap __arena *bmp_alloc(size_t bits) +{ + struct arena_bitmap __arena *bmp; + size_t size = BITS_TO_LONG_LONGS(bits) * sizeof(bmp->bits[0]); + + /* Assume long-aligned masks. */ + if (bits % BITS_PER_LONG_LONG) + return NULL; + + bmp = (struct arena_bitmap __arena *)arena_malloc(size); + if (!bmp) + return NULL; + + bmp_clear(bits, bmp); + + return bmp; +} + +__weak +void bmp_free(struct arena_bitmap __arena *bmp) +{ + arena_free(bmp); +} + +__weak +void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + bmp->bits[BIT_WORD(bit)] |= BIT_MASK(bit); +} + +__weak +void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + bmp->bits[BIT_WORD(bit)] &= ~BIT_MASK(bit); +} + +__weak +bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + return bmp->bits[BIT_WORD(bit)] & BIT_MASK(bit); +} + +__weak +bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + + if (!(old & val)) + return false; + + new = old & ~val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + if (actual == old) + return true; + + } while (can_loop); + + return false; +} + +__weak +bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + + if ((old & val)) + return true; + + new = old | val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + if (actual == old) + return false; + + } while (can_loop); + + return false; +} + +__weak +void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + new = old & ~val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + } while (actual != old && can_loop); +} + +__weak +void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 val = BIT_MASK(bit); + u32 idx = BIT_WORD(bit); + u64 old, new, actual; + + do { + old = bmp->bits[idx]; + new = old | val; + actual = cmpxchg(&bmp->bits[idx], old, new); + + } while (actual != old && can_loop); +} + +__weak +void bmp_clear(size_t bits, struct arena_bitmap __arena *bmp) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + bmp->bits[i] = 0; +} + +static __always_inline u64 bmp_last_word_mask(size_t bits) +{ + u32 rem = bits % BITS_PER_LONG_LONG; + + return rem ? (1ULL << rem) - 1 : ~0ULL; +} + +__weak +void bmp_and(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + dst->bits[i] = src1->bits[i] & src2->bits[i]; + + if (nwords && bits % BITS_PER_LONG_LONG) + dst->bits[nwords - 1] &= bmp_last_word_mask(bits); +} + +__weak +void bmp_or(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src1, struct arena_bitmap __arena *src2) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + dst->bits[i] = src1->bits[i] | src2->bits[i]; + + if (nwords && bits % BITS_PER_LONG_LONG) + dst->bits[nwords - 1] &= bmp_last_word_mask(bits); +} + +__weak +bool bmp_empty(size_t bits, struct arena_bitmap __arena *bmp) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) { + u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL; + + if (bmp->bits[i] & mask) + return false; + } + + return true; +} + +__weak +void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap __arena *src) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + dst->bits[i] = src->bits[i]; + + if (nwords && bits % BITS_PER_LONG_LONG) + dst->bits[nwords - 1] &= bmp_last_word_mask(bits); +} + +__weak +bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) { + u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL; + + if (~big->bits[i] & small->bits[i] & mask) + return false; + } + + return true; +} + +__weak +bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) { + u64 mask = (i == nwords - 1) ? bmp_last_word_mask(bits) : ~0ULL; + + if (arg1->bits[i] & arg2->bits[i] & mask) + return true; + } + + return false; +} + +__weak +void bmp_print(size_t bits, struct arena_bitmap __arena *bmp) +{ + size_t nwords = BITS_TO_LONG_LONGS(bits); + volatile u32 i; + + for (i = zero; i < nwords && can_loop; i++) + arena_stderr("%016llx ", bmp->bits[i]); +} diff --git a/tools/testing/selftests/bpf/libarena/src/common.bpf.c b/tools/testing/selftests/bpf/libarena/src/common.bpf.c index 50be57213dfb..41b1de3452fe 100644 --- a/tools/testing/selftests/bpf/libarena/src/common.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/common.bpf.c @@ -4,9 +4,15 @@ #include <libarena/asan.h> #include <libarena/buddy.h> -const volatile u32 zero = 0; - struct buddy __arena buddy; +volatile u32 zero = 0; + +/* + * Storage for the queue nodes declared by bpf_arena_spin_lock.h. Each program + * linking the arena spinlock provides exactly one definition, so that the array + * is emitted once rather than once per translation unit. + */ +struct arena_qnode __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; int arena_fls(__u64 word) { @@ -38,6 +44,12 @@ __weak int arena_buddy_reset(void) return buddy_init(&buddy); } +SEC("syscall") +__weak int arena_buddy_destroy(void) +{ + return buddy_destroy(&buddy); +} + __weak void __arena *arena_malloc(size_t size) { return buddy_alloc(&buddy, size); diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index b82f572641b7..cdf2d7d3ab32 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c @@ -111,7 +111,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a if (settimeo(fd, opts->timeout_ms)) goto error_close; - if (type == SOCK_STREAM && + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM && setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) { log_err("Failed to enable SO_REUSEADDR"); goto error_close; @@ -128,7 +128,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a goto error_close; } - if (type == SOCK_STREAM) { + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM) { if (listen(fd, opts->backlog ? MAX(opts->backlog, 0) : 1) < 0) { log_err("Failed to listed on socket"); goto error_close; @@ -424,7 +424,8 @@ int make_sockaddr(int family, const char *addr_str, __u16 port, *len = sizeof(*sin6); return 0; } else if (family == AF_UNIX) { - /* Note that we always use abstract unix sockets to avoid having + /* + * Note that we always use abstract unix sockets to avoid having * to clean up leftover files. */ struct sockaddr_un *sun = (void *)addr; @@ -865,7 +866,8 @@ static bool is_ethernet(const u_char *packet) memcpy(&arphdr_type, packet + 8, 2); arphdr_type = ntohs(arphdr_type); - /* Except the following cases, the protocol type contains the + /* + * Except the following cases, the protocol type contains the * Ethernet protocol type for the packet. * * https://www.tcpdump.org/linktypes/LINKTYPE_LINUX_SLL2.html @@ -1033,19 +1035,22 @@ static void *traffic_monitor_thread(void *arg) if (!packet) continue; - /* According to the man page of pcap_dump(), first argument + /* + * According to the man page of pcap_dump(), first argument * is the pcap_dumper_t pointer even it's argument type is * u_char *. */ pcap_dump((u_char *)dumper, &header, packet); - /* Not sure what other types of packets look like. Here, we + /* + * Not sure what other types of packets look like. Here, we * parse only Ethernet and compatible packets. */ if (!is_ethernet(packet)) continue; - /* Skip SLL2 header + /* + * Skip SLL2 header * https://www.tcpdump.org/linktypes/LINKTYPE_LINUX_SLL2.html * * Although the document doesn't mention that, the payload @@ -1079,7 +1084,8 @@ static void *traffic_monitor_thread(void *arg) return NULL; } -/* Prepare the pcap handle to capture packets. +/* + * Prepare the pcap handle to capture packets. * * This pcap is non-blocking and immediate mode is enabled to receive * captured packets as soon as possible. The snaplen is set to 1024 bytes @@ -1150,7 +1156,8 @@ static void encode_test_name(char *buf, size_t len, const char *test_name, const #define PCAP_DIR "/tmp/tmon_pcap" -/* Start to monitor the network traffic in the given network namespace. +/* + * Start to monitor the network traffic in the given network namespace. * * netns: the name of the network namespace to monitor. If NULL, the * current network namespace is monitored. @@ -1255,7 +1262,8 @@ static void traffic_monitor_release(struct tmonitor_ctx *ctx) free(ctx); } -/* Stop the network traffic monitor. +/* + * Stop the network traffic monitor. * * ctx: the context returned by traffic_monitor_start() */ diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h index 79a010c88e11..75133119c04a 100644 --- a/tools/testing/selftests/bpf/network_helpers.h +++ b/tools/testing/selftests/bpf/network_helpers.h @@ -25,6 +25,11 @@ typedef __u16 __sum16; #define VIP_NUM 5 #define MAGIC_BYTES 123 +/* include/linux/net.h */ +#ifndef SOCK_TYPE_MASK +#define SOCK_TYPE_MASK 0xf +#endif + struct network_helper_opts { int timeout_ms; int proto; diff --git a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c index d98577a6babc..1ad5d03d07ad 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_atomics.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_atomics.c @@ -222,7 +222,7 @@ static void test_store_release(struct arena_atomics *skel) "store_release64_result"); } -void test_arena_atomics(void) +void serial_test_arena_atomics(void) { struct arena_atomics *skel; int err; diff --git a/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c b/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c index 4b4adb3f4b71..01fcf4965ea4 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_direct_value.c @@ -66,7 +66,7 @@ cleanup: close(map_fd); } -void test_arena_direct_value(void) +void serial_test_arena_direct_value(void) { if (test__start_subtest("one_past_end")) test_arena_direct_value_one_past_end(); diff --git a/tools/testing/selftests/bpf/prog_tests/arena_htab.c b/tools/testing/selftests/bpf/prog_tests/arena_htab.c index d69fd2465f53..91ccf0402980 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_htab.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_htab.c @@ -81,7 +81,7 @@ static void test_arena_htab_asm(void) arena_htab_asm__destroy(skel); } -void test_arena_htab(void) +void serial_test_arena_htab(void) { if (test__start_subtest("arena_htab_llvm")) test_arena_htab_llvm(); diff --git a/tools/testing/selftests/bpf/prog_tests/arena_list.c b/tools/testing/selftests/bpf/prog_tests/arena_list.c index 4f2866a615ce..2648e06f53d0 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_list.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_list.c @@ -68,7 +68,7 @@ out: arena_list__destroy(skel); } -void test_arena_list(void) +void serial_test_arena_list(void) { if (test__start_subtest("arena_list_1")) test_arena_list_add_del(1, false); diff --git a/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c b/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c new file mode 100644 index 000000000000..14c2d1a1d673 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_mem_usage.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include <sys/user.h> +#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */ +#include <unistd.h> +#define PAGE_SIZE getpagesize() +#endif + +#include "arena_mem_usage.skel.h" + +/* + * arena_map_mem_usage() is surfaced to user space through the map's + * /proc/<pid>/fdinfo/<fd> "memlock:" line (the same value bpftool map show + * prints). Read it directly so the test has no external dependency. + */ +static long map_memlock(int map_fd) +{ + char path[64], line[128]; + long memlock = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", map_fd); + f = fopen(path, "r"); + if (!ASSERT_OK_PTR(f, "open_fdinfo")) + return -1; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "memlock:\t%ld", &memlock) == 1) + break; + } + fclose(f); + ASSERT_NEQ(memlock, -1, "parse_memlock"); + return memlock; +} + +static int run(struct bpf_program *prog, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + + if (!ASSERT_OK(err, name)) + return -1; + if (!ASSERT_OK(opts.retval, name)) + return -1; + return 0; +} + +void serial_test_arena_mem_usage(void) +{ + struct arena_mem_usage *skel; + const long ps = PAGE_SIZE; + char *base; + size_t sz; + int fd, i; + + skel = arena_mem_usage__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_load")) + return; + fd = bpf_map__fd(skel->maps.arena); + + /* Fresh arena: no data pages, and the scratch page is not counted. */ + ASSERT_EQ(map_memlock(fd), 0, "initial"); + + /* BPF-side allocation of 17 pages. */ + skel->bss->alloc_cnt = 17; + if (run(skel->progs.alloc, "alloc")) + goto out; + /* + * A NULL ptr means bpf_arena_alloc_pages() itself failed (e.g. the host + * is under memory pressure), not a miscount -- flag it distinctly so a + * red CI run is not mistaken for a counting bug. + */ + if (!ASSERT_OK_PTR(skel->bss->ptr, "arena_alloc_pages")) + goto out; + ASSERT_EQ(map_memlock(fd), 17 * ps, "after_alloc"); + + /* Free a single page (arena_free_pages page_cnt==1 path). */ + skel->bss->free_byte_off = 0; + skel->bss->free_cnt = 1; + if (run(skel->progs.free_pages, "free_one")) + goto out; + ASSERT_EQ(map_memlock(fd), 16 * ps, "after_free_one"); + + /* Free ten pages in one call (bulk path); only the freed pages count. */ + skel->bss->free_byte_off = 1 * ps; + skel->bss->free_cnt = 10; + if (run(skel->progs.free_pages, "free_bulk")) + goto out; + ASSERT_EQ(map_memlock(fd), 6 * ps, "after_free_bulk"); + + /* Free the remaining six -> arena empty again. */ + skel->bss->free_byte_off = 11 * ps; + skel->bss->free_cnt = 6; + if (run(skel->progs.free_pages, "free_rest")) + goto out; + ASSERT_EQ(map_memlock(fd), 0, "after_free_rest"); + + /* + * User-space fault-in: touching unallocated arena pages allocates them + * through arena_vm_fault(). libbpf mmap()s the arena at map_extra during + * load, so bpf_map__initial_value() hands back that base. + */ + base = bpf_map__initial_value(skel->maps.arena, &sz); + if (!ASSERT_OK_PTR(base, "arena_base")) + goto out; + for (i = 0; i < 8; i++) + base[i * ps] = 1; + ASSERT_EQ(map_memlock(fd), 8 * ps, "after_faultin"); + + /* + * Free the faulted-in pages from BPF. They are mapped into the user vma + * (elevated refcount), so this also exercises the zap path. + */ + skel->bss->ptr = base; + skel->bss->free_byte_off = 0; + skel->bss->free_cnt = 8; + if (run(skel->progs.free_pages, "free_faulted")) + goto out; + ASSERT_EQ(map_memlock(fd), 0, "after_free_faulted"); +out: + arena_mem_usage__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c b/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c index acb9d53b5973..545b05d7a0aa 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c @@ -101,7 +101,7 @@ end: return; } -void test_arena_spin_lock(void) +void serial_test_arena_spin_lock(void) { repeat = 1000; if (test__start_subtest("arena_spin_lock_1")) diff --git a/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c b/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c index f81a0c066505..0c1c6cbfa0f1 100644 --- a/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c +++ b/tools/testing/selftests/bpf/prog_tests/arena_strsearch.c @@ -23,7 +23,7 @@ static void test_arena_str(void) arena_strsearch__destroy(skel); } -void test_arena_strsearch(void) +void serial_test_arena_strsearch(void) { if (test__start_subtest("arena_strsearch")) test_arena_str(); diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c index 77f1c0550c9b..6dbd1487343c 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_qdisc.c @@ -11,6 +11,7 @@ #include "bpf_qdisc_fail__invalid_dynptr.skel.h" #include "bpf_qdisc_fail__invalid_dynptr_slice.skel.h" #include "bpf_qdisc_fail__invalid_dynptr_cross_frame.skel.h" +#include "bpf_qdisc_fail__untrusted_write.skel.h" #include "bpf_qdisc_dynptr_use_after_invalidate_clone.skel.h" #define LO_IFINDEX 1 @@ -230,6 +231,7 @@ void test_ns_bpf_qdisc(void) RUN_TESTS(bpf_qdisc_fail__invalid_dynptr); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_cross_frame); RUN_TESTS(bpf_qdisc_fail__invalid_dynptr_slice); + RUN_TESTS(bpf_qdisc_fail__untrusted_write); RUN_TESTS(bpf_qdisc_dynptr_use_after_invalidate_clone); } diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c index fe30181e6336..eb05fc82f81b 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c @@ -14,6 +14,7 @@ #include "tcp_ca_incompl_cong_ops.skel.h" #include "tcp_ca_unsupp_cong_op.skel.h" #include "tcp_ca_kfunc.skel.h" +#include "tcp_ca_untrusted_btf_write.skel.h" #include "bpf_cc_cubic.skel.h" static const unsigned int total_bytes = 10 * 1024 * 1024; @@ -579,6 +580,15 @@ static void test_tcp_ca_kfunc(void) tcp_ca_kfunc__destroy(skel); } +static void test_untrusted_btf_write(void) +{ + struct tcp_ca_untrusted_btf_write *skel; + + skel = tcp_ca_untrusted_btf_write__open_and_load(); + ASSERT_ERR_PTR(skel, "tcp_ca_untrusted_btf_write__open_and_load"); + tcp_ca_untrusted_btf_write__destroy(skel); +} + static void test_cc_cubic(void) { struct cb_opts cb_opts = { @@ -637,6 +647,8 @@ void test_bpf_tcp_ca(void) test_link_replace(); if (test__start_subtest("tcp_ca_kfunc")) test_tcp_ca_kfunc(); + if (test__start_subtest("untrusted_btf_write")) + test_untrusted_btf_write(); if (test__start_subtest("cc_cubic")) test_cc_cubic(); if (test__start_subtest("dctcp_autoattach_map")) diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 66855cbd6b73..67b9015cbd98 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -4250,6 +4250,54 @@ static struct btf_raw_test raw_tests[] = { .max_entries = 1, }, +/* + * struct inner { + * struct bpf_spin_lock lock; + * }; + * + * struct value { + * struct bpf_spin_lock lock; + * struct inner nested; + * }; + */ +{ + .descr = "struct test duplicate nested unique fields", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_TBD, 1, 0), + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [3] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_STRUCT_ENC(NAME_TBD, 2, 8), /* [4] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_MEMBER_ENC(NAME_TBD, 3, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0bpf_spin_lock\0val\0inner\0lock\0value\0lock\0nested"), + .btf_load_err = true, +}, + +/* + * struct value { + * struct bpf_refcount a; + * struct bpf_refcount b; + * }; + */ +{ + .descr = "struct test duplicate bpf_refcount fields", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_STRUCT_ENC(NAME_TBD, 1, 4), /* [2] */ + BTF_MEMBER_ENC(NAME_TBD, 1, 0), + BTF_STRUCT_ENC(NAME_TBD, 2, 8), /* [3] */ + BTF_MEMBER_ENC(NAME_TBD, 2, 0), + BTF_MEMBER_ENC(NAME_TBD, 2, 32), + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0bpf_refcount\0refs\0value\0a\0b"), + .btf_load_err = true, +}, + { .descr = "struct test repeated fields count overflow", .raw_types = { diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c index b7c18d590b99..5a1e08d39a06 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c @@ -10,6 +10,17 @@ #include "cgroup_iter_memcg.h" #include "cgroup_iter_memcg.skel.h" +/* + * memcg stats are cached per-cpu and only become visible once the periodic + * flusher runs (FLUSH_TIME, 2s), or once pending updates cross + * MEMCG_CHARGE_BATCH * num_online_cpus(). That threshold grows with the CPU + * count, so on a large machine a single pass does not reach it and + * bpf_mem_cgroup_flush_stats() returns without flushing anything. Retry for + * long enough to cover a flusher cycle. + */ +#define MEMCG_STAT_RETRIES 16 +#define MEMCG_STAT_RETRY_DELAY_US (250 * 1000) + static int read_stats(struct bpf_link *link) { int fd, ret = 0; @@ -35,11 +46,13 @@ static int read_stats(struct bpf_link *link) static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* * Increase memcg anon usage by mapping and writing * to a new anon region. @@ -53,6 +66,12 @@ static void test_anon(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->nr_anon_mapped && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_anon_mapped, 0, "final anon mapped val"); cleanup: @@ -61,6 +80,7 @@ cleanup: static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; char *path; @@ -76,6 +96,7 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) fd = open(path, O_CREAT | O_RDWR, 0644); if (!ASSERT_OK_FD(fd, "open fd")) return; +retry: if (!ASSERT_OK(ftruncate(fd, len), "ftruncate")) goto cleanup_fd; @@ -88,6 +109,13 @@ static void test_file(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup_map; + if ((!memcg_query->nr_file_pages || !memcg_query->nr_file_mapped) && + ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->nr_file_pages, 0, "final file value"); ASSERT_GT(memcg_query->nr_file_mapped, 0, "final file mapped value"); @@ -100,6 +128,7 @@ cleanup_fd: static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; size_t len; int fd; @@ -113,12 +142,18 @@ static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK_FD(fd, "memfd_create")) return; +retry: if (!ASSERT_OK(fallocate(fd, 0, 0, len), "fallocate")) goto cleanup; if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->nr_shmem && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + goto retry; + } + ASSERT_GT(memcg_query->nr_shmem, 0, "final shmem value"); cleanup: @@ -127,11 +162,13 @@ cleanup: static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) { + int retries = 0; void *map; size_t len; len = sysconf(_SC_PAGESIZE) * 1024; +retry: /* Create region to use for triggering a page fault. */ map = mmap(NULL, len, PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (!ASSERT_NEQ(map, MAP_FAILED, "mmap anon")) @@ -143,6 +180,12 @@ static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query) if (!ASSERT_OK(read_stats(link), "read stats")) goto cleanup; + if (!memcg_query->pgfault && ++retries < MEMCG_STAT_RETRIES) { + usleep(MEMCG_STAT_RETRY_DELAY_US); + munmap(map, len); + goto retry; + } + ASSERT_GT(memcg_query->pgfault, 0, "final pgfault val"); cleanup: diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index 92c20803ea76..2523c07a16c6 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -5,6 +5,7 @@ #include <bpf/btf.h> #include "bind4_prog.skel.h" #include "freplace_progmap.skel.h" +#include "fentry_sleepable.skel.h" #include "xdp_dummy.skel.h" typedef int (*test_cb)(struct bpf_object *obj); @@ -335,18 +336,6 @@ out: bpf_object__close(pkt_obj); } - -static void test_func_sockmap_update(void) -{ - const char *prog_name[] = { - "freplace/cls_redirect", - }; - test_fexit_bpf2bpf_common("./freplace_cls_redirect.bpf.o", - "./test_cls_redirect.bpf.o", - ARRAY_SIZE(prog_name), - prog_name, false, NULL); -} - static void test_func_replace_void(void) { const char *prog_name[] = { @@ -588,6 +577,60 @@ out: freplace_progmap__destroy(skel); } +static void test_sleepable_fentry_to_xdp(void) +{ + struct fentry_sleepable *skel = NULL; + struct xdp_dummy *skel_xdp = NULL; + int ifindex, prog_fd, err; + char buff[64] = {}; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + ifindex = if_nametoindex("lo"); + if (!ASSERT_GT(ifindex, 0, "if_nametoindex")) + return; + + skel_xdp = xdp_dummy__open_and_load(); + if (!ASSERT_OK_PTR(skel_xdp, "xdp_dummy__open_and_load")) + return; + + skel = fentry_sleepable__open(); + if (!ASSERT_OK_PTR(skel, "fentry_sleepable__open")) + goto out; + + skel->bss->user_ptr = buff; + + prog_fd = bpf_program__fd(skel_xdp->progs.__x64_sys_nop); + err = bpf_program__set_attach_target(skel->progs.fentry_xdp, prog_fd, "__x64_sys_nop"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = fentry_sleepable__load(skel); + ASSERT_ERR(err, "fentry_sleepable__load"); + if (err) + goto out; + + skel->links.fentry_xdp = bpf_program__attach_trace(skel->progs.fentry_xdp); + if (!ASSERT_OK_PTR(skel->links.fentry_xdp, "bpf_program__attach_trace")) + goto out; + + skel_xdp->links.__x64_sys_nop = bpf_program__attach_xdp(skel_xdp->progs.__x64_sys_nop, + ifindex); + if (!ASSERT_OK_PTR(skel_xdp->links.__x64_sys_nop, "bpf_program__attach_xdp")) + goto out; + + err = system("ping -q -c 1 -W 1 127.0.0.1 > /dev/null"); + ASSERT_OK(err, "ping"); + ASSERT_ERR(skel->bss->retval, "retval"); + +out: + fentry_sleepable__destroy(skel); + xdp_dummy__destroy(skel_xdp); +} + /* NOTE: affect other tests, must run in serial mode */ void serial_test_fexit_bpf2bpf(void) { @@ -599,8 +642,6 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace(); if (test__start_subtest("func_replace_verify")) test_func_replace_verify(); - if (test__start_subtest("func_sockmap_update")) - test_func_sockmap_update(); if (test__start_subtest("func_replace_return_code")) test_func_replace_return_code(); if (test__start_subtest("func_map_prog_compatibility")) @@ -621,4 +662,6 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace_int_with_void(); if (test__start_subtest("freplace_void")) test_func_replace_void(); + if (test__start_subtest("sleepable_fentry_to_xdp")) + test_sleepable_fentry_to_xdp(); } diff --git a/tools/testing/selftests/bpf/prog_tests/fib_lookup.c b/tools/testing/selftests/bpf/prog_tests/fib_lookup.c index bd7658958004..8f4779dd802e 100644 --- a/tools/testing/selftests/bpf/prog_tests/fib_lookup.c +++ b/tools/testing/selftests/bpf/prog_tests/fib_lookup.c @@ -2,6 +2,7 @@ /* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */ #include <linux/rtnetlink.h> +#include <linux/if_ether.h> #include <sys/types.h> #include <net/if.h> @@ -23,6 +24,7 @@ #define IPV4_TBID_ADDR "172.0.0.254" #define IPV4_TBID_NET "172.0.0.0" #define IPV4_TBID_DST "172.0.0.2" +#define IPV4_TBID_NONEIGH_DST "172.0.0.5" #define IPV6_TBID_ADDR "fd00::FFFF" #define IPV6_TBID_NET "fd00::" #define IPV6_TBID_DST "fd00::2" @@ -37,6 +39,41 @@ #define IPV6_LOCAL "fd01::3" #define IPV6_GW1 "fd01::1" #define IPV6_GW2 "fd01::2" +#define VLAN_ID 100 +#define VLAN_IFACE "veth1.100" +#define VLAN_ID_DOWN 102 +#define VLAN_IFACE_DOWN "veth1.102" +#define QINQ_OUTER_IFACE "veth1.200" +#define QINQ_INNER_IFACE "veth1.200.300" +#define VLAN_TABLE "300" +#define IPV4_VLAN_IFACE_ADDR "10.5.0.254" +#define IPV4_VLAN_EGRESS_DST "10.5.0.2" +#define IPV4_QINQ_DST "10.7.0.2" +#define IPV4_VLAN_DST "10.6.0.2" +#define IPV4_VLAN_GW "10.5.0.1" +#define IPV6_VLAN_IFACE_ADDR "fd02::254" +#define IPV6_VLAN_EGRESS_DST "fd02::2" +#define IPV6_VLAN_DST "fd03::2" +#define IPV6_VLAN_GW "fd02::1" +#define VLAN_VID_UNUSED 999 +#define VRF_IFACE "vrf-blue" +#define VRF_TABLE "1000" +#define VRF_VLAN_ID 101 +#define VRF_VLAN_IFACE "veth1.101" +#define IPV4_VRF_IFACE_ADDR "10.8.0.254" +#define IPV4_VRF_GW "10.8.0.1" +#define IPV4_VRF_DST "10.9.0.2" +#define TBID_VLAN_ID 50 +#define TBID_VLAN_IFACE "veth2.50" +#define IPV4_TBID_VLAN_DST "172.2.0.2" +#define IPV4_BOND_VLAN_DST "10.11.0.2" +#define IPV4_VLAN_MTU_DST "10.5.9.2" +#define QINQ_AD_VLAN_ID 200 +#define QINQ_INNER_VLAN_ID 300 +#define BOND_IFACE "bond99" +#define BOND_PORT "veth3" +#define BOND_PORT_PEER "veth4" +#define BOND_VLAN_ID 500 #define DMAC "11:11:11:11:11:11" #define DMAC_INIT { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, } #define DMAC2 "01:01:01:01:01:01" @@ -52,6 +89,17 @@ struct fib_lookup_test { __u32 tbid; __u8 dmac[6]; __u32 mark; + /* + * input tag with BPF_FIB_LOOKUP_VLAN_INPUT; expected output tag + * with BPF_FIB_LOOKUP_VLAN (checked when check_vlan is set) + */ + __u16 vlan_proto; + __u16 vlan_id; + bool check_vlan; + const char *expected_dev; /* expected params->ifindex after lookup */ + const char *iif; /* override the default veth1 input device */ + __u16 tot_len; /* triggers the in-lookup mtu check when set */ + __u16 expected_mtu; /* expected mtu_result (union with tot_len) */ }; static const struct fib_lookup_test tests[] = { @@ -79,6 +127,17 @@ static const struct fib_lookup_test tests[] = { .daddr = IPV4_TBID_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, .tbid = 100, .dmac = DMAC_INIT2, }, + /* + * An error that returns after the egress device is resolved must + * report the egress ifindex, not the input. This routes from input + * veth1 via veth2 (table 100) to a dst with no neighbour, so + * input != egress, pinning NO_NEIGH to the egress device. + */ + { .desc = "IPv4 NO_NEIGH reports the egress ifindex, not the input", + .daddr = IPV4_TBID_NONEIGH_DST, + .expected_ret = BPF_FIB_LKUP_RET_NO_NEIGH, + .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, .tbid = 100, + .expected_dev = "veth2", }, { .desc = "IPv6 TBID lookup failure", .daddr = IPV6_TBID_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID, @@ -142,12 +201,237 @@ static const struct fib_lookup_test tests[] = { .expected_dst = IPV6_GW1, .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, .mark = MARK, }, + /* vlan egress resolution */ + /* + * Invariant the VLAN-egress arms jointly enforce: a + * BPF_FIB_LOOKUP_VLAN SUCCESS always carries a physical, + * xmit-capable ifindex; no SUCCESS ever returns a VLAN-device + * ifindex. Reducible arms pin ifindex == the physical parent; the + * QinQ and foreign-netns arms pin VLAN_FAILURE with params->ifindex + * left at the input, so a regression to best-effort (SUCCESS + the + * VLAN ifindex) fails one. + */ + { .desc = "IPv4 VLAN egress, no flag", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = VLAN_IFACE, .check_vlan = true, }, + { .desc = "IPv4 VLAN egress, single VLAN", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + /* + * skb path without tot_len: mtu_result is the VLAN device's mtu + * (1400), not the parent's (1500) + */ + { .desc = "IPv4 VLAN egress, skb-path mtu is the VLAN device's without the flag", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = VLAN_IFACE, .check_vlan = true, .expected_mtu = 1400, }, + { .desc = "IPv4 VLAN egress, flag set but egress is not a VLAN", + .daddr = IPV4_NUD_FAILED_ADDR, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + { .desc = "IPv4 VLAN egress, QinQ not reducible (VLAN_FAILURE)", + .daddr = IPV4_QINQ_DST, + .expected_ret = BPF_FIB_LKUP_RET_VLAN_FAILURE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + { .desc = "IPv4 QinQ egress without the flag (escape hatch)", + .daddr = IPV4_QINQ_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = QINQ_INNER_IFACE, }, + { .desc = "IPv6 VLAN egress, single VLAN", + .daddr = IPV6_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress, neighbour on the VLAN device", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, .dmac = DMAC_INIT, }, + { .desc = "IPv4 VLAN egress in OUTPUT mode", + .daddr = IPV4_VLAN_EGRESS_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .iif = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_OUTPUT | BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress over a bond", + .daddr = IPV4_BOND_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = BOND_IFACE, .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv4 VLAN egress via TBID table", + .daddr = IPV4_TBID_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .lookup_flags = BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_TBID | + BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .tbid = 100, + .expected_dev = "veth2", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = TBID_VLAN_ID, }, + { .desc = "IPv4 VLAN egress, success writes mtu_result with the swap", + .daddr = IPV4_VLAN_MTU_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .tot_len = 500, .expected_mtu = 1000, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN egress, FRAG_NEEDED reports mtu, swap unwritten", + .daddr = IPV4_VLAN_MTU_DST, .expected_ret = BPF_FIB_LKUP_RET_FRAG_NEEDED, + .tot_len = 1400, .expected_mtu = 1000, + .lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH, + .expected_dev = "veth1", .check_vlan = true, }, + /* vlan tag as lookup input */ + { .desc = "IPv4 VLAN input, no flag", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_GW1, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input, tag selects subinterface route", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv6 VLAN input, tag selects subinterface route", + .daddr = IPV6_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV6_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input and egress combined", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = "veth1", + .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, neighbour resolved on the route", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, .expected_dev = VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, .dmac = DMAC_INIT2, }, + { .desc = "IPv4 VLAN input, source address from the subinterface", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_src = IPV4_VLAN_IFACE_ADDR, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SRC | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + /* + * VRF: the resolved subinterface is enslaved, so the l3mdev rule + * (full lookup) and l3mdev_fib_table_rcu() (DIRECT) must select + * the VRF table from the resolved ingress + */ + { .desc = "IPv4 VLAN input, VRF subinterface, no flag", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_GW1, + .lookup_flags = BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input, tag selects VRF table", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VRF_GW, .expected_dev = VRF_VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VRF_VLAN_ID, }, + { .desc = "IPv4 VLAN input, DIRECT uses VRF table from resolved ingress", + .daddr = IPV4_VRF_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VRF_GW, .expected_dev = VRF_VLAN_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_DIRECT | + BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VRF_VLAN_ID, }, + /* + * failure arms also assert params is left untouched: ifindex still + * names the physical device and the input tag bytes survive + */ + { .desc = "IPv4 VLAN input, invalid proto", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = 0x1234, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, unmatched VID", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_VID_UNUSED, }, + { .desc = "IPv4 VLAN input, subinterface down", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID_DOWN, }, + /* + * the resolver runs before the forwarding check, so on devices + * with forwarding off FWD_DISABLED (not NOT_FWDED) proves the tag + * resolved to that device and the lookup used it as ingress + */ + { .desc = "IPv4 VLAN input, 802.1ad tag", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021AD, .vlan_id = QINQ_AD_VLAN_ID, }, + { .desc = "IPv4 VLAN input, PCP and DEI bits ignored in TCI", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_SUCCESS, + .expected_dst = IPV4_VLAN_GW, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = 0xe000 | VLAN_ID, }, + { .desc = "IPv4 VLAN input, inner QinQ device from VLAN ifindex", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .iif = QINQ_OUTER_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = QINQ_INNER_VLAN_ID, }, + /* + * bonding: the VLANs live on the master, as on receive, where the + * frame is steered to the master before VLAN processing; a port + * ifindex does not match (ports carry vid state but no VLAN devs) + */ + { .desc = "IPv4 VLAN input, tag on bond master resolves", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_FWD_DISABLED, + .iif = BOND_IFACE, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv4 VLAN input, tag on bond port does not match", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .iif = BOND_PORT, .expected_dev = BOND_PORT, .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = BOND_VLAN_ID, }, + { .desc = "IPv6 VLAN input, invalid proto", + .daddr = IPV6_VLAN_DST, .expected_ret = -EINVAL, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = 0x1234, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input, VID 0 priority tag fails closed", + .daddr = IPV4_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = 0, }, + { .desc = "IPv6 VLAN input, unmatched VID", + .daddr = IPV6_VLAN_DST, .expected_ret = BPF_FIB_LKUP_RET_NOT_FWDED, + .expected_dev = "veth1", .check_vlan = true, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_SKIP_NEIGH, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_VID_UNUSED, }, + { .desc = "unknown flag bit rejected", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = (1 << 14) | BPF_FIB_LOOKUP_SKIP_NEIGH, }, + { .desc = "IPv4 VLAN input rejected with TBID", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_TBID, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, + { .desc = "IPv4 VLAN input rejected with OUTPUT", + .daddr = IPV4_VLAN_DST, .expected_ret = -EINVAL, + .lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | BPF_FIB_LOOKUP_OUTPUT, + .vlan_proto = ETH_P_8021Q, .vlan_id = VLAN_ID, }, }; static int setup_netns(void) { int err; + /* + * a new netns copies the IPv4 conf from init_net, so on a host with + * forwarding enabled the arms that expect FWD_DISABLED would see the + * lookup succeed instead; pin it off here and enable it per device + */ + err = write_sysctl("/proc/sys/net/ipv4/conf/all/forwarding", "0"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf.all.forwarding)")) + goto fail; + + err = write_sysctl("/proc/sys/net/ipv4/conf/default/forwarding", "0"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf.default.forwarding)")) + goto fail; + SYS(fail, "ip link add veth1 type veth peer name veth2"); SYS(fail, "ip link set dev veth1 up"); SYS(fail, "ip link set dev veth2 up"); @@ -204,6 +488,105 @@ static int setup_netns(void) SYS(fail, "ip rule add prio 2 fwmark %d lookup %s", MARK, MARK_TABLE); SYS(fail, "ip -6 rule add prio 2 fwmark %d lookup %s", MARK, MARK_TABLE); + /* + * Setup for vlan tests: a subinterface for egress resolution and + * tag-as-input, a QinQ stack, and an iif rule so the input tests + * observe which device the lookup used as ingress. + */ + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VLAN_IFACE, VLAN_ID); + SYS(fail, "ip link set dev %s up", VLAN_IFACE); + /* + * lower than the veth1 parent (1500): the skb-path mtu check uses the + * FIB result (VLAN) device, so mtu_result is this value, which the + * no-flag arm below pins + */ + SYS(fail, "ip link set dev %s mtu 1400", VLAN_IFACE); + SYS(fail, "ip addr add %s/24 dev %s", IPV4_VLAN_IFACE_ADDR, VLAN_IFACE); + SYS(fail, "ip addr add %s/64 dev %s nodad", IPV6_VLAN_IFACE_ADDR, VLAN_IFACE); + + /* + * stays down: the input flag must treat its tag the way real + * ingress treats a frame arriving on a down VLAN device (drop) + */ + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VLAN_IFACE_DOWN, VLAN_ID_DOWN); + + err = write_sysctl("/proc/sys/net/ipv4/conf/" VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf." VLAN_IFACE ".forwarding)")) + goto fail; + + err = write_sysctl("/proc/sys/net/ipv6/conf/" VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv6.conf." VLAN_IFACE ".forwarding)")) + goto fail; + + SYS(fail, "ip link add link veth1 name %s type vlan proto 802.1ad id 200", + QINQ_OUTER_IFACE); + SYS(fail, "ip link add link %s name %s type vlan id 300", + QINQ_OUTER_IFACE, QINQ_INNER_IFACE); + SYS(fail, "ip link set dev %s up", QINQ_OUTER_IFACE); + SYS(fail, "ip link set dev %s up", QINQ_INNER_IFACE); + SYS(fail, "ip route add %s/32 dev %s", IPV4_QINQ_DST, QINQ_INNER_IFACE); + + SYS(fail, "ip route add %s/32 via %s", IPV4_VLAN_DST, IPV4_GW1); + SYS(fail, "ip route add table %s %s/32 via %s", + VLAN_TABLE, IPV4_VLAN_DST, IPV4_VLAN_GW); + SYS(fail, "ip rule add prio 3 iif %s lookup %s", VLAN_IFACE, VLAN_TABLE); + SYS(fail, "ip -6 route add %s/128 via %s", IPV6_VLAN_DST, IPV6_GW1); + SYS(fail, "ip -6 route add table %s %s/128 via %s", + VLAN_TABLE, IPV6_VLAN_DST, IPV6_VLAN_GW); + SYS(fail, "ip -6 rule add prio 3 iif %s lookup %s", VLAN_IFACE, VLAN_TABLE); + + /* a bond with one port and a VLAN on the bond */ + SYS(fail, "ip link add %s type bond", BOND_IFACE); + SYS(fail, "ip link add %s type veth peer name %s", BOND_PORT, BOND_PORT_PEER); + SYS(fail, "ip link set %s master %s", BOND_PORT, BOND_IFACE); + SYS(fail, "ip link set dev %s up", BOND_IFACE); + SYS(fail, "ip link set dev %s up", BOND_PORT); + SYS(fail, "ip link add link %s name %s.%d type vlan id %d", + BOND_IFACE, BOND_IFACE, BOND_VLAN_ID, BOND_VLAN_ID); + SYS(fail, "ip link set dev %s.%d up", BOND_IFACE, BOND_VLAN_ID); + SYS(fail, "ip route add %s/32 dev %s.%d", + IPV4_BOND_VLAN_DST, BOND_IFACE, BOND_VLAN_ID); + + /* + * a VRF with its own dedicated subinterface (the iif rules above + * must not see it), for the table-selection-by-ingress cases + */ + SYS(fail, "ip link add %s type vrf table %s", VRF_IFACE, VRF_TABLE); + SYS(fail, "ip link set dev %s up", VRF_IFACE); + SYS(fail, "ip link add link veth1 name %s type vlan id %d", + VRF_VLAN_IFACE, VRF_VLAN_ID); + SYS(fail, "ip link set %s master %s", VRF_VLAN_IFACE, VRF_IFACE); + SYS(fail, "ip link set dev %s up", VRF_VLAN_IFACE); + SYS(fail, "ip addr add %s/24 dev %s", IPV4_VRF_IFACE_ADDR, VRF_VLAN_IFACE); + err = write_sysctl("/proc/sys/net/ipv4/conf/" VRF_VLAN_IFACE "/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.conf." VRF_VLAN_IFACE ".forwarding)")) + goto fail; + SYS(fail, "ip route add %s/32 via %s", IPV4_VRF_DST, IPV4_GW1); + SYS(fail, "ip route add table %s %s/32 via %s", + VRF_TABLE, IPV4_VRF_DST, IPV4_VRF_GW); + + /* neighbours on the VLAN subinterface for the non-SKIP_NEIGH cases */ + err = write_sysctl("/proc/sys/net/ipv4/neigh/" VLAN_IFACE "/gc_stale_time", "900"); + if (!ASSERT_OK(err, "write_sysctl(net.ipv4.neigh." VLAN_IFACE ".gc_stale_time)")) + goto fail; + SYS(fail, "ip neigh add %s dev %s lladdr %s nud stale", + IPV4_VLAN_EGRESS_DST, VLAN_IFACE, DMAC); + SYS(fail, "ip neigh add %s dev %s lladdr %s nud stale", + IPV4_VLAN_GW, VLAN_IFACE, DMAC2); + + /* a VLAN on veth2 with a route in the tbid test table */ + SYS(fail, "ip link add link veth2 name %s type vlan id %d", + TBID_VLAN_IFACE, TBID_VLAN_ID); + SYS(fail, "ip link set dev %s up", TBID_VLAN_IFACE); + SYS(fail, "ip route add table 100 %s/32 dev %s", + IPV4_TBID_VLAN_DST, TBID_VLAN_IFACE); + + /* a locked-mtu route via the subinterface for the FRAG_NEEDED case */ + SYS(fail, "ip route add %s/32 dev %s mtu lock 1000", + IPV4_VLAN_MTU_DST, VLAN_IFACE); + return 0; fail: return -1; @@ -218,9 +601,16 @@ static int set_lookup_params(struct bpf_fib_lookup *params, memset(params, 0, sizeof(*params)); params->l4_protocol = IPPROTO_TCP; - params->ifindex = ifindex; + params->ifindex = test->iif ? if_nametoindex(test->iif) : ifindex; params->tbid = test->tbid; params->mark = test->mark; + params->tot_len = test->tot_len; + + /* h_vlan_proto/h_vlan_TCI union with tbid */ + if (test->lookup_flags & BPF_FIB_LOOKUP_VLAN_INPUT) { + params->h_vlan_proto = htons(test->vlan_proto); + params->h_vlan_TCI = htons(test->vlan_id); + } if (inet_pton(AF_INET6, test->daddr, params->ipv6_dst) == 1) { params->family = AF_INET6; @@ -298,7 +688,7 @@ void test_fib_lookup(void) struct nstoken *nstoken = NULL; struct __sk_buff skb = { }; struct fib_lookup *skel; - int prog_fd, err, ret, i; + int prog_fd, xdp_fd, err, ret, i; /* The test does not use the skb->data, so * use pkt_v6 for both v6 and v4 test. @@ -309,11 +699,16 @@ void test_fib_lookup(void) .ctx_in = &skb, .ctx_size_in = sizeof(skb), ); + LIBBPF_OPTS(bpf_test_run_opts, xdp_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + ); skel = fib_lookup__open_and_load(); if (!ASSERT_OK_PTR(skel, "skel open_and_load")) return; prog_fd = bpf_program__fd(skel->progs.fib_lookup); + xdp_fd = bpf_program__fd(skel->progs.fib_lookup_xdp); SYS(fail, "ip netns add %s", NS_TEST); @@ -343,6 +738,16 @@ void test_fib_lookup(void) if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) continue; + /* + * BPF_FIB_LOOKUP_VLAN is XDP-only; the tc helper rejects it. + * These cases are exercised on the XDP path below. + */ + if (tests[i].lookup_flags & BPF_FIB_LOOKUP_VLAN) { + ASSERT_EQ(skel->bss->fib_lookup_ret, -EINVAL, + "tc rejects BPF_FIB_LOOKUP_VLAN"); + continue; + } + ASSERT_EQ(skel->bss->fib_lookup_ret, tests[i].expected_ret, "fib_lookup_ret"); @@ -352,6 +757,21 @@ void test_fib_lookup(void) if (tests[i].expected_dst) assert_dst_ip(fib_params, tests[i].expected_dst); + if (tests[i].expected_dev) + ASSERT_EQ(fib_params->ifindex, + if_nametoindex(tests[i].expected_dev), "ifindex"); + + if (tests[i].expected_mtu) + ASSERT_EQ(fib_params->mtu_result, tests[i].expected_mtu, + "mtu_result"); + + if (tests[i].check_vlan) { + ASSERT_EQ(fib_params->h_vlan_proto, + htons(tests[i].vlan_proto), "h_vlan_proto"); + ASSERT_EQ(fib_params->h_vlan_TCI, + htons(tests[i].vlan_id), "h_vlan_TCI"); + } + ret = memcmp(tests[i].dmac, fib_params->dmac, sizeof(tests[i].dmac)); if (!ASSERT_EQ(ret, 0, "dmac not match")) { char expected[18], actual[18]; @@ -361,17 +781,330 @@ void test_fib_lookup(void) printf("dmac expected %s actual %s ", expected, actual); } - // ensure tbid is zero'd out after fib lookup. - if (tests[i].lookup_flags & BPF_FIB_LOOKUP_DIRECT) { + /* + * ensure tbid is zero'd out after fib lookup. With + * BPF_FIB_LOOKUP_VLAN the union holds the packed vlan + * fields instead, so skip the check for those. + */ + if ((tests[i].lookup_flags & BPF_FIB_LOOKUP_DIRECT) && + !(tests[i].lookup_flags & BPF_FIB_LOOKUP_VLAN)) { if (!ASSERT_EQ(skel->bss->fib_params.tbid, 0, "expected fib_params.tbid to be zero")) goto fail; } } + /* + * Re-run the cases through bpf_xdp_fib_lookup(). test_run uses the + * current netns' loopback for ctx->rxq->dev, so dev_net() is NS_TEST + * and the lookup runs against its FIB. The path-independent results + * (return code, swapped ifindex, vlan tag, gateway) must match the skb + * path; the no-tot_len mtu_result is skb-specific and not rechecked. + */ + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (set_lookup_params(fib_params, &tests[i], skb.ifindex)) + continue; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = tests[i].lookup_flags; + + err = bpf_prog_test_run_opts(xdp_fd, &xdp_opts); + if (!ASSERT_OK(err, "xdp test_run")) + continue; + + if (!ASSERT_EQ(skel->bss->fib_lookup_ret, tests[i].expected_ret, + "xdp fib_lookup_ret")) + printf("(xdp) %s\n", tests[i].desc); + + if (tests[i].expected_dev) + ASSERT_EQ(fib_params->ifindex, + if_nametoindex(tests[i].expected_dev), + "xdp ifindex"); + + if (tests[i].expected_dst) + assert_dst_ip(fib_params, tests[i].expected_dst); + + if (tests[i].check_vlan) { + ASSERT_EQ(fib_params->h_vlan_proto, + htons(tests[i].vlan_proto), "xdp h_vlan_proto"); + ASSERT_EQ(fib_params->h_vlan_TCI, + htons(tests[i].vlan_id), "xdp h_vlan_TCI"); + } + + ret = memcmp(tests[i].dmac, fib_params->dmac, sizeof(tests[i].dmac)); + ASSERT_EQ(ret, 0, "xdp dmac"); + + /* + * mtu_result from a tot_len lookup is the route mtu and is + * path-independent; the no-tot_len arm reads dev->mtu and is + * skb-only, so gate on tot_len + */ + if (tests[i].expected_mtu && tests[i].tot_len) + ASSERT_EQ(fib_params->mtu_result, tests[i].expected_mtu, + "xdp mtu_result"); + } + fail: if (nstoken) close_netns(nstoken); SYS_NOFAIL("ip netns del " NS_TEST); fib_lookup__destroy(skel); } + +#define NS_VLAN_A "fib_lookup_vlan_ns_a" +#define NS_VLAN_B "fib_lookup_vlan_ns_b" +#define IPV4_VLAN_NETNS_ADDR "10.66.0.1" +#define IPV4_VLAN_NETNS_DST "10.66.0.2" + +/* + * A VLAN device can be moved to another netns while staying registered + * on its parent. Neither direction may then cross the boundary: the + * egress flag must not publish the foreign parent's ifindex, and the + * input flag must fail closed rather than use a foreign ingress. + */ +void test_fib_lookup_vlan_netns(void) +{ + struct bpf_fib_lookup *fib_params; + struct nstoken *nstoken = NULL; + struct __sk_buff skb = { }; + struct fib_lookup *skel = NULL; + int prog_fd, xdp_fd, err, parent_idx, vlan_idx; + + LIBBPF_OPTS(bpf_test_run_opts, run_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + .ctx_in = &skb, + .ctx_size_in = sizeof(skb), + ); + LIBBPF_OPTS(bpf_test_run_opts, xdp_opts, + .data_in = &pkt_v6, + .data_size_in = sizeof(pkt_v6), + ); + + skel = fib_lookup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + prog_fd = bpf_program__fd(skel->progs.fib_lookup); + xdp_fd = bpf_program__fd(skel->progs.fib_lookup_xdp); + fib_params = &skel->bss->fib_params; + + SYS(fail, "ip netns add %s", NS_VLAN_A); + SYS(fail, "ip netns add %s", NS_VLAN_B); + + nstoken = open_netns(NS_VLAN_A); + if (!ASSERT_OK_PTR(nstoken, "open_netns(a)")) + goto fail; + + SYS(fail, "ip link add veth7 type veth peer name veth8"); + SYS(fail, "ip link set dev veth7 up"); + SYS(fail, "ip link add link veth7 name veth7.66 type vlan id 66"); + SYS(fail, "ip link set veth7.66 netns %s", NS_VLAN_B); + /* + * up it in B before the input lookup: the move closed it, and a + * down device fails the resolver on IFF_UP before reaching the + * netns check this subtest exists to pin + */ + SYS(fail, "ip -n %s link set dev veth7.66 up", NS_VLAN_B); + + parent_idx = if_nametoindex("veth7"); + if (!ASSERT_NEQ(parent_idx, 0, "if_nametoindex(veth7)")) + goto fail; + + /* + * give this netns a route to the destination: the lookup below runs + * against this FIB, so without the route a kernel that resolved the + * moved device anyway would still return NOT_FWDED and the arm would + * pass for the wrong reason + */ + SYS(fail, "ip route add %s/32 dev veth7", IPV4_VLAN_NETNS_DST); + + /* + * input: the moved device is still in veth7's VLAN group, but it + * lives in another netns, so the lookup must fail closed + */ + skb.ifindex = parent_idx; + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = parent_idx; + fib_params->h_vlan_proto = htons(ETH_P_8021Q); + fib_params->h_vlan_TCI = htons(66); + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(dst)")) + goto fail; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN_INPUT | + BPF_FIB_LOOKUP_SKIP_NEIGH; + err = bpf_prog_test_run_opts(prog_fd, &run_opts); + if (!ASSERT_OK(err, "test_run(input)")) + goto fail; + ASSERT_EQ(skel->bss->fib_lookup_ret, BPF_FIB_LKUP_RET_NOT_FWDED, + "input across netns fails closed"); + ASSERT_EQ(fib_params->ifindex, parent_idx, "ifindex untouched"); + ASSERT_EQ(fib_params->h_vlan_TCI, htons(66), "tag untouched"); + + close_netns(nstoken); + nstoken = open_netns(NS_VLAN_B); + if (!ASSERT_OK_PTR(nstoken, "open_netns(b)")) + goto fail; + + /* + * egress: the fib result is the VLAN device here, but its parent + * is in the other netns, so the swap must not happen + */ + SYS(fail, "ip addr add %s/24 dev veth7.66", IPV4_VLAN_NETNS_ADDR); + err = write_sysctl("/proc/sys/net/ipv4/conf/veth7.66/forwarding", "1"); + if (!ASSERT_OK(err, "write_sysctl(forwarding)")) + goto fail; + + vlan_idx = if_nametoindex("veth7.66"); + if (!ASSERT_NEQ(vlan_idx, 0, "if_nametoindex(veth7.66)")) + goto fail; + + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = vlan_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(dst)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_NETNS_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)")) + goto fail; + + skel->bss->fib_lookup_ret = -1; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | + BPF_FIB_LOOKUP_SKIP_NEIGH; + err = bpf_prog_test_run_opts(xdp_fd, &xdp_opts); + if (!ASSERT_OK(err, "test_run(egress)")) + goto fail; + ASSERT_EQ(skel->bss->fib_lookup_ret, BPF_FIB_LKUP_RET_VLAN_FAILURE, + "egress returns VLAN_FAILURE"); + ASSERT_EQ(fib_params->ifindex, vlan_idx, + "foreign parent not published"); + ASSERT_EQ(fib_params->h_vlan_TCI, 0, "vlan fields zero"); + +fail: + if (nstoken) + close_netns(nstoken); + SYS_NOFAIL("ip netns del " NS_VLAN_A); + SYS_NOFAIL("ip netns del " NS_VLAN_B); + fib_lookup__destroy(skel); +} + +#define REDIRECT_NPKTS 1000 +#define NS_REDIRECT "fib_lookup_redirect_ns" + +/* + * The egress flag exists so an XDP program can redirect to the physical + * parent. A redirect that lands on a VLAN device is dropped at + * xdp_do_flush(), because a VLAN device has no ndo_xdp_xmit. Drive real + * frames with BPF_F_TEST_XDP_LIVE_FRAMES, which runs the native + * xdp_do_redirect() + xdp_do_flush() path: a reducible VLAN egress + * resolves to veth1 and is delivered to its peer veth2, while a QinQ + * egress returns VLAN_FAILURE and is passed to the stack instead of + * redirected to a device that would silently drop it. + */ +void test_fib_lookup_vlan_redirect(void) +{ + int redirect_fd, err, veth1_idx, veth2_idx = -1; + struct bpf_fib_lookup *fib_params; + struct nstoken *nstoken = NULL; + struct fib_lookup *skel = NULL; + bool xdp_attached = false; + + LIBBPF_OPTS(bpf_test_run_opts, lf_opts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .flags = BPF_F_TEST_XDP_LIVE_FRAMES, + .repeat = REDIRECT_NPKTS, + ); + + skel = fib_lookup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + redirect_fd = bpf_program__fd(skel->progs.fib_lookup_redirect); + fib_params = &skel->bss->fib_params; + + SYS(fail, "ip netns add %s", NS_REDIRECT); + nstoken = open_netns(NS_REDIRECT); + if (!ASSERT_OK_PTR(nstoken, "open_netns")) + goto fail; + if (setup_netns()) + goto fail; + + veth1_idx = if_nametoindex("veth1"); + veth2_idx = if_nametoindex("veth2"); + if (!ASSERT_NEQ(veth1_idx, 0, "if_nametoindex(veth1)") || + !ASSERT_NEQ(veth2_idx, 0, "if_nametoindex(veth2)")) + goto fail; + + /* + * A redirect to veth1 is delivered to its peer veth2. veth_xdp_xmit() + * only accepts the frame if veth2's NAPI is up, which on veth means + * veth2 carries an XDP program; xdp_count tallies what arrives. + */ + err = bpf_xdp_attach(veth2_idx, bpf_program__fd(skel->progs.xdp_count), + XDP_FLAGS_DRV_MODE, NULL); + if (!ASSERT_OK(err, "attach xdp_count on veth2")) + goto fail; + xdp_attached = true; + + /* reducible VLAN egress: resolves to the physical parent veth1 */ + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = veth1_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_IFACE_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_VLAN_EGRESS_DST, &fib_params->ipv4_dst), + 1, "inet_pton(reducible dst)")) + goto fail; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH; + skel->bss->redirected = 0; + skel->bss->passed = 0; + skel->bss->delivered = 0; + + err = bpf_prog_test_run_opts(redirect_fd, &lf_opts); + if (!ASSERT_OK(err, "test_run(reducible egress)")) + goto fail; + ASSERT_EQ(skel->bss->redirected, REDIRECT_NPKTS, "reducible egress redirected"); + ASSERT_EQ(skel->bss->passed, 0, "reducible egress not passed"); + ASSERT_GT(skel->bss->delivered, 0, "reducible egress delivered to veth2"); + + /* + * QinQ egress: not reducible, so the lookup returns VLAN_FAILURE and + * the program passes the frame instead of redirecting to the inner + * VLAN device. redirected == 0 is the assertion that matters: the + * program did not redirect to a device that would drop the frame at + * xdp_do_flush(). veth2's delivered count is not checked here, since + * a passed frame can still reach veth2 through the stack's forwarding + * path, which is unrelated to the redirect under test. + */ + memset(fib_params, 0, sizeof(*fib_params)); + fib_params->family = AF_INET; + fib_params->l4_protocol = IPPROTO_TCP; + fib_params->ifindex = veth1_idx; + if (!ASSERT_EQ(inet_pton(AF_INET, IPV4_IFACE_ADDR, &fib_params->ipv4_src), + 1, "inet_pton(src)") || + !ASSERT_EQ(inet_pton(AF_INET, IPV4_QINQ_DST, &fib_params->ipv4_dst), + 1, "inet_pton(qinq dst)")) + goto fail; + skel->bss->lookup_flags = BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_SKIP_NEIGH; + skel->bss->redirected = 0; + skel->bss->passed = 0; + + err = bpf_prog_test_run_opts(redirect_fd, &lf_opts); + if (!ASSERT_OK(err, "test_run(qinq egress)")) + goto fail; + ASSERT_EQ(skel->bss->passed, REDIRECT_NPKTS, "qinq egress passed"); + ASSERT_EQ(skel->bss->redirected, 0, "qinq egress not redirected"); + +fail: + if (xdp_attached) + bpf_xdp_detach(veth2_idx, XDP_FLAGS_DRV_MODE, NULL); + if (nstoken) + close_netns(nstoken); + SYS_NOFAIL("ip netns del " NS_REDIRECT); + fib_lookup__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c index f589eefbf9fb..0918321c8e63 100644 --- a/tools/testing/selftests/bpf/prog_tests/fill_link_info.c +++ b/tools/testing/selftests/bpf/prog_tests/fill_link_info.c @@ -4,6 +4,7 @@ #include <string.h> #include <linux/bpf.h> #include <linux/limits.h> +#include <bpf/btf.h> #include <test_progs.h> #include "trace_helpers.h" #include "test_fill_link_info.skel.h" @@ -24,6 +25,22 @@ static __u64 kmulti_cookies[] = { 3, 1, 2 }; #define KPROBE_FUNC "bpf_fentry_test1" static __u64 kprobe_addr; +static const char * const tmulti_syms[] = { + "bpf_fentry_test2", + "bpf_fentry_test1", + "bpf_fentry_test3", +}; + +static __u64 tmulti_cookies[] = { 30, 10, 20 }; +#define TRACING_MULTI_CNT ARRAY_SIZE(tmulti_syms) + +struct tmulti_target { + const char *name; + __u64 addr; + __u64 cookie; + __u32 id; +}; + #define UPROBE_FILE "/proc/self/exe" static ssize_t uprobe_offset; /* uprobe attach point */ @@ -396,6 +413,224 @@ static void test_kprobe_multi_fill_link_info(struct test_fill_link_info *skel, bpf_link__destroy(link); } +static int tmulti_target_cmp(const void *a, const void *b) +{ + const struct tmulti_target *ta = a; + const struct tmulti_target *tb = b; + + return (ta->id > tb->id) - (ta->id < tb->id); +} + +static int setup_tmulti_targets(const struct bpf_program *prog, + struct tmulti_target *targets, + __u32 *btf_obj_id) +{ + struct bpf_prog_info prog_info; + __u32 len = sizeof(prog_info); + struct btf *btf; + int err, i; + __s32 id; + + btf = btf__load_vmlinux_btf(); + if (!ASSERT_OK_PTR(btf, "btf__load_vmlinux_btf")) + return -1; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + id = btf__find_by_name_kind(btf, tmulti_syms[i], BTF_KIND_FUNC); + if (!ASSERT_GT(id, 0, "btf__find_by_name_kind")) + goto error; + + targets[i].name = tmulti_syms[i]; + targets[i].addr = ksym_get_addr(tmulti_syms[i]); + targets[i].cookie = tmulti_cookies[i]; + targets[i].id = id; + } + + memset(&prog_info, 0, len); + err = bpf_prog_get_info_by_fd(bpf_program__fd(prog), &prog_info, &len); + if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd")) + goto error; + if (!ASSERT_GT(prog_info.attach_btf_obj_id, 0, "attach_btf_obj_id")) + goto error; + *btf_obj_id = prog_info.attach_btf_obj_id; + + /* + * The kernel tracing multi attach sorts ids. We sort as well, + * so we can easily compare ids and cookies later. + */ + qsort(targets, TRACING_MULTI_CNT, sizeof(targets[0]), tmulti_target_cmp); + btf__free(btf); + return 0; + +error: + btf__free(btf); + return -1; +} + +static int verify_tracing_multi_link_info(int fd, const struct bpf_program *prog, + const struct tmulti_target *targets, + __u32 btf_obj_id, bool has_cookies) +{ + enum bpf_attach_type attach_type = bpf_program__expected_attach_type(prog); + __u64 addrs[TRACING_MULTI_CNT], cookies[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT]; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + memset(&info, 0, sizeof(info)); + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + + memset(ids, 0, sizeof(ids)); + memset(cookies, 0, sizeof(cookies)); + memset(addrs, 0, sizeof(addrs)); + + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.addrs = ptr_to_u64(addrs); + info.tracing_multi.cookies = has_cookies ? ptr_to_u64(cookies) : 0; + info.tracing_multi.count = TRACING_MULTI_CNT; + + err = bpf_link_get_info_by_fd(fd, &info, &len); + if (!ASSERT_OK(err, "bpf_link_get_info_by_fd")) + return -1; + + if (!ASSERT_EQ(info.type, BPF_LINK_TYPE_TRACING_MULTI, "info.type")) + return -1; + + ASSERT_EQ(info.tracing_multi.attach_type, attach_type, "info.tracing_multi.attach_type"); + ASSERT_EQ(info.tracing_multi.count, TRACING_MULTI_CNT, "info.tracing_multi.count"); + ASSERT_EQ(info.tracing_multi.btf_obj_id, btf_obj_id, "tracing_multi.btf_obj_id"); + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + ASSERT_EQ(cookies[i], has_cookies ? targets[i].cookie : 0, "tracing_multi.cookies"); + + if (targets[i].addr) { + struct ksym *ksym; + + if (!ASSERT_NEQ(addrs[i], 0, "tracing_multi.addrs")) + return -1; + ksym = ksym_search(addrs[i]); + if (!ASSERT_OK_PTR(ksym, "ksym_search")) + return -1; + ASSERT_STREQ(ksym->name, targets[i].name, "tracing_multi.addr_name"); + } else { + ASSERT_EQ(addrs[i], 0, "tracing_multi.addrs"); + } + } + + return 0; +} + +static void verify_tracing_multi_invalid_user_buffer(int fd, const struct tmulti_target *targets) +{ + __u32 ids[TRACING_MULTI_CNT] = {}; + struct bpf_link_info info; + __u32 len = sizeof(info); + int err, i; + + /* Wrong info setup (ids != NULL and cnt == 0) -> EINVAL */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EINVAL, "tracing_multi.invalid_count"); + + /* Smaller than actual count provided -> ENOSPC */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT - 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -ENOSPC, "tracing_multi.small_count"); + for (i = 0; i < TRACING_MULTI_CNT - 1; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.partial_ids"); + /* check that the last entry is not populated */ + ASSERT_EQ(ids[i], 0, "tracing_multi.partial_ids"); + + /* Bigger than actual count provided -> OK */ + memset(ids, 0, sizeof(ids)); + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = ptr_to_u64(ids); + info.tracing_multi.count = TRACING_MULTI_CNT + 1; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_OK(err, "tracing_multi.big_count"); + for (i = 0; i < TRACING_MULTI_CNT; i++) + ASSERT_EQ(ids[i], targets[i].id, "tracing_multi.ids"); + + /* Invalid ids pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.ids = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_btf_ids"); + + /* Invalid cookies pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.cookies = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_cookies"); + + /* Invalid addrs pointer -> EFAULT */ + memset(&info, 0, sizeof(info)); + info.tracing_multi.addrs = 0x1; + info.tracing_multi.count = TRACING_MULTI_CNT; + err = bpf_link_get_info_by_fd(fd, &info, &len); + ASSERT_EQ(err, -EFAULT, "tracing_multi.bad_addrs"); +} + +static void test_tracing_multi_fill_link_info(struct test_fill_link_info *skel, + bool has_cookies, bool invalid) +{ + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + struct tmulti_target targets[TRACING_MULTI_CNT]; + __u32 ids[TRACING_MULTI_CNT], btf_obj_id; + __u64 cookies[TRACING_MULTI_CNT]; + struct bpf_link *link; + int link_fd, err, i; + +#ifndef __x86_64__ + test__skip(); + return; +#endif + + if (setup_tmulti_targets(skel->progs.tmulti_run, targets, &btf_obj_id)) + return; + + for (i = 0; i < TRACING_MULTI_CNT; i++) { + ids[i] = targets[i].id; + cookies[i] = targets[i].cookie; + } + + opts.ids = ids; + opts.cnt = TRACING_MULTI_CNT; + if (has_cookies) + opts.cookies = cookies; + + link = bpf_program__attach_tracing_multi(skel->progs.tmulti_run, NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_tracing_multi")) + return; + + link_fd = bpf_link__fd(link); + if (invalid) { + verify_tracing_multi_invalid_user_buffer(link_fd, targets); + } else { + err = verify_tracing_multi_link_info(link_fd, skel->progs.tmulti_run, + targets, btf_obj_id, has_cookies); + ASSERT_OK(err, "verify_tracing_multi_link_info"); + } + + bpf_link__destroy(link); +} + #define SEC(name) __attribute__((section(name), used)) static short uprobe_link_info_sema_1 SEC(".probes"); @@ -640,6 +875,13 @@ void test_fill_link_info(void) if (test__start_subtest("kprobe_multi_invalid_ubuff")) test_kprobe_multi_fill_link_info(skel, true, true, true); + if (test__start_subtest("tracing_multi_link_info")) { + test_tracing_multi_fill_link_info(skel, false, false); + test_tracing_multi_fill_link_info(skel, true, false); + } + if (test__start_subtest("tracing_multi_invalid_ubuff")) + test_tracing_multi_fill_link_info(skel, true, true); + if (test__start_subtest("uprobe_multi_link_info")) test_uprobe_multi_fill_link_info(skel, false, false); if (test__start_subtest("uretprobe_multi_link_info")) diff --git a/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c b/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c new file mode 100644 index 000000000000..1b5c738ab81f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/get_smp_processor_id.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <test_progs.h> +#include "bpf/libbpf_internal.h" +#include "get_smp_processor_id.skel.h" + +void test_get_smp_processor_id(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts, + .flags = BPF_F_TEST_RUN_ON_CPU, + .cpu = 0, + ); + struct get_smp_processor_id *skel; + int prog_fd, err, online_cpu_nr, i; + bool *online = NULL; + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", + &online, &online_cpu_nr); + if (!ASSERT_OK(err, "parse_cpu_mask_file")) + return; + + skel = get_smp_processor_id__open_and_load(); + if (!ASSERT_OK_PTR(skel, "get_smp_processor_id__open_and_load")) + goto cleanup; + + prog_fd = bpf_program__fd(skel->progs.call_bpf_get_smp_processor_id); + + for (i = 0; i < online_cpu_nr; i++) { + if (!online[i]) + continue; + + opts.cpu = i; + skel->bss->cpu_nr_result = -1; + + err = bpf_prog_test_run_opts(prog_fd, &opts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto cleanup; + + ASSERT_EQ(skel->bss->cpu_nr_result, opts.cpu, "cpu_nr_result"); + } + +cleanup: + free(online); + get_smp_processor_id__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 8466332d7406..5671c31085cd 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -1,5 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> +#include "bpf/libbpf_internal.h" +#include "test_global_percpu_data.skel.h" +#include "test_global_percpu_data.lskel.h" void test_global_data_init(void) { @@ -60,3 +63,336 @@ out: free(newval); bpf_object__close(obj); } + +static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_fd, int *runp) +{ + struct test_global_percpu_data__percpu *data = NULL; + int i, err, key = 0, num_online, run = 0; + __u64 args[2] = {0x1234ULL, 0x5678ULL}; + size_t data_sz; + bool *online; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", &online, &num_online); + if (!ASSERT_OK(err, "parse_cpu_mask_file")) + return; + + data_sz = map ? bpf_map__value_size(map) : sizeof(*data); + data = calloc(1, data_sz); + if (!ASSERT_OK_PTR(data, "calloc percpu data")) + goto out; + + /* run on every online-CPU */ + for (i = 0; i < num_online; i++) { + __u64 flags; + + if (!online[i]) + continue; + + topts.cpu = i; + topts.retval = -1; + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + ASSERT_EQ(topts.retval, 0, "bpf_prog_test_run_opts retval"); + + memset(data, 0, data_sz); + flags = ((__u64) i << 32) | BPF_F_CPU; + if (map) + err = bpf_map__lookup_elem(map, &key, sizeof(key), data, data_sz, flags); + else + err = bpf_map_lookup_elem_flags(map_fd, &key, data, flags); + if (!ASSERT_OK(err, "lookup_elem on cpu")) + break; + + ASSERT_EQ(*runp, ++run, "run"); + ASSERT_EQ(data->cpu_id[0], i, "cpu_id"); + ASSERT_EQ(data->data, 1, "data"); + ASSERT_TRUE(data->set, "set"); + ASSERT_EQ(data->nums[6], 0xc0de, "nums[6]"); + ASSERT_EQ(data->struct_data.i, 1, "struct_data.i"); + ASSERT_TRUE(data->struct_data.set, "struct_data.set"); + ASSERT_EQ(data->struct_data.nums[6], 0xc0de, "struct_data.nums[6]"); + } + +out: + free(data); + free(online); +} + +static void test_global_percpu_data_init(void) +{ + struct test_global_percpu_data__percpu init_value = {}; + struct test_global_percpu_data__percpu *init_data; + const __u32 desired_sz = sysconf(_SC_PAGE_SIZE); + struct test_global_percpu_data *skel = NULL; + size_t init_data_sz; + struct bpf_map *map; + int prog_fd, err; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu, "skel->percpu")) + goto out; + if (!ASSERT_OK_PTR(skel->data_percpu, "skel->data_percpu")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_data, "skel->percpu_data")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_looooooooong, "skel->percpu_looooooooong")) + goto out; + + ASSERT_STREQ(bpf_map__name(skel->maps.percpu_data), ".percpu.data", + ".percpu.data map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.data_percpu), ".data.percpu", + ".data.percpu map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.percpu_looooooooong), ".percpu.looooooooong", + "long map name"); + ASSERT_STREQ(bpf_map__name(skel->maps.percpu), ".percpu", "map name"); + ASSERT_EQ(skel->percpu->data, -1, "skel->percpu->data"); + ASSERT_FALSE(skel->percpu->set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->nums[6], 0, "skel->percpu->nums[6]"); + ASSERT_EQ(skel->percpu->struct_data.i, -1, "struct_data.i"); + ASSERT_FALSE(skel->percpu->struct_data.set, "struct_data.set"); + ASSERT_EQ(skel->percpu->struct_data.nums[6], 0, "struct_data.nums[6]"); + + map = skel->maps.percpu; + if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type")) + goto out; + + init_value.data = 2; + init_value.nums[6] = -1; + init_value.struct_data.i = 2; + init_value.struct_data.nums[6] = -1; + err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value)); + if (!ASSERT_OK(err, "bpf_map__set_initial_value")) + goto out; + + init_data = bpf_map__initial_value(map, &init_data_sz); + if (!ASSERT_OK_PTR(init_data, "bpf_map__initial_value")) + goto out; + + ASSERT_EQ(init_data->data, init_value.data, "init_value data"); + ASSERT_EQ(init_data->set, init_value.set, "init_value set"); + ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i"); + ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6], + "init_value struct_data.nums[6]"); + ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size"); + ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data"); + ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data"); + ASSERT_EQ(skel->percpu->set, init_value.set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i, + "skel->percpu->struct_data.i"); + ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6], + "skel->percpu->struct_data.nums[6]"); + + ASSERT_GT(desired_sz, sizeof(init_value), "desired_sz"); + err = bpf_map__set_value_size(map, desired_sz); + if (!ASSERT_OK(err, "bpf_map__set_value_size")) + goto out; + if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "percpu value size")) + goto out; + if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "percpu BTF value type")) + goto out; + + init_data = bpf_map__initial_value(map, &init_data_sz); + if (!ASSERT_OK_PTR(init_data, "resized bpf_map__initial_value")) + goto out; + if (!ASSERT_EQ(init_data_sz, desired_sz, "resized initial value size")) + goto out; + if (!ASSERT_EQ(init_data->data, init_value.data, "resized initial value data")) + goto out; + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + ASSERT_OK_PTR(skel->percpu, "skel->percpu"); + + prog_fd = bpf_program__fd(skel->progs.update_percpu_data); + test_percpu_data_on_cpus(map, bpf_map__fd(map), prog_fd, &skel->bss->run); + +out: + test_global_percpu_data__destroy(skel); +} + +static void test_global_percpu_data_lskel(void) +{ + struct test_global_percpu_data_lskel *lskel = NULL; + int prog_fd, map_fd; + + lskel = test_global_percpu_data_lskel__open_and_load(); + if (!ASSERT_OK_PTR(lskel, "test_global_percpu_data_lskel__open_and_load")) + goto out; + + map_fd = lskel->maps.percpu.map_fd; + prog_fd = lskel->progs.update_percpu_data.prog_fd; + test_percpu_data_on_cpus(NULL, map_fd, prog_fd, &lskel->bss->run); + +out: + test_global_percpu_data_lskel__destroy(lskel); +} + +static int create_rdonly_percpu_array(void) +{ + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .map_flags = BPF_F_RDONLY_PROG, + ); + int key = 0, map_fd, err; + __u64 value = 0; + + map_fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "percpu_ro_map", sizeof(int), + sizeof(__u64), 1, &map_opts); + if (!ASSERT_GE(map_fd, 0, "bpf_map_create")) + return -1; + + err = bpf_map_update_elem(map_fd, &key, &value, BPF_F_ALL_CPUS); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out; + + err = bpf_map_freeze(map_fd); + if (!ASSERT_OK(err, "bpf_map_freeze")) + goto out; + + return map_fd; + +out: + close(map_fd); + return -1; +} + +static void test_global_percpu_data_rdonly_direct_read(void) +{ + /* + * Raw instructions with manually prepared rdonly percpu_array map + * for testing direct-read global percpu data, because libbpf + * doesn't have rdonly internal percpu_array map support for + * global percpu data. + */ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_EXIT_INSN(), + }; + int map_fd, prog_fd; + + map_fd = create_rdonly_percpu_array(); + if (map_fd < 0) + return; + + insns[0].imm = map_fd; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns, + ARRAY_SIZE(insns), NULL); + if (ASSERT_GE(prog_fd, 0, "bpf_prog_load")) + close(prog_fd); + close(map_fd); +} + +static void test_global_percpu_data_rdonly_direct_write(void) +{ + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts); + /* See the comment in test_global_percpu_data_rdonly_direct_read() */ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 0), + BPF_EXIT_INSN(), + }; + char log_buf[256] = {}; + int map_fd, prog_fd; + + prog_opts.log_buf = log_buf; + prog_opts.log_size = sizeof(log_buf); + prog_opts.log_level = 1; + + map_fd = create_rdonly_percpu_array(); + if (map_fd < 0) + return; + + insns[0].imm = map_fd; + prog_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "percpu_ro_prog", "GPL", insns, + ARRAY_SIZE(insns), &prog_opts); + if (!ASSERT_LT(prog_fd, 0, "bpf_prog_load")) + close(prog_fd); + else + ASSERT_HAS_SUBSTR(log_buf, "write into map forbidden", "verifier log"); + close(map_fd); +} + +static void test_global_percpu_data_verifier_log(void) +{ + RUN_TESTS(test_global_percpu_data); +} + +static void test_global_percpu_data_iter(void) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + struct test_global_percpu_data *skel; + union bpf_iter_link_info linfo = {}; + struct bpf_link *link = NULL; + int fd, num_cpus, len, err; + char buf[16]; + + num_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(num_cpus, 0, "libbpf_num_possible_cpus")) + return; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + return; + + skel->rodata->num_cpus = num_cpus; + skel->rodata->num_off = offsetof(struct test_global_percpu_data__percpu, + struct_data.nums[6]); + skel->rodata->elem_sz = roundup(sizeof(struct test_global_percpu_data__percpu), 8); + skel->percpu->struct_data.nums[6] = 0xc0de; + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + linfo.map.map_fd = bpf_map__fd(skel->maps.percpu); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + link = bpf_program__attach_iter(skel->progs.dump_percpu_data, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_iter")) + goto out; + + fd = bpf_iter_create(bpf_link__fd(link)); + if (!ASSERT_GE(fd, 0, "bpf_iter_create")) + goto out; + + while ((len = read(fd, buf, sizeof(buf))) > 0) + do { } while (0); + ASSERT_EQ(len, 0, "read iter"); + ASSERT_TRUE(skel->bss->run_iter, "run_iter"); + ASSERT_EQ(skel->bss->sum, 0xc0de * num_cpus, "sum"); + + close(fd); +out: + bpf_link__destroy(link); + test_global_percpu_data__destroy(skel); +} + +void test_global_percpu_data(void) +{ + if (!feat_supported(NULL, FEAT_PERCPU_DATA)) { + test__skip(); + return; + } + + if (test__start_subtest("init")) + test_global_percpu_data_init(); + if (test__start_subtest("lskel")) + test_global_percpu_data_lskel(); + if (test__start_subtest("rdonly_direct_read")) + test_global_percpu_data_rdonly_direct_read(); + if (test__start_subtest("rdonly_direct_write")) + test_global_percpu_data_rdonly_direct_write(); + test_global_percpu_data_verifier_log(); + if (test__start_subtest("iter")) + test_global_percpu_data_iter(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c index 56b5baef35c8..602ce30f1720 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c +++ b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c @@ -23,6 +23,7 @@ static void global_map_resize_bss_subtest(void) struct bpf_map *map; const __u32 desired_sz = sizeof(skel->bss->sum) + sysconf(_SC_PAGE_SIZE) * 2; size_t array_len, actual_sz, new_sz; + int *array; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -58,10 +59,13 @@ static void global_map_resize_bss_subtest(void) goto teardown; /* fill the newly resized array with ones, - * skipping the first element which was previously set + * skipping the first element which was previously set; + * access through a plain pointer to avoid -Warray-bounds + * since the array was resized beyond its declared length. */ + array = skel->bss->array; for (int i = 1; i < array_len; i++) - skel->bss->array[i] = 1; + array[i] = 1; /* set global const values before loading */ skel->rodata->pid = getpid(); diff --git a/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c b/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c new file mode 100644 index 000000000000..9318d4bc7ce8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/icmp_send_kfunc.c @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <network_helpers.h> +#include <cgroup_helpers.h> +#include <linux/errqueue.h> +#include <poll.h> +#include <unistd.h> +#include "icmp_send.skel.h" + +#define TIMEOUT_MS 1000 + +#define ICMP_DEST_UNREACH 3 +#define ICMPV6_DEST_UNREACH 1 + +#define ICMP_HOST_UNREACH 1 +#define ICMP_FRAG_NEEDED 4 +#define NR_ICMP_UNREACH 15 +#define ICMPV6_REJECT_ROUTE 6 + +#define KFUNC_RET_UNSET -1 + +static int connect_to_fd_nonblock(int server_fd) +{ + struct sockaddr_storage addr; + socklen_t len = sizeof(addr); + int fd, err, on = 1; + + if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) + return -1; + + fd = socket(addr.ss_family, SOCK_STREAM | SOCK_NONBLOCK, 0); + if (fd < 0) + return -1; + + if (addr.ss_family == AF_INET6 && + setsockopt(fd, IPPROTO_IPV6, IPV6_RECVERR, &on, sizeof(on)) < 0) { + close(fd); + return -1; + } + + err = connect(fd, (struct sockaddr *)&addr, len); + if (err < 0 && errno != EINPROGRESS) { + close(fd); + return -1; + } + + return fd; +} + +static void read_icmp_errqueue(int sockfd, int expected_code, int af) +{ + int expected_ee_type = (af == AF_INET) ? ICMP_DEST_UNREACH : + ICMPV6_DEST_UNREACH; + int expected_origin = (af == AF_INET) ? SO_EE_ORIGIN_ICMP : + SO_EE_ORIGIN_ICMP6; + int expected_level = (af == AF_INET) ? IPPROTO_IP : IPPROTO_IPV6; + int expected_type = (af == AF_INET) ? IP_RECVERR : IPV6_RECVERR; + struct sock_extended_err *sock_err; + char ctrl_buf[512]; + struct msghdr msg = { + .msg_control = ctrl_buf, + .msg_controllen = sizeof(ctrl_buf), + }; + struct pollfd pfd = { + .fd = sockfd, + .events = POLLERR, + }; + struct cmsghdr *cm; + ssize_t n; + + if (!ASSERT_GE(poll(&pfd, 1, TIMEOUT_MS), 1, "poll_errqueue")) + return; + + n = recvmsg(sockfd, &msg, MSG_ERRQUEUE); + if (!ASSERT_GE(n, 0, "recvmsg_errqueue")) + return; + + cm = CMSG_FIRSTHDR(&msg); + if (!ASSERT_NEQ(cm, NULL, "cm_firsthdr_null")) + return; + + for (; cm; cm = CMSG_NXTHDR(&msg, cm)) { + if (cm->cmsg_level != expected_level || + cm->cmsg_type != expected_type) + continue; + + sock_err = (struct sock_extended_err *)CMSG_DATA(cm); + + if (!ASSERT_EQ(sock_err->ee_origin, expected_origin, + "sock_err_origin")) + return; + if (!ASSERT_EQ(sock_err->ee_type, expected_ee_type, + "sock_err_type_dest_unreach")) + return; + ASSERT_EQ(sock_err->ee_code, expected_code, "sock_err_code"); + return; + } + + ASSERT_FAIL("no IP_RECVERR/IPV6_RECVERR control message found"); +} + +static bool valid_unreach_code(int code, int af) +{ + if (code < 0) + return false; + + if (af == AF_INET) + return code <= NR_ICMP_UNREACH && code != ICMP_FRAG_NEEDED; + + return code <= ICMPV6_REJECT_ROUTE; +} + +static void trigger_prog_read_icmp_errqueue(struct icmp_send *skel, int code, + int af, const char *ip) +{ + int srv_fd = -1, client_fd = -1; + int port; + + srv_fd = start_server(af, SOCK_STREAM, ip, 0, TIMEOUT_MS); + if (!ASSERT_OK_FD(srv_fd, "start_server")) + return; + + port = get_socket_local_port(srv_fd); + if (!ASSERT_GE(port, 0, "get_socket_local_port")) { + close(srv_fd); + return; + } + + skel->bss->server_port = ntohs(port); + skel->bss->unreach_type = (af == AF_INET) ? ICMP_DEST_UNREACH : + ICMPV6_DEST_UNREACH; + skel->bss->unreach_code = code; + skel->data->kfunc_ret = KFUNC_RET_UNSET; + + client_fd = connect_to_fd_nonblock(srv_fd); + if (!ASSERT_OK_FD(client_fd, "client_connect_nonblock")) { + close(srv_fd); + return; + } + + if (valid_unreach_code(code, af)) + read_icmp_errqueue(client_fd, code, af); + + close(client_fd); + close(srv_fd); +} + +static void run_icmp_test(struct icmp_send *skel, int af, const char *ip, + int max_code) +{ + for (int code = 0; code <= max_code; code++) { + if (af == AF_INET && code == ICMP_FRAG_NEEDED) + continue; + + trigger_prog_read_icmp_errqueue(skel, code, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, 0, "kfunc_ret"); + } + + /* Test invalid codes */ + trigger_prog_read_icmp_errqueue(skel, -1, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + + trigger_prog_read_icmp_errqueue(skel, max_code + 1, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + + if (af == AF_INET) { + trigger_prog_read_icmp_errqueue(skel, ICMP_FRAG_NEEDED, af, ip); + ASSERT_EQ(skel->data->kfunc_ret, -EINVAL, "kfunc_ret"); + } +} + +static void run_icmp_no_route_test(struct icmp_send *skel, int af) +{ + union { + struct ipv4_packet v4; + struct ipv6_packet v6; + } pkt; + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = &pkt, + ); + int err; + + switch (af) { + case AF_INET: + pkt.v4 = pkt_v4; + pkt.v4.iph.version = 4; + pkt.v4.iph.daddr = htonl(INADDR_LOOPBACK); + pkt.v4.tcp.dest = htons(80); + opts.data_size_in = sizeof(pkt.v4); + skel->bss->unreach_type = ICMP_DEST_UNREACH; + break; + case AF_INET6: + pkt.v6 = pkt_v6; + pkt.v6.iph.version = 6; + pkt.v6.iph.daddr = in6addr_loopback; + pkt.v6.tcp.dest = htons(80); + opts.data_size_in = sizeof(pkt.v6); + skel->bss->unreach_type = ICMPV6_DEST_UNREACH; + break; + default: + ASSERT_FAIL("af_not_supported"); + return; + } + + skel->bss->server_port = 80; + skel->data->kfunc_ret = KFUNC_RET_UNSET; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.egress), &opts); + if (!ASSERT_OK(err, "test_run")) + return; + + ASSERT_EQ(skel->data->kfunc_ret, -ENETUNREACH, "kfunc_ret_no_route"); +} + +void test_icmp_send_unreach_cgroup(void) +{ + struct icmp_send *skel; + int cgroup_fd = -1; + + skel = icmp_send__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + cgroup_fd = test__join_cgroup("/icmp_send_unreach_cgroup"); + if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup")) + goto cleanup; + + skel->links.egress = + bpf_program__attach_cgroup(skel->progs.egress, cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.egress, "prog_attach_cgroup")) + goto cleanup; + + if (test__start_subtest("ipv4")) + run_icmp_test(skel, AF_INET, "127.0.0.1", NR_ICMP_UNREACH); + + if (test__start_subtest("ipv6")) + run_icmp_test(skel, AF_INET6, "::1", ICMPV6_REJECT_ROUTE); + + if (test__start_subtest("no_route_ipv4")) + run_icmp_no_route_test(skel, AF_INET); + + if (test__start_subtest("no_route_ipv6")) + run_icmp_no_route_test(skel, AF_INET6); + +cleanup: + icmp_send__destroy(skel); + if (cgroup_fd >= 0) + close(cgroup_fd); +} + +void test_icmp_send_unreach_recursion(void) +{ + struct icmp_send *skel; + int cgroup_fd = -1; + int err; + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + return; + + skel = icmp_send__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + + cgroup_fd = get_root_cgroup(); + if (!ASSERT_OK_FD(cgroup_fd, "get_root_cgroup")) + goto cleanup; + + skel->data->target_pid = getpid(); + skel->links.recursion = + bpf_program__attach_cgroup(skel->progs.recursion, cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.recursion, "prog_attach_cgroup")) + goto cleanup; + + trigger_prog_read_icmp_errqueue(skel, ICMP_HOST_UNREACH, AF_INET, + "127.0.0.1"); + + /* + * Because there's recursion involved, the first call will return at + * index 1 since it will return the second, and the second call will + * return at index 0 since it will return the first. + */ + ASSERT_EQ(skel->bss->rec_count, 2, "rec_count"); + ASSERT_EQ(skel->data->rec_kfunc_rets[0], -EBUSY, "kfunc_rets[0]"); + ASSERT_EQ(skel->data->rec_kfunc_rets[1], 0, "kfunc_rets[1]"); + +cleanup: + icmp_send__destroy(skel); + if (cgroup_fd >= 0) + close(cgroup_fd); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c index 3df07680f9e0..2b39cc1b09f9 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c @@ -66,11 +66,15 @@ static struct kfunc_test_params kfunc_tests[] = { TC_FAIL(kfunc_call_test_get_mem_fail_rdonly, 0, "R0 cannot write into rdonly_mem"), TC_FAIL(kfunc_call_test_get_mem_fail_use_after_free, 0, "invalid mem access 'scalar'"), TC_FAIL(kfunc_call_test_get_mem_fail_oob, 0, "min value is outside of the allowed memory range"), + TC_FAIL(kfunc_call_test_get_mem_fail_zero_size, 0, "min value is outside of the allowed memory range"), + TC_FAIL(kfunc_call_test_get_mem_fail_oversized, 0, "allocation size exceeds u32 max"), TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"), TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"), TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 expected pointer to ctx, but got scalar"), + TC_FAIL(kfunc_call_test_spin_lock_unsafe, 0, "function calls are not allowed while holding a lock"), /* success cases */ + TC_TEST(kfunc_call_test_spin_lock_safe, 0), TC_TEST(kfunc_call_test1, 12), TC_TEST(kfunc_call_test2, 3), TC_TEST(kfunc_call_test4, -1234), diff --git a/tools/testing/selftests/bpf/prog_tests/ksock.c b/tools/testing/selftests/bpf/prog_tests/ksock.c new file mode 100644 index 000000000000..dd6b167623d9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ksock.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <arpa/inet.h> + +#include "test_progs.h" +#include "network_helpers.h" +#include "ksock_lsm.skel.h" +#include "ksock_lsm_verifier.skel.h" + +#define NS_TEST "ksock_lsm_ns" +#define RECV_PORT 7777 +#define RECV_TIMEOUT_SEC 5 + +struct ksock_test_env { + struct nstoken *nstoken; + int rfd; +}; + +static bool ksock_test_env_setup(struct ksock_test_env *env) +{ + struct sockaddr_in addr = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + .sin_port = htons(RECV_PORT), + }; + struct timeval tv = { .tv_sec = RECV_TIMEOUT_SEC }; + int err; + + memset(env, 0, sizeof(*env)); + env->rfd = -1; + + if (!ASSERT_OK(make_netns(NS_TEST), "make_netns")) + goto fail; + + env->nstoken = open_netns(NS_TEST); + if (!ASSERT_OK_PTR(env->nstoken, "open_netns")) + goto fail; + + env->rfd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (!ASSERT_OK_FD(env->rfd, "receiver socket")) + goto fail; + + err = bind(env->rfd, (struct sockaddr *)&addr, sizeof(addr)); + if (!ASSERT_OK(err, "bind receiver")) + goto fail; + + err = setsockopt(env->rfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + if (!ASSERT_OK(err, "set rcvtimeo")) + goto fail; + + return true; + +fail: + return false; +} + +void test_ksock_lsm(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct ksock_test_env env; + struct sockaddr_in trigger_addr = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + }; + struct ksock_lsm *skel; + char recv_data[sizeof(skel->data->send_data)] = {}; + ssize_t n; + int tfd = -1; + int err; + + skel = ksock_lsm__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel open_and_load")) + return; + + if (!ksock_test_env_setup(&env)) + goto fail; + + /* Step 1: Run the setup SYSCALL prog to create the ksock */ + skel->bss->ipv4_remote = htonl(INADDR_LOOPBACK); + skel->bss->remote_port = RECV_PORT; + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_setup), + &opts); + if (!ASSERT_OK(err, "ksock_setup run")) + goto fail; + if (!ASSERT_OK(opts.retval, "ksock_setup retval")) + goto fail; + + /* Step 2: Attach LSM prog and trigger socket_bind from userspace */ + skel->links.ksock_socket_bind = + bpf_program__attach_lsm(skel->progs.ksock_socket_bind); + if (!ASSERT_OK_PTR(skel->links.ksock_socket_bind, + "attach socket_bind lsm")) + goto fail; + + tfd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if (!ASSERT_OK_FD(tfd, "trigger socket")) + goto fail; + + skel->bss->target_pid = getpid(); + err = bind(tfd, (struct sockaddr *)&trigger_addr, sizeof(trigger_addr)); + skel->bss->target_pid = 0; + if (!ASSERT_OK(err, "trigger bind")) + goto fail; + + /* Step 3: Verify the LSM hook sent the notification */ + if (!ASSERT_EQ(skel->data->send_ret, sizeof(skel->data->send_data), + "LSM send bytes")) + goto fail; + + n = recvfrom(env.rfd, recv_data, sizeof(recv_data), 0, NULL, NULL); + if (ASSERT_EQ(n, sizeof(recv_data), "recvfrom len")) + ASSERT_MEMEQ(recv_data, skel->data->send_data, sizeof(recv_data), + "payload match"); + +fail: + if (tfd >= 0) + close(tfd); + if (env.rfd >= 0) + close(env.rfd); + if (env.nstoken) + close_netns(env.nstoken); + remove_netns(NS_TEST); + ksock_lsm__destroy(skel); +} + +void test_ksock_lsm_verifier(void) +{ + RUN_TESTS(ksock_lsm_verifier); +} diff --git a/tools/testing/selftests/bpf/prog_tests/ksock_wq.c b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c new file mode 100644 index 000000000000..d6dc20b8f95b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/ksock_wq.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include <unistd.h> + +#include "test_progs.h" +#include "ksock_wq.skel.h" + +#define CALLBACK_WAIT_RETRIES 1000 +#define CALLBACK_WAIT_US 1000 + +void test_ksock_wq(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct ksock_wq *skel; + u32 callback_done; + int err, i; + + skel = ksock_wq__open_and_load(); + if (!ASSERT_OK_PTR(skel, "ksock_wq open and load")) + return; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.ksock_wq_start), + &opts); + if (!ASSERT_OK(err, "run ksock_wq_start")) + goto out; + if (!ASSERT_OK(opts.retval, "ksock_wq_start retval")) + goto out; + + for (i = 0; i < CALLBACK_WAIT_RETRIES; i++) { + if (__atomic_load_n(&skel->bss->callback_done, __ATOMIC_ACQUIRE)) + break; + usleep(CALLBACK_WAIT_US); + } + callback_done = __atomic_load_n(&skel->bss->callback_done, + __ATOMIC_ACQUIRE); + if (!ASSERT_EQ(callback_done, 1, "workqueue callback completed")) + goto out; + + ASSERT_EQ(skel->bss->create_err, -EOPNOTSUPP, + "workqueue create rejected"); + +out: + ksock_wq__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/libarena.c b/tools/testing/selftests/bpf/prog_tests/libarena.c index 61ea68dce410..daade4150af6 100644 --- a/tools/testing/selftests/bpf/prog_tests/libarena.c +++ b/tools/testing/selftests/bpf/prog_tests/libarena.c @@ -15,7 +15,12 @@ static void run_libarena_test(struct libarena *skel, struct bpf_program *prog, { int ret; - if (!strstr(name, "test_buddy")) { + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); if (!ASSERT_OK(ret, "arena_buddy_reset")) return; @@ -24,7 +29,6 @@ static void run_libarena_test(struct libarena *skel, struct bpf_program *prog, ret = libarena_run_prog(bpf_program__fd(prog)); ASSERT_OK(ret, name); - } static void *run_libarena_parallel_prog(void *arg) @@ -69,6 +73,7 @@ static int run_libarena_parallel_test_workers(struct libarena *skel, uint32_t nthreads; void *thread_ret; int ret, err = 0; + int worker_err; int i; for (nthreads = 0; nthreads < UINT_MAX; nthreads++) { @@ -114,7 +119,22 @@ static int run_libarena_parallel_test_workers(struct libarena *skel, continue; } - err = err ?: (long)thread_ret; + worker_err = (long)thread_ret; + + /* + * A worker that bails out because another one already gave up + * reports -EINTR. It is collateral damage that carries no + * information, so skip it entirely: never let it become the + * reported error, and don't log it either. + */ + if (!worker_err || worker_err == -EINTR) + continue; + + if (!err) + err = worker_err; + + fprintf(stdout, "%.*s__%d returned %d\n", (int)prefixlen, name, + i, worker_err); } free(threads); @@ -198,7 +218,7 @@ static void run_libarena_parallel_test(struct libarena *skel, struct bpf_program run_libarena_parallel_fini(skel, name, prefixlen); } -void test_libarena(void) +void serial_test_libarena(void) { struct arena_alloc_reserve_args args; struct libarena *skel; diff --git a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c index d59d9dd12ef2..9c31b17dbf39 100644 --- a/tools/testing/selftests/bpf/prog_tests/libarena_asan.c +++ b/tools/testing/selftests/bpf/prog_tests/libarena_asan.c @@ -17,7 +17,12 @@ static void run_libarena_asan_test(struct libarena_asan *skel, { int ret; - if (!strstr(name, "test_buddy")) { + if (strstr(name, "test_buddy")) { + /* Buddy tests initialize the allocator directly. */ + ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_destroy)); + if (!ASSERT_OK(ret, "arena_buddy_destroy")) + return; + } else { ret = libarena_run_prog(bpf_program__fd(skel->progs.arena_buddy_reset)); if (!ASSERT_OK(ret, "arena_buddy_reset")) return; @@ -80,7 +85,7 @@ out: * Run the test depending on whether LLVM can compile arena ASAN * programs. */ -void test_libarena_asan(void) +void serial_test_libarena_asan(void) { #ifdef HAS_BPF_ARENA_ASAN run_test(); @@ -90,4 +95,3 @@ void test_libarena_asan(void) return; } - diff --git a/tools/testing/selftests/bpf/prog_tests/linked_list.c b/tools/testing/selftests/bpf/prog_tests/linked_list.c index 8defea0253ed..c3d133c6a00d 100644 --- a/tools/testing/selftests/bpf/prog_tests/linked_list.c +++ b/tools/testing/selftests/bpf/prog_tests/linked_list.c @@ -68,6 +68,7 @@ static struct { { "obj_type_id_oor", "local type ID argument must be in range [0, U32_MAX]" }, { "obj_new_no_composite", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" }, { "obj_new_no_struct", "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct" }, + { "obj_new_flex_array", "access beyond struct obj_new_flex" }, { "obj_drop_non_zero_off", "R1 must have zero offset when passed to release func" }, { "new_null_ret", "R0 invalid mem access 'ptr_or_null_'" }, { "obj_new_acq", "Unreleased reference id=" }, diff --git a/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c index a970798e1173..28bc4b117f41 100644 --- a/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c +++ b/tools/testing/selftests/bpf/prog_tests/lsm_bdev.c @@ -17,6 +17,7 @@ #include <stdlib.h> #include <string.h> #include <sys/stat.h> +#include <sys/sysmacros.h> #include <sys/types.h> #include <unistd.h> #include "lsm_bdev.skel.h" @@ -172,7 +173,7 @@ void test_lsm_bdev(void) if (!ASSERT_OK(stat(DM_DEV_PATH, &st), "stat dm dev")) goto remove_dm; - dev_key = (__u32)st.st_rdev; + dev_key = (major(st.st_rdev) << 20) | minor(st.st_rdev); /* Look up the device in the BPF map and verify. */ err = bpf_map__lookup_elem(skel->maps.verity_devices, diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c index 6606f0ed9a9a..39e8a3b8b6af 100644 --- a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c @@ -410,7 +410,8 @@ close_netns: static int check_ping_ok(const char *ns1) { SYS(fail, "ip netns exec %s ping -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP4_ADDR_DST); - SYS(fail, "ip netns exec %s ping6 -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP6_ADDR_DST); + SYS(fail, "ip netns exec %s %s -c 1 -W1 -I veth1 %s > /dev/null", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); return 0; fail: return -1; @@ -424,7 +425,8 @@ static int check_ping_fails(const char *ns1) if (!ret) return -1; - ret = SYS_NOFAIL("ip netns exec %s ping6 -c 1 -W1 -I veth1 %s", ns1, IP6_ADDR_DST); + ret = SYS_NOFAIL("ip netns exec %s %s -c 1 -W1 -I veth1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ret) return -1; @@ -657,9 +659,10 @@ static void lwt_ip_encap_vxlan(bool ipv4_encap) skel->bss->fexit_triggered = false; if (ipv4_encap) - SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); + SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); else - SYS(out, "ip netns exec %s ping6 -c 1 -W1 %s", ns1, IP6_ADDR_DST); + SYS(out, "ip netns exec %s %s -c 1 -W1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ASSERT_TRUE(skel->bss->fexit_triggered, "fexit_triggered")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/mptcp.c b/tools/testing/selftests/bpf/prog_tests/mptcp.c index 8fade8bdc451..32dfc1c511af 100644 --- a/tools/testing/selftests/bpf/prog_tests/mptcp.c +++ b/tools/testing/selftests/bpf/prog_tests/mptcp.c @@ -264,7 +264,7 @@ static int verify_mptcpify(int server_fd, int client_fd) return err; } -static int run_mptcpify(int cgroup_fd) +static int run_mptcpify(int cgroup_fd, int type) { int server_fd, client_fd, err = 0; struct mptcpify *mptcpify_skel; @@ -280,7 +280,7 @@ static int run_mptcpify(int cgroup_fd) goto out; /* without MPTCP */ - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); + server_fd = start_server(AF_INET, type, NULL, 0, 0); if (!ASSERT_GE(server_fd, 0, "start_server")) { err = -EIO; goto out; @@ -317,7 +317,14 @@ static void test_mptcpify(void) if (!ASSERT_OK_PTR(netns, "netns_new")) goto fail; - ASSERT_OK(run_mptcpify(cgroup_fd), "run_mptcpify"); + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM), "run_mptcpify"); + /* userspace sets flags such as SOCK_CLOEXEC together with the type; + * the BPF prog must still upgrade the socket to MPTCP. See + * update_socket_protocol() in net/socket.c, which runs before the + * type is masked with SOCK_TYPE_MASK. + */ + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM | SOCK_CLOEXEC), + "run_mptcpify_cloexec"); fail: netns_free(netns); diff --git a/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c b/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c index 246eb259c08a..6a07b2b418d1 100644 --- a/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c @@ -34,6 +34,8 @@ static void test_success(void) bpf_program__set_autoload(skel->progs.rcu_read_lock_global_subprog, true); bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog_lock, true); bpf_program__set_autoload(skel->progs.rcu_read_lock_subprog_unlock, true); + bpf_program__set_autoload(skel->progs.non_own_ref_untrusted_ld, true); + bpf_program__set_autoload(skel->progs.rcu_untrusted_union_ld, true); err = rcu_read_lock__load(skel); if (!ASSERT_OK(err, "skel_load")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c b/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c index f0a8c828f8f1..7541f4966abc 100644 --- a/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/res_spin_lock.c @@ -3,6 +3,8 @@ #include <test_progs.h> #include <network_helpers.h> #include <sys/sysinfo.h> +#include <sys/syscall.h> +#include <linux/perf_event.h> #include "res_spin_lock.skel.h" #include "res_spin_lock_fail.skel.h" @@ -102,11 +104,29 @@ end: void serial_test_res_spin_lock_stress(void) { + struct perf_event_attr attr = { + .size = sizeof(attr), + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + }; + int pmu_fd; + if (libbpf_num_possible_cpus() < 3) { test__skip(); return; } + pmu_fd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, 0); + if (pmu_fd < 0) { + if (errno == ENOENT || errno == EOPNOTSUPP) { + test__skip(); + return; + } + ASSERT_OK(-errno, "perf_event_open pmu probe"); + return; + } + close(pmu_fd); + ASSERT_OK(load_module("bpf_test_rqspinlock.ko", false), "load module AA"); sleep(5); unload_module("bpf_test_rqspinlock", false); diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c index 41dfaaabb73f..3f9949e8227d 100644 --- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c +++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c @@ -10,7 +10,25 @@ #include <linux/btf_ids.h> #include "test_progs.h" -static int duration; +#define BTF_DATA_FILE "resolve_btfids.test.o.BTF" + +#define DECL_TAG_FASTCALL "bpf_fastcall" +#define DECL_TAG_KFUNC "bpf_kfunc" +#define TYPE_ATTR_ARENA "address_space(1)" +#define ARENA_ARG(n) (1U << (n)) + +#ifndef KF_FASTCALL +#define KF_FASTCALL (1 << 12) +#endif +#ifndef KF_ARENA_RET +#define KF_ARENA_RET (1 << 13) +#endif +#ifndef KF_ARENA_ARG1 +#define KF_ARENA_ARG1 (1 << 14) +#endif +#ifndef KF_ARENA_ARG2 +#define KF_ARENA_ARG2 (1 << 15) +#endif struct symbol { const char *name; @@ -28,12 +46,50 @@ struct symbol test_symbols[] = { { "func", BTF_KIND_FUNC, -1 }, }; +struct kfunc_symbol { + const char *name; + s32 id; + u32 flags; + u32 arena_args; + bool arena_ret; +}; + +static struct kfunc_symbol kfunc_symbols[] = { + { "kfunc_a", -1, 0, 0, false }, + { "kfunc_b", -1, KF_FASTCALL, 0, false }, + { "kfunc_c", -1, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2, + ARENA_ARG(0) | ARENA_ARG(1), true }, + { "kfunc_d", -1, KF_ARENA_ARG2, ARENA_ARG(1), false }, + { "kfunc_e", -1, 0, ARENA_ARG(0) | ARENA_ARG(1) | ARENA_ARG(2) | + ARENA_ARG(3) | ARENA_ARG(4), false }, + { "kfunc_f", -1, 0, ARENA_ARG(1), false }, + { "kfunc_g", -1, KF_ARENA_RET, ARENA_ARG(0) | ARENA_ARG(1), true }, +}; + /* Align the .BTF_ids section to 4 bytes */ asm ( ".pushsection " BTF_IDS_SECTION " ,\"a\"; \n" ".balign 4, 0; \n" ".popsection; \n"); +/* + * test_list_local, test_set and test_kfunc_set are .local symbols placed + * in .BTF_ids by inline asm, and are read here directly by C name. To the + * compiler they are plain, default-visibility extern objects. + * + * When test_progs is linked as a position-independent executable (PIE), + * taking the address of such an extern is routed through the GOT. The + * GNU assembler on aarch64 unconditionally converts references to .local + * symbols into section + addend form (".BTF_ids + <offset>"), but a GOT + * slot cannot carry an addend (the AArch64 ELF spec mandates zero), so + * the linker resolves it to the .BTF_ids base. + * + * Mark them hidden so the compiler treats them as non-interposable and + * emits a direct, addend-preserving PC-relative access instead of a GOT + * load, in both PIE and non-PIE builds. test_list_global is .globl and + * not affected, so it is left at default visibility. + */ +#pragma GCC visibility push(hidden) BTF_ID_LIST(test_list_local) BTF_ID_UNUSED BTF_ID(typedef, S) @@ -43,24 +99,49 @@ BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) -extern __u32 test_list_global[]; -BTF_ID_LIST_GLOBAL(test_list_global, 1) -BTF_ID_UNUSED +BTF_SET_START(test_set) BTF_ID(typedef, S) BTF_ID(typedef, T) BTF_ID(typedef, U) BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) +BTF_SET_END(test_set) -BTF_SET_START(test_set) +BTF_KFUNCS_START(test_kfunc_set) +BTF_ID_FLAGS(func, kfunc_a) +BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_e) +BTF_ID_FLAGS(func, kfunc_f) +BTF_ID_FLAGS(func, kfunc_g, KF_ARENA_RET) +BTF_KFUNCS_END(test_kfunc_set) + +/* + * Same kfuncs in reverse declaration order, so resolve_btfids has to + * actually sort at least one of the two sets. + */ +BTF_KFUNCS_START(test_kfunc_set_rev) +BTF_ID_FLAGS(func, kfunc_g, KF_ARENA_RET) +BTF_ID_FLAGS(func, kfunc_f) +BTF_ID_FLAGS(func, kfunc_e) +BTF_ID_FLAGS(func, kfunc_d, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_c, KF_ARENA_RET | KF_ARENA_ARG1 | KF_ARENA_ARG2) +BTF_ID_FLAGS(func, kfunc_b, KF_FASTCALL) +BTF_ID_FLAGS(func, kfunc_a) +BTF_KFUNCS_END(test_kfunc_set_rev) +#pragma GCC visibility pop + +extern __u32 test_list_global[]; +BTF_ID_LIST_GLOBAL(test_list_global, 1) +BTF_ID_UNUSED BTF_ID(typedef, S) BTF_ID(typedef, T) BTF_ID(typedef, U) BTF_ID(struct, S) BTF_ID(union, U) BTF_ID(func, func) -BTF_SET_END(test_set) static int __resolve_symbol(struct btf *btf, int type_id) @@ -70,10 +151,10 @@ __resolve_symbol(struct btf *btf, int type_id) unsigned int i; type = btf__type_by_id(btf, type_id); - if (!type) { - PRINT_FAIL("Failed to get type for ID %d\n", type_id); + if (!ASSERT_OK_PTR(type, "btf__type_by_id")) return -1; - } + + str = btf__name_by_offset(btf, type->name_off); for (i = 0; i < ARRAY_SIZE(test_symbols); i++) { if (test_symbols[i].id >= 0) @@ -82,64 +163,118 @@ __resolve_symbol(struct btf *btf, int type_id) if (BTF_INFO_KIND(type->info) != test_symbols[i].type) continue; - str = btf__name_by_offset(btf, type->name_off); - if (!str) { - PRINT_FAIL("Failed to get name for BTF ID %d\n", type_id); - return -1; - } - if (!strcmp(str, test_symbols[i].name)) test_symbols[i].id = type_id; } + if (!btf_is_func(type)) + return 0; + + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + if (kfunc_symbols[i].id >= 0) + continue; + if (!strcmp(str, kfunc_symbols[i].name)) + kfunc_symbols[i].id = type_id; + } + return 0; } -static int resolve_symbols(void) +static int resolve_symbols(struct btf *btf) { - struct btf *btf; + __u32 nr = btf__type_cnt(btf); int type_id; - __u32 nr; - btf = btf__parse_raw("resolve_btfids.test.o.BTF"); - if (CHECK(libbpf_get_error(btf), "resolve", - "Failed to load BTF from resolve_btfids.test.o.BTF\n")) - return -1; + for (type_id = 1; type_id < nr; type_id++) { + if (__resolve_symbol(btf, type_id)) + return -1; + } + return 0; +} + +static bool btf_has_decl_tag(struct btf *btf, const char *tag_name, s32 target_id) +{ + const struct btf_type *t; + const char *name; + int nr, id; nr = btf__type_cnt(btf); + for (id = 1; id < nr; id++) { + t = btf__type_by_id(btf, id); + if (!btf_is_decl_tag(t)) + continue; + if (t->type != (__u32)target_id) + continue; + if (btf_decl_tag(t)->component_idx != -1) + continue; + name = btf__name_by_offset(btf, t->name_off); + if (strcmp(name, tag_name) == 0) + return true; + } + return false; +} - for (type_id = 1; type_id < nr; type_id++) { - if (__resolve_symbol(btf, type_id)) - break; +static void check_kfunc_set(struct btf_id_set8 *set) +{ + unsigned int i, j; + + ASSERT_EQ(set->flags, BTF_SET8_KFUNCS, "kfunc_set_flags"); + ASSERT_EQ(set->cnt, ARRAY_SIZE(kfunc_symbols), "kfunc_set_cnt"); + + for (i = 0; i < set->cnt; i++) { + for (j = 0; j < ARRAY_SIZE(kfunc_symbols); j++) { + if (kfunc_symbols[j].id == (s32)set->pairs[i].id) { + ASSERT_EQ(set->pairs[i].flags, + kfunc_symbols[j].flags, "kfunc_flags_check"); + break; + } + } + + ASSERT_TRUE(j < ARRAY_SIZE(kfunc_symbols), "kfunc_id_found"); + + if (i > 0) { + ASSERT_LE(set->pairs[i - 1].id, + set->pairs[i].id, "kfunc_sort_check"); + } } +} - btf__free(btf); - return 0; +/* True if @id is PTR -> TYPE_TAG(kflag=1, "address_space(1)") -> pointee */ +static bool is_arena_tagged_ptr(struct btf *btf, __u32 id) +{ + const struct btf_type *ptr, *tag; + const char *name; + + ptr = btf__type_by_id(btf, id); + if (!btf_is_ptr(ptr)) + return false; + tag = btf__type_by_id(btf, ptr->type); + if (!btf_is_type_tag(tag) || !btf_kflag(tag)) + return false; + name = btf__name_by_offset(btf, tag->name_off); + return strcmp(name, TYPE_ATTR_ARENA) == 0; } void test_resolve_btfids(void) { __u32 *test_list, *test_lists[] = { test_list_local, test_list_global }; unsigned int i, j; - int ret = 0; + struct btf *btf; - if (resolve_symbols()) + btf = btf__parse_raw(BTF_DATA_FILE); + if (!ASSERT_OK_PTR(btf, "btf_parse")) return; + if (resolve_symbols(btf)) + goto out; + /* Check BTF_ID_LIST(test_list_local) and * BTF_ID_LIST_GLOBAL(test_list_global) IDs */ for (j = 0; j < ARRAY_SIZE(test_lists); j++) { test_list = test_lists[j]; - for (i = 0; i < ARRAY_SIZE(test_symbols); i++) { - ret = CHECK(test_list[i] != test_symbols[i].id, - "id_check", - "wrong ID for %s (%d != %d)\n", - test_symbols[i].name, - test_list[i], test_symbols[i].id); - if (ret) - return; - } + for (i = 0; i < ARRAY_SIZE(test_symbols); i++) + ASSERT_EQ(test_list[i], test_symbols[i].id, test_symbols[i].name); } /* Check BTF_SET_START(test_set) IDs */ @@ -153,15 +288,60 @@ void test_resolve_btfids(void) break; } - ret = CHECK(!found, "id_check", - "ID %d not found in test_symbols\n", - test_set.ids[i]); - if (ret) + if (!ASSERT_TRUE(found, "id_in_test_symbols")) break; - if (i > 0) { - if (!ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check")) - return; + if (i > 0) + ASSERT_LE(test_set.ids[i - 1], test_set.ids[i], "sort_check"); + } + + check_kfunc_set(&test_kfunc_set); + check_kfunc_set(&test_kfunc_set_rev); + + /* Check resolve_btfids emitted a bpf_kfunc decl_tag for each kfunc */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + ASSERT_TRUE(btf_has_decl_tag(btf, DECL_TAG_KFUNC, + kfunc_symbols[i].id), + kfunc_symbols[i].name); + } + + /* Check resolve_btfids emitted bpf_fastcall for KF_FASTCALL kfuncs */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + if (kfunc_symbols[i].flags & KF_FASTCALL) { + ASSERT_TRUE(btf_has_decl_tag(btf, DECL_TAG_FASTCALL, + kfunc_symbols[i].id), + kfunc_symbols[i].name); } } + + /* + * Check resolve_btfids wrapped exactly the arena-flagged or suffixed + * return/args with the address_space(1) type attribute, and left other + * pointers/returns untouched. + */ + for (i = 0; i < ARRAY_SIZE(kfunc_symbols); i++) { + const struct btf_type *fn, *proto; + const struct btf_param *params; + const char *name = kfunc_symbols[i].name; + u32 arena_args = kfunc_symbols[i].arena_args; + __u32 nr; + + fn = btf__type_by_id(btf, kfunc_symbols[i].id); + if (!ASSERT_TRUE(btf_is_func(fn), name)) + continue; + proto = btf__type_by_id(btf, fn->type); + if (!ASSERT_TRUE(btf_is_func_proto(proto), name)) + continue; + params = btf_params(proto); + nr = btf_vlen(proto); + + ASSERT_EQ(is_arena_tagged_ptr(btf, proto->type), + kfunc_symbols[i].arena_ret, name); + for (j = 0; j < nr; j++) + ASSERT_EQ(is_arena_tagged_ptr(btf, params[j].type), + !!(arena_args & ARENA_ARG(j)), name); + } + +out: + btf__free(btf); } diff --git a/tools/testing/selftests/bpf/prog_tests/sha256.c b/tools/testing/selftests/bpf/prog_tests/sha256.c index 604a0b1423d5..5edbc6194b07 100644 --- a/tools/testing/selftests/bpf/prog_tests/sha256.c +++ b/tools/testing/selftests/bpf/prog_tests/sha256.c @@ -25,10 +25,10 @@ void test_sha256(void) size_t i; data = malloc(MAX_LEN); - if (!ASSERT_OK_PTR(data, "malloc")) + if (!ASSERT_NEQ(data, NULL, "malloc")) goto out; digests = malloc((MAX_LEN + 1) * SHA256_DIGEST_LENGTH); - if (!ASSERT_OK_PTR(digests, "malloc")) + if (!ASSERT_NEQ(digests, NULL, "malloc")) goto out; /* Generate MAX_LEN bytes of "random" data deterministically. */ diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 5fc417e31fc6..77381d345435 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -11,6 +11,8 @@ #include <linux/keyctl.h> #include <linux/bpf.h> +#include <bpf/btf.h> + #include "bpf/libbpf_internal.h" /* for libbpf_sha256() */ #include "bpf/skel_internal.h" /* for loader ctx layout (bpf_loader_ctx etc) */ @@ -19,8 +21,6 @@ #include "test_signed_loader_data.skel.h" #include "test_signed_loader_lsm.skel.h" -#define SIG_MATCH_INSNS 33 /* excl (5) + 4 * sha-dword (7) */ - enum { BPF_SIG_UNSIGNED = 0, BPF_SIG_VERIFIED, @@ -35,7 +35,8 @@ enum { }; static int load_loader(const void *insns, __u32 insns_sz, int map_fd, - const void *sig, __u32 sig_sz, __s32 keyring_id) + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt) { union bpf_attr attr; int fd; @@ -52,6 +53,7 @@ static int load_loader(const void *insns, __u32 insns_sz, int map_fd, attr.signature_size = sig_sz; attr.keyring_id = keyring_id; } + attr.fd_array_cnt = fd_array_cnt; memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, offsetofend(union bpf_attr, keyring_id)); @@ -62,14 +64,12 @@ static int run_gen_loader(const void *insns, __u32 insns_sz, const void *data, __u32 data_sz, const void *excl, __u32 excl_sz, const void *sig, __u32 sig_sz, - bool get_hash, void *ctx, __u32 ctx_sz, bool *loader_ran) + void *ctx, __u32 ctx_sz, bool *loader_ran) { LIBBPF_OPTS(bpf_map_create_opts, mopts, .excl_prog_hash = excl, .excl_prog_hash_size = excl_sz); - __u8 hbuf[SHA256_DIGEST_LENGTH]; - struct bpf_map_info info; - __u32 ilen = sizeof(info), key = 0; + __u32 key = 0; union bpf_attr attr; int map_fd, prog_fd, ret; @@ -87,15 +87,6 @@ static int run_gen_loader(const void *insns, __u32 insns_sz, ret = -errno; goto out_map; } - if (get_hash) { - memset(&info, 0, sizeof(info)); - info.hash = ptr_to_u64(hbuf); - info.hash_size = sizeof(hbuf); - if (bpf_map_get_info_by_fd(map_fd, &info, &ilen)) { - ret = -errno; - goto out_map; - } - } memset(&attr, 0, sizeof(attr)); attr.prog_type = BPF_PROG_TYPE_SYSCALL; @@ -108,6 +99,7 @@ static int run_gen_loader(const void *insns, __u32 insns_sz, attr.signature = ptr_to_u64(sig); attr.signature_size = sig_sz; attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.fd_array_cnt = 1; } memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, @@ -236,79 +228,6 @@ out: return ret; } -static void check_sig_match_shape(const struct bpf_insn *in, int n) -{ - int a = -1, cleanup = -1, i, base, t, br[5], nb = 0; - - /* BPF_PSEUDO_MAP_IDX (the struct bpf_map * form) is used only here. */ - for (i = 0; i + 1 < n; i++) { - if (in[i].code == (BPF_LD | BPF_IMM | BPF_DW) && - in[i].src_reg == BPF_PSEUDO_MAP_IDX) { - a = i; - break; - } - } - if (!ASSERT_GE(a, 0, "emit_signature_match present")) - return; - if (!ASSERT_LE(a + SIG_MATCH_INSNS, n, "block fits in program")) - return; - - /* excl check: r2 = *(u32 *)(map + 32); if r2 != 1 goto cleanup */ - ASSERT_EQ(in[a + 2].code, (BPF_LDX | BPF_MEM | BPF_W), "excl load width"); - ASSERT_EQ(in[a + 2].off, SHA256_DIGEST_LENGTH, "excl field offset"); - ASSERT_EQ(in[a + 4].code, (BPF_JMP | BPF_JNE | BPF_K), "excl branch op"); - ASSERT_EQ(in[a + 4].imm, 1, "excl compared to 1"); - br[nb++] = a + 4; - - /* 4 sha-dword checks: r2 = *(u64 *)(map + i*8); if r2 != r3 goto cleanup */ - for (i = 0; i < 4; i++) { - base = a + 5 + i * 7; - ASSERT_EQ(in[base + 2].code, (BPF_LDX | BPF_MEM | BPF_DW), "sha load width"); - ASSERT_EQ(in[base + 2].off, i * 8, "sha dword offset"); - ASSERT_EQ(in[base + 3].code, (BPF_LD | BPF_IMM | BPF_DW), "sha imm64 (H_meta)"); - ASSERT_EQ(in[base + 6].code, (BPF_JMP | BPF_JNE | BPF_X), "sha branch op"); - br[nb++] = base + 6; - } - - /* - * Locate the real cleanup label so we can pin the exact jump target, - * not just "some backward label". bpf_gen__init() emits the cleanup - * block as a prog-fd close loop whose first instruction is the label - * every error branch jumps to. - */ - for (i = 0; i + 2 < a; i++) { - if (in[i].code == (BPF_LDX | BPF_MEM | BPF_W) && - in[i].dst_reg == BPF_REG_1 && in[i].src_reg == BPF_REG_10 && - in[i + 1].code == (BPF_JMP | BPF_JSLE | BPF_K) && - in[i + 1].dst_reg == BPF_REG_1 && in[i + 1].imm == 0 && - in[i + 1].off == 1 && - in[i + 2].code == (BPF_JMP | BPF_CALL) && - in[i + 2].imm == BPF_FUNC_sys_close) { - cleanup = i; - break; - } - } - if (!ASSERT_GE(cleanup, 0, "cleanup label located")) - return; - for (i = 0; i < nb; i++) { - t = br[i] + 1 + in[br[i]].off; - ASSERT_EQ(t, cleanup, "sig-match lands on cleanup"); - } - /* - * Same invariant for every other cleanup-bound jump in the program: - * emit_check_err() is the only source of "if (r7 < 0) goto cleanup", - * so each of those must also resolve exactly to cleanup. - */ - for (i = 0, t = 0; i < n; i++) { - if (in[i].code != (BPF_JMP | BPF_JSLT | BPF_K) || - in[i].dst_reg != BPF_REG_7 || in[i].imm != 0 || in[i].off >= 0) - continue; - ASSERT_EQ(i + 1 + in[i].off, cleanup, "err-check lands on cleanup"); - t++; - } - ASSERT_GT(t, 0, "found emit_check_err jumps"); -} - struct gen_loader_fixture { struct test_signed_loader *skel; struct gen_loader_opts gopts; @@ -372,16 +291,6 @@ static void gen_loader_fixture_fini(struct gen_loader_fixture *f) test_signed_loader__destroy(f->skel); } -static void metadata_check_shape(void) -{ - struct gen_loader_fixture f; - - if (gen_loader_fixture_init(&f) == 0) - check_sig_match_shape((const struct bpf_insn *)f.gopts.insns, - f.gopts.insns_sz / sizeof(struct bpf_insn)); - gen_loader_fixture_fini(&f); -} - static void metadata_match(void) { struct gen_loader_fixture f; @@ -391,94 +300,265 @@ static void metadata_match(void) if (gen_loader_fixture_init(&f) == 0) { r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob, f.data_sz, f.excl, sizeof(f.excl), NULL, 0, - true, f.ctx, f.ctx_sz, &ran); + f.ctx, f.ctx_sz, &ran); ASSERT_TRUE(ran, "loader ran"); ASSERT_EQ(r, 0, "honest loader retval"); } gen_loader_fixture_fini(&f); } -static void metadata_sha_mismatch(void) +static void signature_enforced(void) { + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; struct gen_loader_fixture f; - bool ran; - int r; + int fd; if (gen_loader_fixture_init(&f) == 0) { /* - * blob[0] lives in the loader's fd_array scratch (first add_data in - * bpf_gen__init); a 0-map program never reads it, so flipping it - * changes only map->sha. The metadata check is the only thing that - * can notice -> isolates emit_signature_match. + * A present-but-invalid signature (the cert bytes are not a + * PKCS#7 signature) must be rejected at load: the signature + * path is honored, not ignored. (The valid path is covered by + * the signed lskels.) Pin -EBADMSG, the PKCS#7 parse failure: + * a looser fd < 0 check could also be satisfied by the sparse + * fd_array rejection (-EACCES) that the loader's map reference + * would trip even if the signature were silently ignored. */ - f.blob[0] ^= 0xff; - r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob, - f.data_sz, f.excl, sizeof(f.excl), NULL, 0, - true, f.ctx, f.ctx_sz, &ran); - ASSERT_TRUE(ran, "loader ran"); - ASSERT_EQ(r, -EINVAL, "tampered blob rejected by emit_signature_match"); + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(fd, -EBADMSG, "invalid signature rejected at load"); + if (fd >= 0) + close(fd); } gen_loader_fixture_fini(&f); } -static void metadata_not_exclusive(void) +static void signed_nonexcl_fd_array_rejected(void) { + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; struct gen_loader_fixture f; - bool ran; - int r; + int map_fd, fd; if (gen_loader_fixture_init(&f) == 0) { /* - * Correct blob but a non-exclusive metadata map: the verifier does - * not reject (excl_prog_sha unset), so the runtime map->excl == 1 - * check in the loader must. + * A signed program may only bind exclusive maps through fd_array + * (their contents are folded into the signature). Binding a + * non-exclusive map is rejected, before the signature is even + * examined. */ - r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob, - f.data_sz, NULL, 0, NULL, 0, true, f.ctx, - f.ctx_sz, &ran); - ASSERT_TRUE(ran, "loader ran"); - ASSERT_EQ(r, -EINVAL, "non-exclusive metadata map rejected"); + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "nonexcl", 4, + f.data_sz, 1, NULL); + if (ASSERT_OK_FD(map_fd, "nonexcl_map")) { + if (ASSERT_OK(bpf_map_freeze(map_fd), "freeze")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, + map_fd, junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EPERM, + "non-exclusive map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + } + close(map_fd); + } } gen_loader_fixture_fini(&f); } -static void metadata_hash_not_computed(void) +static void signed_unfrozen_fd_array_rejected(void) { + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + LIBBPF_OPTS(bpf_map_create_opts, mopts); struct gen_loader_fixture f; - bool ran; - int r; + __u32 key = 0; + int map_fd, fd; if (gen_loader_fixture_init(&f) == 0) { /* - * Correct, exclusive, frozen map, but its hash was never computed - * (no OBJ_GET_INFO_BY_FD), so map->sha stays zero. The loader must - * fail closed rather than treat an unset hash as a match. + * The metadata map must be frozen before a signed load so the + * folded bytes cannot change afterwards. Bind an exclusive map + * with matching contents but skip the freeze: the load must be + * rejected by the frozen check with -EPERM. The exclusivity + * check right after it would pass, so the errno uniquely pins + * the freeze requirement. */ - r = run_gen_loader(f.gopts.insns, f.gopts.insns_sz, f.blob, - f.data_sz, f.excl, sizeof(f.excl), NULL, 0, - false, f.ctx, f.ctx_sz, &ran); - ASSERT_TRUE(ran, "loader ran"); - ASSERT_EQ(r, -EINVAL, "uncomputed metadata hash rejected"); + mopts.excl_prog_hash = f.excl; + mopts.excl_prog_hash_size = sizeof(f.excl); + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "unfrozen", 4, + f.data_sz, 1, &mopts); + if (ASSERT_OK_FD(map_fd, "unfrozen_map")) { + if (ASSERT_OK(bpf_map_update_elem(map_fd, &key, f.blob, 0), + "update")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, + map_fd, junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EPERM, + "unfrozen map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + } + close(map_fd); + } } gen_loader_fixture_fini(&f); } -static void signature_enforced(void) +static void signed_nonarray_fd_array_rejected(void) +{ + static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + LIBBPF_OPTS(bpf_map_create_opts, mopts); + struct gen_loader_fixture f; + int map_fd, fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * Only a plain BPF_MAP_TYPE_ARRAY may be folded into the + * signature. An exclusive map of any other type is rejected + * (-EINVAL) rather than folded - this is the type gate that + * keeps arena maps (map_direct_value_addr() returns a user + * address) and insn-array maps (buffer smaller than value_size) + * out of the hashed region, where the old code would have + * memcpy()'d from them. A hash map stands in here: it is + * exclusive (bound to the loader digest) but not an array. + */ + mopts.excl_prog_hash = f.excl; + mopts.excl_prog_hash_size = sizeof(f.excl); + map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, "excl_hash", 4, 4, 1, + &mopts); + if (ASSERT_OK_FD(map_fd, "excl_hash_map")) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, + junk, sizeof(junk), + KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_EQ(fd, -EINVAL, + "non-array map in signed fd_array rejected"); + if (fd >= 0) + close(fd); + close(map_fd); + } + } + gen_loader_fixture_fini(&f); +} + +static int setup_meta_map(const struct gen_loader_fixture *f); + +static void signed_btf_fd_array_rejected(void) +{ + char dir_tmpl[] = "/tmp/signed_loader_btfXXXXXX", *dir = NULL; + __u32 sig_sz = 8192; + int map_fd = -1, prog_fd = -1; + unsigned char *buf = NULL; + struct gen_loader_fixture f; + bool have_fixture = false; + struct btf *btf = NULL; + union bpf_attr attr; + int fds[2]; + __u8 sig[8192]; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + /* + * fd_array binds maps and BTFs alike, but only exclusive array maps are + * folded into the signature. Build an otherwise genuinely signed load - + * insns || metadata, exclusive frozen map at fd_array[0] - then smuggle + * an extra BTF into fd_array[1]. A signed program may not bind any BTF, + * so resolving the fd_array entries rejects the BTF with -EACCES (in + * __add_used_btf(), before the signature is even verified). + */ + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, sig, + &sig_sz), "sign insns||metadata")) + goto out; + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + goto out; + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new_empty")) + goto out; + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto out; + + fds[0] = map_fd; + fds[1] = btf__fd(btf); + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(f.gopts.insns); + attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(fds); + attr.fd_array_cnt = 2; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + ASSERT_EQ(prog_fd < 0 ? -errno : prog_fd, -EACCES, + "BTF in signed fd_array rejected"); + if (prog_fd >= 0) + close(prog_fd); +out: + if (btf) + btf__free(btf); + if (map_fd >= 0) + close(map_fd); + if (have_fixture) + gen_loader_fixture_fini(&f); + if (dir) + run_setup("cleanup", dir); + free(buf); +} + +static void signature_failure_logs(void) { static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; + char log_buf[1024] = {}; struct gen_loader_fixture f; + union bpf_attr attr; int fd; if (gen_loader_fixture_init(&f) == 0) { /* - * A present-but-invalid signature (the cert bytes are not a - * PKCS#7 signature) must be rejected at load: the signature - * path is honored, not ignored. (The valid path is covered by - * the signed lskels.) + * Signature verification now runs inside bpf_check(), so a + * failure is reported through the verifier log. A present-but- + * invalid signature is rejected and the log says why. */ - fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, - sizeof(junk), KEY_SPEC_SESSION_KEYRING); + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(f.gopts.insns); + attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(junk); + attr.signature_size = sizeof(junk); + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = sizeof(log_buf); + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + + fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); ASSERT_LT(fd, 0, "invalid signature rejected at load"); + if (fd >= 0) + close(fd); + ASSERT_HAS_SUBSTR(log_buf, "signature verification failed", + "verifier logs signature failure"); } gen_loader_fixture_fini(&f); } @@ -495,8 +575,31 @@ static void signature_too_large(void) * is rejected before the buffer is read. */ fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, - 64 << 20, KEY_SPEC_SESSION_KEYRING); + 64 << 20, KEY_SPEC_SESSION_KEYRING, 0); ASSERT_EQ(fd, -EINVAL, "oversized signature rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + +static void signature_zero_size(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int fd; + + if (gen_loader_fixture_init(&f) == 0) { + /* + * A present signature with signature_size == 0 is rejected + * up front, before the keyring is resolved or the signature + * buffer is read. + */ + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + 0, KEY_SPEC_SESSION_KEYRING, 0); + ASSERT_EQ(fd, -EINVAL, "zero-size signature rejected"); + if (fd >= 0) + close(fd); } gen_loader_fixture_fini(&f); } @@ -515,8 +618,10 @@ static void signature_bad_keyring(void) * large positive serial takes the user-keyring path and won't exist. */ fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, - sizeof(junk), INT_MAX); + sizeof(junk), INT_MAX, 0); ASSERT_EQ(fd, -EINVAL, "signature with bad keyring_id rejected"); + if (fd >= 0) + close(fd); } gen_loader_fixture_fini(&f); } @@ -575,7 +680,7 @@ static void metadata_ctx_max_entries_ignored(void) memcpy(blob, gopts.data, data_sz); r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz, - excl, sizeof(excl), NULL, 0, true, ctx, ctx_sz, &ran); + excl, sizeof(excl), NULL, 0, ctx, ctx_sz, &ran); if (!ASSERT_TRUE(ran, "loader ran") || !ASSERT_EQ(r, 0, "loader retval")) goto free_blob; @@ -661,7 +766,7 @@ static void metadata_ctx_initial_value_ignored(void) memcpy(blob, gopts.data, data_sz); r = run_gen_loader(gopts.insns, gopts.insns_sz, blob, data_sz, - excl, sizeof(excl), NULL, 0, true, ctx, ctx_sz, &ran); + excl, sizeof(excl), NULL, 0, ctx, ctx_sz, &ran); if (!ASSERT_TRUE(ran, "loader ran") || !ASSERT_EQ(r, 0, "loader retval")) goto free_blob; @@ -714,6 +819,7 @@ static void signature_authenticates_insns(void) __u8 excl[SHA256_DIGEST_LENGTH], sig[8192]; __u32 sig_sz = sizeof(sig), insns_sz, data_sz, ctx_sz; unsigned char *insns = NULL, *tampered = NULL, *blob = NULL; + unsigned char *signbuf = NULL; int nr_maps = 0, nr_progs = 0, r; struct bpf_program *p; struct bpf_map *m; @@ -760,29 +866,141 @@ static void signature_authenticates_insns(void) memcpy(blob, gopts.data, data_sz); libbpf_sha256(insns, insns_sz, excl); - if (!ASSERT_OK(sign_buf(dir, insns, insns_sz, sig, &sig_sz), "sign-file")) + signbuf = malloc((size_t)insns_sz + data_sz); + if (!ASSERT_OK_PTR(signbuf, "signbuf")) + goto cleanup; + memcpy(signbuf, insns, insns_sz); + memcpy(signbuf + insns_sz, blob, data_sz); + if (!ASSERT_OK(sign_buf(dir, signbuf, insns_sz + data_sz, sig, &sig_sz), + "sign-file")) goto cleanup; memset(ctx, 0, ctx_sz); ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), - sig, sig_sz, true, ctx, ctx_sz, &ran); + sig, sig_sz, ctx, ctx_sz, &ran); ASSERT_TRUE(ran, "valid signature: loader loaded and ran"); ASSERT_EQ(r, 0, "valid signature accepted"); close_loader_ctx_fds(ctx, nr_maps, nr_progs); memcpy(tampered, insns, insns_sz); tampered[insns_sz / 2] ^= 0xff; + /* + * Bind the metadata map to the tampered loader's own digest, so the + * verifier's exclusive-map check (excl_prog_sha == prog->digest) passes + * and the signature - verified after the maps are resolved - is what + * rejects the load. This is the attacker's best case: even after + * re-binding the exclusive map to their tampered loader, the signature + * over the original insns || metadata still fails. (Leaving the map + * bound to the original digest would instead trip the excl check first.) + */ + libbpf_sha256(tampered, insns_sz, excl); memset(ctx, 0, ctx_sz); ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; r = run_gen_loader(tampered, insns_sz, blob, data_sz, excl, sizeof(excl), - sig, sig_sz, true, ctx, ctx_sz, &ran); + sig, sig_sz, ctx, ctx_sz, &ran); ASSERT_FALSE(ran, "tampered loader rejected before run"); ASSERT_EQ(r, -EKEYREJECTED, "signature is bound to the instructions"); cleanup: free(insns); free(tampered); free(blob); + free(signbuf); + free(ctx); + test_signed_loader__destroy(skel); + run_setup("cleanup", dir); +} + +static void signature_authenticates_metadata(void) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + struct test_signed_loader *skel = NULL; + __u8 excl[SHA256_DIGEST_LENGTH], sig[8192]; + __u32 sig_sz = sizeof(sig), insns_sz, data_sz, ctx_sz; + unsigned char *insns = NULL, *blob = NULL; + unsigned char *signbuf = NULL; + int nr_maps = 0, nr_progs = 0, r; + struct bpf_program *p; + struct bpf_map *m; + void *ctx = NULL; + bool ran; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + skel = test_signed_loader__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + goto cleanup; + if (!ASSERT_OK(bpf_object__gen_loader(skel->obj, &gopts), "gen_loader")) + goto cleanup; + if (!ASSERT_OK(bpf_object__load(skel->obj), "gen_load")) + goto cleanup; + + bpf_object__for_each_program(p, skel->obj) + nr_progs++; + bpf_object__for_each_map(m, skel->obj) + nr_maps++; + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + insns_sz = gopts.insns_sz; + data_sz = gopts.data_sz; + ctx = calloc(1, ctx_sz); + insns = malloc(insns_sz); + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(ctx, "ctx") || + !ASSERT_OK_PTR(insns, "insns") || + !ASSERT_OK_PTR(blob, "blob")) + goto cleanup; + memcpy(insns, gopts.insns, insns_sz); + memcpy(blob, gopts.data, data_sz); + libbpf_sha256(insns, insns_sz, excl); + + signbuf = malloc((size_t)insns_sz + data_sz); + if (!ASSERT_OK_PTR(signbuf, "signbuf")) + goto cleanup; + memcpy(signbuf, insns, insns_sz); + memcpy(signbuf + insns_sz, blob, data_sz); + if (!ASSERT_OK(sign_buf(dir, signbuf, insns_sz + data_sz, sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_TRUE(ran, "valid signature: loader loaded and ran"); + ASSERT_EQ(r, 0, "valid signature accepted"); + close_loader_ctx_fds(ctx, nr_maps, nr_progs); + + /* + * Tamper the metadata after signing while leaving the instructions + * and thus the exclusive hash binding untouched: the map freezes + * fine and excl_prog_sha still matches the loader's digest, so the + * load reaches signature verification, which folds the live frozen + * map bytes into the checked payload and must reject the modified + * blob. A kernel folding anything but the map contents themselves + * would wrongly accept this load. + */ + blob[data_sz / 2] ^= 0xff; + memset(ctx, 0, ctx_sz); + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + r = run_gen_loader(insns, insns_sz, blob, data_sz, excl, sizeof(excl), + sig, sig_sz, ctx, ctx_sz, &ran); + ASSERT_FALSE(ran, "tampered metadata rejected before run"); + ASSERT_EQ(r, -EKEYREJECTED, "signature is bound to the metadata"); +cleanup: + free(insns); + free(blob); + free(signbuf); free(ctx); test_signed_loader__destroy(skel); run_setup("cleanup", dir); @@ -1007,10 +1225,11 @@ static void lsm_signature_verdict(void) { char dir_tmpl[] = "/tmp/signed_loader_lsmXXXXXX", *dir = NULL; struct test_signed_loader_lsm *lsm = NULL; + __u32 sig_sz = 8192, msig_sz = 8192; int map_fd = -1, prog_fd = -1; bool have_fixture = false; struct gen_loader_fixture f; - __u32 sig_sz = 8192; + unsigned char *buf; __s32 ses_serial; __u8 sig[8192]; @@ -1029,7 +1248,7 @@ static void lsm_signature_verdict(void) if (!ASSERT_OK_FD(map_fd, "meta_map_unsigned")) goto out; lsm->bss->seen = 0; - prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, NULL, 0, 0); + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, NULL, 0, 0, 0); close(map_fd); map_fd = -1; if (!ASSERT_OK_FD(prog_fd, "unsigned loader load")) @@ -1062,22 +1281,51 @@ static void lsm_signature_verdict(void) goto out; lsm->bss->seen = 0; prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, - sig_sz, KEY_SPEC_SESSION_KEYRING); + sig_sz, KEY_SPEC_SESSION_KEYRING, 0); close(map_fd); map_fd = -1; - if (!ASSERT_OK_FD(prog_fd, "signed loader load")) - goto out; - close(prog_fd); + ASSERT_EQ(prog_fd, -EACCES, "unfolded metadata rejected"); + if (prog_fd >= 0) + close(prog_fd); prog_fd = -1; ses_serial = syscall(__NR_keyctl, KEYCTL_GET_KEYRING_ID, KEY_SPEC_SESSION_KEYRING, 0); ASSERT_EQ(lsm->bss->seen, 1, "signed: one observed load"); - ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, "signed verdict"); + ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, + "admission saw a valid signature"); ASSERT_EQ(lsm->bss->sig_keyring_type, BPF_SIG_KEYRING_USER, "signed keyring type"); ASSERT_GT(ses_serial, 0, "session keyring serial resolved"); ASSERT_EQ(lsm->bss->sig_keyring_serial, ses_serial, "signed: validated against session keyring"); + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "meta_signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, + sig, &msig_sz), "sign insns||metadata")) { + free(buf); + goto out; + } + free(buf); + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map_bound")) + goto out; + lsm->bss->seen = 0; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, + msig_sz, KEY_SPEC_SESSION_KEYRING, 1); + close(map_fd); + map_fd = -1; + if (!ASSERT_OK_FD(prog_fd, "metadata-bound loader load")) + goto out; + close(prog_fd); + prog_fd = -1; + ASSERT_EQ(lsm->bss->seen, 1, "metadata: one observed load"); + ASSERT_EQ(lsm->bss->sig_verdict, BPF_SIG_VERIFIED, + "metadata-bound verdict"); out: if (map_fd >= 0) close(map_fd); @@ -1090,22 +1338,471 @@ out: test_signed_loader_lsm__destroy(lsm); } +/* + * Load-time metadata verification: the kernel folds the frozen metadata map + * into the signature (insns || metadata) and checks it at BPF_PROG_LOAD via + * fd_array_cnt, rather than the loader checking from within BPF. Sign that + * concatenation, hand the kernel the map, and confirm the signed loader loads, + * runs, and installs its target. + */ +static int loadtime_drive(const char *dir, const void *insns, __u32 insns_sz, + const void *data, __u32 data_sz, const __u8 *excl, + void *ctx, __u32 ctx_sz, int *load_ret, bool *ran) +{ + LIBBPF_OPTS(bpf_map_create_opts, mopts, + .excl_prog_hash = excl, + .excl_prog_hash_size = SHA256_DIGEST_LENGTH); + __u32 sig_sz = 8192, key = 0; + unsigned char *buf = NULL; + int map_fd, prog_fd, ret = 0; + union bpf_attr attr; + __u8 sig[8192]; + + *ran = false; + *load_ret = 0; + + /* + * Metadata map, bound to the loader digest and frozen, exactly as + * skel_internal.h's bpf_load_and_run() sets it up. + */ + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, + data_sz, 1, &mopts); + if (map_fd < 0) { + ret = -errno; + goto out_load; + } + if (bpf_map_update_elem(map_fd, &key, data, 0) || bpf_map_freeze(map_fd)) { + ret = -errno; + goto out_load; + } + + /* Sign insns || metadata, the same bytes the kernel reconstructs. */ + buf = malloc((size_t)insns_sz + data_sz); + if (!buf) { + ret = -ENOMEM; + goto out_load; + } + memcpy(buf, insns, insns_sz); + memcpy(buf + insns_sz, data, data_sz); + ret = sign_buf(dir, buf, insns_sz + data_sz, sig, &sig_sz); + if (ret) + goto out_load; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = insns_sz / sizeof(struct bpf_insn); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + attr.fd_array_cnt = 1; + memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + if (prog_fd < 0) { + ret = -errno; + goto out_load; + } + + memset(&attr, 0, sizeof(attr)); + attr.test.prog_fd = prog_fd; + attr.test.ctx_in = ptr_to_u64(ctx); + attr.test.ctx_size_in = ctx_sz; + if (syscall(__NR_bpf, BPF_PROG_RUN, &attr, + offsetofend(union bpf_attr, test)) < 0) { + ret = -errno; + goto out_prog; + } + *ran = true; + ret = (int)attr.test.retval; +out_prog: + close(prog_fd); + goto out_map; +out_load: + *load_ret = ret; +out_map: + free(buf); + if (map_fd >= 0) + close(map_fd); + return ret; +} + +static void loadtime_verify(struct bpf_object *obj, int expect_maps) +{ + LIBBPF_OPTS(gen_loader_opts, gopts, .gen_hash = true); + char dir_tmpl[] = "/tmp/signed_loader_ltXXXXXX", *dir = NULL; + int nr_maps = 0, nr_progs = 0, load_ret = 0, r; + __u8 excl[SHA256_DIGEST_LENGTH]; + struct bpf_prog_desc *pd; + struct bpf_map_desc *md; + unsigned char *blob = NULL; + struct bpf_program *p; + struct bpf_map *m; + __u32 ctx_sz, data_sz; + void *ctx = NULL; + bool ran = false; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + if (!ASSERT_OK(bpf_object__gen_loader(obj, &gopts), "gen_loader")) + goto out; + if (!ASSERT_OK(bpf_object__load(obj), "gen_load")) + goto out; + + bpf_object__for_each_program(p, obj) + nr_progs++; + bpf_object__for_each_map(m, obj) + nr_maps++; + if (!ASSERT_EQ(nr_maps, expect_maps, "fixture map count")) + goto out; + + ctx_sz = sizeof(struct bpf_loader_ctx) + + nr_maps * sizeof(struct bpf_map_desc) + + nr_progs * sizeof(struct bpf_prog_desc); + ctx = calloc(1, ctx_sz); + if (!ASSERT_OK_PTR(ctx, "ctx_alloc")) + goto out; + ((struct bpf_loader_ctx *)ctx)->sz = ctx_sz; + + data_sz = gopts.data_sz; + blob = malloc(data_sz); + if (!ASSERT_OK_PTR(blob, "blob_alloc")) + goto out; + memcpy(blob, gopts.data, data_sz); + + /* excl_prog_hash = SHA256(loader insns) == the loader's prog->digest. */ + libbpf_sha256(gopts.insns, gopts.insns_sz, excl); + + r = loadtime_drive(dir, gopts.insns, gopts.insns_sz, blob, data_sz, + excl, ctx, ctx_sz, &load_ret, &ran); + ASSERT_OK(load_ret, "signed loader loaded (insns || metadata)"); + ASSERT_TRUE(ran, "loader ran"); + ASSERT_EQ(r, 0, "loader installed its target"); + + md = (struct bpf_map_desc *)((char *)ctx + sizeof(struct bpf_loader_ctx)); + pd = (struct bpf_prog_desc *)(md + nr_maps); + ASSERT_GT(pd[0].prog_fd, 0, "target program installed"); + if (nr_maps) + ASSERT_GT(md[0].map_fd, 0, "target map installed"); + + close_loader_ctx_fds(ctx, nr_maps, nr_progs); +out: + free(blob); + free(ctx); + if (dir) + run_setup("cleanup", dir); +} + +static void loadtime_no_map(void) +{ + struct test_signed_loader *skel = test_signed_loader__open(); + + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + loadtime_verify(skel->obj, 0); + test_signed_loader__destroy(skel); +} + +static void loadtime_with_map(void) +{ + struct test_signed_loader_map *skel = test_signed_loader_map__open(); + + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + loadtime_verify(skel->obj, 1); + test_signed_loader_map__destroy(skel); +} + +/* + * A signed program need not bind any map. A plain BPF_PROG_TYPE_SYSCALL + * program with no fd_array is signed over its instructions alone: the kernel + * verifies the signature, folds no metadata, and the program loads. Exercise + * the fd_array == NULL / fd_array_cnt == 0 path, and confirm the signature + * still authenticates the instructions (a tampered copy is rejected). + */ +static void signed_no_fd_array(void) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int prog_fd, err; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + + /* No metadata map: the signed payload is the instructions alone. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + /* fd_array and fd_array_cnt deliberately left NULL/0. */ + memcpy(attr.prog_name, "signed_nomap", sizeof("signed_nomap")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + if (!ASSERT_GE(prog_fd, 0, "map-less signed program loaded")) { + if (prog_fd >= 0) + close(prog_fd); + goto cleanup; + } + close(prog_fd); + + /* The signature covers the instructions, so tampering must be rejected. */ + insns[0].imm = 1; + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EKEYREJECTED, "tampered map-less program rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +} + +/* + * A signed program may reach maps only through fd_array indices, so the kernel + * folds (and thus attests) them. A direct BPF_PSEUDO_MAP_FD reference - a raw, + * unfolded fd baked into the signed instructions - is rejected by the verifier. + */ +static void signed_map_by_fd_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_LD_MAP_FD(BPF_REG_1, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loaderXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int map_fd, prog_fd, err; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "sig_mapfd", 4, 4, 1, NULL); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + insns[0].imm = map_fd; /* bake the raw map fd into the ld_imm64 */ + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + goto out_map; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + goto out_map; + } + + /* Sign the instructions, raw map fd and all. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + /* No fd_array: the map is reached by a raw fd in the instructions. */ + memcpy(attr.prog_name, "signed_mapfd", sizeof("signed_mapfd")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EINVAL, "signed program referencing a map by fd rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +out_map: + close(map_fd); +} + +/* + * A signed program may reach maps only through the continuous fd_array, so the + * kernel folds (and thus attests) them. Referencing a map by fd_array *index* + * while leaving fd_array_cnt at 0 selects the sparse path, which resolves a map + * the signature never covered; the verifier rejects it up front with -EACCES. + */ +static void signed_sparse_fd_array_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_IDX, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loader_spXXXXXX", *dir; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + int map_fd, prog_fd, err; + + map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "sig_sparse", 4, 4, 1, NULL); + if (!ASSERT_GE(map_fd, 0, "map_create")) + return; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + goto out_map; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + goto out_map; + } + + /* Sign the instructions alone; the sparse map is not folded. */ + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(&map_fd); + attr.fd_array_cnt = 0; /* sparse: force lazy map resolution */ + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "signed_sparse", sizeof("signed_sparse")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + ASSERT_EQ(err, -EACCES, "signed program binding a sparse fd_array map rejected"); + if (prog_fd >= 0) + close(prog_fd); +cleanup: + run_setup("cleanup", dir); +out_map: + close(map_fd); +} + +static void signed_module_kfunc_rejected(void) +{ + struct bpf_insn insns[] = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 1, 1), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + char dir_tmpl[] = "/tmp/signed_loader_kfnXXXXXX", *dir; + int prog_fd, err, fds[2]; + struct btf *btf = NULL; + __u32 sig_sz = 8192; + union bpf_attr attr; + __u8 sig[8192]; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + rmdir(dir); + return; + } + if (!ASSERT_OK(sign_buf(dir, insns, sizeof(insns), sig, &sig_sz), + "sign-file")) + goto cleanup; + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf_new_empty")) + goto cleanup; + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf_load")) + goto cleanup; + fds[0] = -1; + fds[1] = btf__fd(btf); + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SYSCALL; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = ARRAY_SIZE(insns); + attr.license = ptr_to_u64("Dual BSD/GPL"); + attr.prog_flags = BPF_F_SLEEPABLE; + attr.fd_array = ptr_to_u64(fds); + attr.fd_array_cnt = 0; /* sparse: force lazy kfunc BTF resolution */ + attr.signature = ptr_to_u64(sig); + attr.signature_size = sig_sz; + attr.keyring_id = KEY_SPEC_SESSION_KEYRING; + memcpy(attr.prog_name, "signed_kfunc", sizeof("signed_kfunc")); + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, + offsetofend(union bpf_attr, keyring_id)); + err = prog_fd < 0 ? -errno : prog_fd; + if (prog_fd >= 0) + close(prog_fd); + + ASSERT_EQ(err, -EACCES, "module kfunc BTF in signed program rejected"); +cleanup: + if (btf) + btf__free(btf); + run_setup("cleanup", dir); +} + void test_signed_loader(void) { - if (test__start_subtest("metadata_check_shape")) - metadata_check_shape(); + if (test__start_subtest("loadtime_no_map")) + loadtime_no_map(); + if (test__start_subtest("loadtime_with_map")) + loadtime_with_map(); if (test__start_subtest("metadata_match")) metadata_match(); - if (test__start_subtest("metadata_sha_mismatch")) - metadata_sha_mismatch(); - if (test__start_subtest("metadata_not_exclusive")) - metadata_not_exclusive(); - if (test__start_subtest("metadata_hash_not_computed")) - metadata_hash_not_computed(); if (test__start_subtest("signature_enforced")) signature_enforced(); + if (test__start_subtest("signed_nonexcl_fd_array_rejected")) + signed_nonexcl_fd_array_rejected(); + if (test__start_subtest("signed_unfrozen_fd_array_rejected")) + signed_unfrozen_fd_array_rejected(); + if (test__start_subtest("signed_nonarray_fd_array_rejected")) + signed_nonarray_fd_array_rejected(); + if (test__start_subtest("signed_btf_fd_array_rejected")) + signed_btf_fd_array_rejected(); + if (test__start_subtest("signed_module_kfunc_rejected")) + signed_module_kfunc_rejected(); + if (test__start_subtest("signature_failure_logs")) + signature_failure_logs(); if (test__start_subtest("signature_too_large")) signature_too_large(); + if (test__start_subtest("signature_zero_size")) + signature_zero_size(); if (test__start_subtest("signature_bad_keyring")) signature_bad_keyring(); if (test__start_subtest("metadata_ctx_max_entries_ignored")) @@ -1114,6 +1811,8 @@ void test_signed_loader(void) metadata_ctx_initial_value_ignored(); if (test__start_subtest("signature_authenticates_insns")) signature_authenticates_insns(); + if (test__start_subtest("signature_authenticates_metadata")) + signature_authenticates_metadata(); if (test__start_subtest("hash_requires_frozen")) hash_requires_frozen(); if (test__start_subtest("no_update_after_freeze")) @@ -1132,4 +1831,10 @@ void test_signed_loader(void) map_hash_unsupported_type(); if (test__start_subtest("lsm_signature_verdict")) lsm_signature_verdict(); + if (test__start_subtest("signed_no_fd_array")) + signed_no_fd_array(); + if (test__start_subtest("signed_map_by_fd_rejected")) + signed_map_by_fd_rejected(); + if (test__start_subtest("signed_sparse_fd_array_rejected")) + signed_sparse_fd_array_rejected(); } diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index e5fc038d747b..1fef6ec2ba7a 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -7,7 +7,6 @@ #include "test_progs.h" #include "test_skmsg_load_helpers.skel.h" -#include "test_sockmap_update.skel.h" #include "test_sockmap_invalid_update.skel.h" #include "test_sockmap_skb_verdict_attach.skel.h" #include "test_sockmap_progs_query.skel.h" @@ -235,53 +234,6 @@ out: test_skmsg_load_helpers__destroy(skel); } -static void test_sockmap_update(enum bpf_map_type map_type) -{ - int err, prog, src; - struct test_sockmap_update *skel; - struct bpf_map *dst_map; - const __u32 zero = 0; - char dummy[14] = {0}; - LIBBPF_OPTS(bpf_test_run_opts, topts, - .data_in = dummy, - .data_size_in = sizeof(dummy), - .repeat = 1, - ); - __s64 sk; - - sk = connected_socket_v4(); - if (!ASSERT_NEQ(sk, -1, "connected_socket_v4")) - return; - - skel = test_sockmap_update__open_and_load(); - if (!ASSERT_OK_PTR(skel, "open_and_load")) - goto close_sk; - - prog = bpf_program__fd(skel->progs.copy_sock_map); - src = bpf_map__fd(skel->maps.src); - if (map_type == BPF_MAP_TYPE_SOCKMAP) - dst_map = skel->maps.dst_sock_map; - else - dst_map = skel->maps.dst_sock_hash; - - err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST); - if (!ASSERT_OK(err, "update_elem(src)")) - goto out; - - err = bpf_prog_test_run_opts(prog, &topts); - if (!ASSERT_OK(err, "test_run")) - goto out; - if (!ASSERT_NEQ(topts.retval, 0, "test_run retval")) - goto out; - - compare_cookies(skel->maps.src, dst_map); - -out: - test_sockmap_update__destroy(skel); -close_sk: - close(sk); -} - static void test_sockmap_invalid_update(void) { struct test_sockmap_invalid_update *skel; @@ -1422,10 +1374,6 @@ void test_sockmap_basic(void) test_skmsg_helpers(BPF_MAP_TYPE_SOCKMAP); if (test__start_subtest("sockhash sk_msg load helpers")) test_skmsg_helpers(BPF_MAP_TYPE_SOCKHASH); - if (test__start_subtest("sockmap update")) - test_sockmap_update(BPF_MAP_TYPE_SOCKMAP); - if (test__start_subtest("sockhash update")) - test_sockmap_update(BPF_MAP_TYPE_SOCKHASH); if (test__start_subtest("sockmap update in unsafe context")) test_sockmap_invalid_update(); if (test__start_subtest("sockmap copy")) diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt.c b/tools/testing/selftests/bpf/prog_tests/sockopt.c index eaac83a7f388..6c96f2d9fccf 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/tcp.h> #include <test_progs.h> #include <io_uring/mini_liburing.h> #include "cgroup_helpers.h" @@ -284,6 +285,27 @@ static struct sockopt_test { .io_uring_support = true, }, { + .descr = "getsockopt: deny negative ctx->optlen in TCP_ZEROCOPY_RECEIVE", + .insns = { + /* ctx->optlen = -1 */ + BPF_MOV64_IMM(BPF_REG_0, -1), + BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, + offsetof(struct bpf_sockopt, optlen)), + + /* return 1 */ + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_EXIT_INSN(), + }, + .attach_type = BPF_CGROUP_GETSOCKOPT, + .expected_attach_type = BPF_CGROUP_GETSOCKOPT, + + .get_level = IPPROTO_TCP, + .get_optname = TCP_ZEROCOPY_RECEIVE, + .get_optlen = sizeof(struct tcp_zerocopy_receive), + + .error = EFAULT_GETSOCKOPT, + }, + { .descr = "getsockopt: ignore >PAGE_SIZE optlen", .insns = { /* write 0xFF to the first optval byte */ diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index c3cce5c292bd..e4e9374309e2 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c @@ -103,6 +103,12 @@ void test_stream_arena_fault_address(void) test_address(skel->progs.stream_arena_read_fault, &skel->bss->fault_addr); if (test__start_subtest("write_fault")) test_address(skel->progs.stream_arena_write_fault, &skel->bss->fault_addr); + if (test__start_subtest("load_acquire_fault")) + test_address(skel->progs.stream_arena_load_acquire_fault, &skel->bss->fault_addr); + if (test__start_subtest("xchg_fault")) + test_address(skel->progs.stream_arena_xchg_fault, &skel->bss->fault_addr); + if (test__start_subtest("cmpxchg_fault")) + test_address(skel->progs.stream_arena_cmpxchg_fault, &skel->bss->fault_addr); stream__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index a5a226d0104c..c5c9d6c359bb 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -12,6 +12,9 @@ #include "tailcall_cgrp_storage_no_storage.skel.h" #include "tailcall_cgrp_storage.skel.h" #include "tailcall_sleepable.skel.h" +#include "tailcall_callback.skel.h" +#include "tailcall_bpf2bpf2.skel.h" +#include "tailcall_bpf2bpf_fexit.skel.h" /* test_tailcall_1 checks basic functionality by patching multiple locations * in a single program for a single tail call slot with nop->jmp, jmp->nop @@ -1901,6 +1904,55 @@ out: tailcall_sleepable__destroy(skel); } +static void test_tailcall_callback(void) +{ + RUN_TESTS(tailcall_callback); +} + +static void test_tailcall_bpf2bpf_fexit_links(void) +{ + struct tailcall_bpf2bpf_fexit *skel1 = NULL, *skel2 = NULL; + struct tailcall_bpf2bpf2 *skel_tc; + int err, prog_fd; + + skel_tc = tailcall_bpf2bpf2__open_and_load(); + if (!ASSERT_OK_PTR(skel_tc, "tailcall_bpf2bpf2__open_and_load")) + return; + + skel1 = tailcall_bpf2bpf_fexit__open(); + if (!ASSERT_OK_PTR(skel1, "tailcall_bpf2bpf_fexit__open")) + goto out; + + prog_fd = bpf_program__fd(skel_tc->progs.classifier_0); + err = bpf_program__set_attach_target(skel1->progs.fexit, prog_fd, "subprog_tail"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = tailcall_bpf2bpf_fexit__load(skel1); + if (!ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load")) + goto out; + + skel1->links.fexit = bpf_program__attach_trace(skel1->progs.fexit); + if (!ASSERT_OK_PTR(skel1->links.fexit, "bpf_program__attach_trace")) + goto out; + + skel2 = tailcall_bpf2bpf_fexit__open(); + if (!ASSERT_OK_PTR(skel2, "tailcall_bpf2bpf_fexit__open")) + goto out; + + err = bpf_program__set_attach_target(skel2->progs.fexit, prog_fd, "subprog_tail"); + if (!ASSERT_OK(err, "bpf_program__set_attach_target")) + goto out; + + err = tailcall_bpf2bpf_fexit__load(skel2); + ASSERT_OK(err, "tailcall_bpf2bpf_fexit__load"); + +out: + tailcall_bpf2bpf_fexit__destroy(skel1); + tailcall_bpf2bpf_fexit__destroy(skel2); + tailcall_bpf2bpf2__destroy(skel_tc); +} + void test_tailcalls(void) { if (test__start_subtest("tailcall_1")) @@ -1967,4 +2019,7 @@ void test_tailcalls(void) test_tailcall_cgrp_storage_no_storage_leaf(); if (test__start_subtest("tailcall_cgrp_storage_no_storage_bridge")) test_tailcall_cgrp_storage_no_storage_bridge(); + test_tailcall_callback(); + if (test__start_subtest("tailcall_bpf2bpf_fexit_links")) + test_tailcall_bpf2bpf_fexit_links(); } diff --git a/tools/testing/selftests/bpf/prog_tests/task_kfunc.c b/tools/testing/selftests/bpf/prog_tests/task_kfunc.c index e6e95c1416e6..30d403028f98 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_kfunc.c +++ b/tools/testing/selftests/bpf/prog_tests/task_kfunc.c @@ -176,6 +176,14 @@ static const char * const success_tests[] = { "test_task_from_pid_current", "test_task_from_pid_invalid", "task_kfunc_acquire_trusted_walked", + "task_kfunc_acquire_after_spin_unlock_non_sleepable", + "task_kfunc_acquire_after_spin_unlock_explicit_rcu", + "task_kfunc_acquire_after_spin_unlock_preempt_disabled", + "task_kfunc_acquire_after_spin_unlock_irq_disabled", + "task_kfunc_acquire_after_rcu_unlock_preempt_disabled", + "task_kfunc_acquire_after_rcu_unlock_irq_disabled", + "task_kfunc_acquire_after_preempt_enable_explicit_rcu", + "task_kfunc_acquire_after_irq_restore_explicit_rcu", "test_task_kfunc_flavor_relo", "test_task_kfunc_flavor_relo_not_found", }; diff --git a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c index 64fbda082309..af8968b89ad7 100644 --- a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c +++ b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c @@ -192,6 +192,8 @@ static int create_netkit(int mode, char *prim, char *peer) req.n.nlmsg_len += sizeof(struct ifinfomsg); addattr_l(&req.n, sizeof(req), IFLA_IFNAME, peer, strlen(peer)); addattr_nest_end(&req.n, peer_info); + addattr32(&req.n, sizeof(req), IFLA_NETKIT_SCRUB, + NETKIT_SCRUB_NONE); addattr_nest_end(&req.n, data); addattr_nest_end(&req.n, linkinfo); @@ -405,6 +407,24 @@ fail: return -1; } +static struct bpf_link *netns_attach_nk(const char *ns, int ifindex, + struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_netkit_opts, optl); + struct nstoken *nstoken = NULL; + struct bpf_link *link = NULL; + + nstoken = open_netns(ns); + if (!ASSERT_OK_PTR(nstoken, "setns")) + goto cleanup; + + link = bpf_program__attach_netkit(prog, ifindex, &optl); +cleanup: + if (nstoken) + close_netns(nstoken); + return link; +} + static void test_tcp(int family, const char *addr, __u16 port) { int listen_fd = -1, accept_fd = -1, client_fd = -1; @@ -1082,6 +1102,53 @@ done: close_netns(nstoken); } +static void test_tc_redirect_peer_ing(struct netns_setup_result *setup_result) +{ + struct test_tc_peer *skel; + struct nstoken *nstoken; + int err; + + nstoken = open_netns(NS_FWD); + if (!ASSERT_OK_PTR(nstoken, "setns fwd")) + return; + + skel = test_tc_peer__open(); + if (!ASSERT_OK_PTR(skel, "test_tc_peer__open")) + goto done; + + skel->rodata->IFINDEX_SRC = setup_result->ifindex_src_fwd; + skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd; + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_src_ing, + BPF_NETKIT_PRIMARY), 0, "src_prog_attach_type"); + ASSERT_EQ(bpf_program__set_expected_attach_type(skel->progs.tc_dst_ing, + BPF_NETKIT_PRIMARY), 0, "dst_prog_attach_type"); + + err = test_tc_peer__load(skel); + if (!ASSERT_OK(err, "test_tc_peer__load")) + goto done; + + skel->links.tc_src_ing = netns_attach_nk(NS_SRC, + setup_result->ifindex_src, + skel->progs.tc_src_ing); + if (!ASSERT_OK_PTR(skel->links.tc_src_ing, "attach_src")) + goto done; + skel->links.tc_dst_ing = netns_attach_nk(NS_DST, + setup_result->ifindex_dst, + skel->progs.tc_dst_ing); + if (!ASSERT_OK_PTR(skel->links.tc_dst_ing, "attach_dst")) + goto done; + + if (!ASSERT_OK(set_forwarding(false), "disable forwarding")) + goto done; + + test_connectivity(); + +done: + if (skel) + test_tc_peer__destroy(skel); + close_netns(nstoken); +} + static int tun_open(char *name) { struct ifreq ifr; @@ -1280,6 +1347,7 @@ static void *test_tc_redirect_run_tests(void *arg) RUN_TEST(tc_redirect_peer, MODE_VETH); RUN_TEST(tc_redirect_peer, MODE_NETKIT); + RUN_TEST(tc_redirect_peer_ing, MODE_NETKIT); RUN_TEST(tc_redirect_peer_l3, MODE_VETH); RUN_TEST(tc_redirect_peer_l3, MODE_NETKIT); RUN_TEST(tc_redirect_neigh, MODE_VETH); diff --git a/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c new file mode 100644 index 000000000000..d0ba2ca587b0 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_map_uninit.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> + +#include "map_kptr.skel.h" + +void test_map_uninit_mem_exposure(void) +{ + size_t value_sz, slot_sz, lookup_sz, tail_sz; + int err, key, nr_cpus, cpu, map_fd; + __u8 *value = NULL, *zero = NULL; + struct bpf_program *prog; + struct map_kptr *skel; + + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "libbpf_num_possible_cpus")) + return; + + skel = map_kptr__open(); + if (!ASSERT_OK_PTR(skel, "map_kptr__open")) + return; + + bpf_object__for_each_program(prog, skel->obj) { + err = bpf_program__set_autoload(prog, false); + if (!ASSERT_OK(err, "bpf_program__set_autoload")) + goto out; + } + + err = map_kptr__load(skel); + if (!ASSERT_OK(err, "map_kptr__load")) + goto out; + + value_sz = bpf_map__value_size((skel)->maps.pcpu_array); + slot_sz = roundup(value_sz, 8); + tail_sz = slot_sz - value_sz; + if (!ASSERT_NEQ(tail_sz, 0, "tail_sz")) + goto out; + + lookup_sz = slot_sz * nr_cpus; + map_fd = bpf_map__fd(skel->maps.pcpu_array); + + value = malloc(lookup_sz); + zero = calloc(1, tail_sz); + if (!ASSERT_OK_PTR(value, "malloc value") || !ASSERT_OK_PTR(zero, "calloc zero")) + goto out; + + key = 0; + memset(value, 0x2B, lookup_sz); + err = bpf_map_update_elem(map_fd, &key, value, BPF_ANY); + if (!ASSERT_OK(err, "bpf_map_update_elem")) + goto out; + + memset(value, 0xFF, lookup_sz); + err = bpf_map_lookup_elem(map_fd, &key, value); + if (!ASSERT_OK(err, "bpf_map_lookup_elem")) + goto out; + + for (cpu = 0; cpu < nr_cpus; cpu++) { + __u8 *tail = value + cpu * slot_sz + value_sz; + + if (!ASSERT_MEMEQ(tail, zero, tail_sz, "zeroed tail bytes")) + goto out; + } + +out: + free(zero); + free(value); + map_kptr__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c new file mode 100644 index 000000000000..7f9f54ba3fbe --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <test_progs.h> + +#include "struct_ops_arena.skel.h" +#include "struct_ops_arena_attach.skel.h" +#include "struct_ops_arena_fail.skel.h" + +#if defined(__x86_64__) || defined(__aarch64__) +/* + * Attach callbacks with __arena and __arena__nullable arguments and drive + * them through the bpf_testmod_ops3_call_test_arena*() kfuncs. + */ +static void arena_arg(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct struct_ops_arena *skel; + struct bpf_link *link = NULL; + int err; + + skel = struct_ops_arena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_arena); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.trigger), + &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + +out: + bpf_link__destroy(link); + struct_ops_arena__destroy(skel); +} + +/* + * A program with no arena cannot attach to a member with an __arena + * argument. + */ +static void arena_arg_fail(void) +{ + struct struct_ops_arena_fail *skel; + + skel = struct_ops_arena_fail__open_and_load(); + if (ASSERT_ERR_PTR(skel, "struct_ops_arena_fail__open_and_load")) + return; + + struct_ops_arena_fail__destroy(skel); +} + +static void arena_arg_attach_one(int target_fd, const char *prog_name) +{ + struct struct_ops_arena_attach *skel; + struct bpf_program *prog, *pos; + char log_buf[64 * 1024]; + int err; + + skel = struct_ops_arena_attach__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_attach__open")) + return; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, prog_name)) + goto out; + + bpf_object__for_each_program(pos, skel->obj) + bpf_program__set_autoload(pos, pos == prog); + + err = bpf_program__set_attach_target(prog, target_fd, "test_arena_cb"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + log_buf[0] = '\0'; + bpf_program__set_log_buf(prog, log_buf, sizeof(log_buf)); + err = struct_ops_arena_attach__load(skel); + + ASSERT_EQ(err, -EOPNOTSUPP, prog_name); + ASSERT_HAS_SUBSTR(log_buf, "Cannot attach to a target with arena context arguments", + "verifier_log"); + +out: + struct_ops_arena_attach__destroy(skel); +} + +static void arena_arg_attach(void) +{ + struct struct_ops_arena *skel; + int target_fd; + + skel = struct_ops_arena__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena__open_and_load")) + return; + + target_fd = bpf_program__fd(skel->progs.test_arena_cb); + arena_arg_attach_one(target_fd, "fentry_test_arena"); + arena_arg_attach_one(target_fd, "fexit_test_arena"); + arena_arg_attach_one(target_fd, "freplace_test_arena"); + + struct_ops_arena__destroy(skel); +} +#endif + +/* + * Serialized because it attaches the singleton bpf_testmod_ops3, which + * test_struct_ops_private_stack also attaches; registering it twice fails + * with -EEXIST. + */ +void serial_test_struct_ops_arena(void) +{ + /* + * Arena struct_ops arguments need JIT support, currently x86-64 and + * arm64 only. Elsewhere verification fails with "JIT does not support + * arena arguments", so the programs cannot even load. + */ +#if defined(__x86_64__) || defined(__aarch64__) + if (test__start_subtest("arena_arg")) + arena_arg(); + if (test__start_subtest("arena_arg_fail")) + arena_arg_fail(); + if (test__start_subtest("arena_arg_attach")) + arena_arg_attach(); +#else + test__skip(); +#endif +} diff --git a/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c b/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c index 1aa7c9463980..67ba27d69347 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c +++ b/tools/testing/selftests/bpf/prog_tests/test_tc_tunnel.c @@ -438,7 +438,6 @@ static int setup(void) SYS(fail_close_ns_client, "ip link add %s type veth peer name %s", "veth1 mtu 1500 netns " CLIENT_NS " address " MAC_ADDR_VETH1, "veth2 mtu 1500 netns " SERVER_NS " address " MAC_ADDR_VETH2); - SYS(fail_close_ns_client, "ethtool -K veth1 tso off"); SYS(fail_close_ns_client, "ip link set veth1 up"); nstoken_server = open_netns(SERVER_NS); if (!ASSERT_OK_PTR(nstoken_server, "open server ns")) diff --git a/tools/testing/selftests/bpf/prog_tests/test_veristat.c b/tools/testing/selftests/bpf/prog_tests/test_veristat.c index 9aff08ac55c0..11f3de2b66ad 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_veristat.c +++ b/tools/testing/selftests/bpf/prog_tests/test_veristat.c @@ -37,6 +37,14 @@ static struct fixture *init_fixture(void) return fix; } +static void read_output(struct fixture *fix) +{ + ssize_t len = pread(fix->fd, fix->output, fix->sz - 1, 0); + + fix->output[len < 0 ? 0 : len] = 0; + ASSERT_GE(len, 0, "pread"); +} + static void teardown_fixture(struct fixture *fix) { free(fix->output); @@ -74,7 +82,7 @@ static void test_set_global_vars_succeeds(void) " -G \"struct11 [ 7 ] [ 5 ] .struct2[0][1].u.mat[3][0] = 175\" " \ " -vl2 > %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("=0xf000000000000001 ", "var_s64 = 0xf000000000000001"); __CHECK_STR("=0xfedcba9876543210 ", "var_u64 = 0xfedcba9876543210"); __CHECK_STR("=0x80000000 ", "var_s32 = -0x80000000"); @@ -116,7 +124,7 @@ static void test_set_global_vars_from_file_succeeds(void) syncfs(fd); SYS(out, "%s set_global_vars.bpf.o -G \"@%s\" -vl2 > %s", fix->veristat, input_file, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("=0x8000 ", "var_s16 = -32768"); __CHECK_STR("=0xecec ", "var_u16 = 60652"); @@ -134,7 +142,7 @@ static void test_set_global_vars_out_of_range(void) "%s set_global_vars.bpf.o -G \"var_s32 = 2147483648\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("is out of range [-2147483648; 2147483647]", "out of range"); out: @@ -149,7 +157,7 @@ static void test_unsupported_ptr_array_type(void) "%s set_global_vars.bpf.o -G \"ptr_arr[0] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Can't set ptr_arr[0]. Only ints and enums are supported", "ptr_arr"); out: @@ -164,7 +172,7 @@ static void test_array_out_of_bounds(void) "%s set_global_vars.bpf.o -G \"arr[99] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Array index 99 is out of bounds", "arr[99]"); out: @@ -179,7 +187,7 @@ static void test_array_index_not_found(void) "%s set_global_vars.bpf.o -G \"arr[EG2] = 0\" -vl2 2> %s", fix->veristat, fix->tmpfile); - read(fix->fd, fix->output, fix->sz); + read_output(fix); __CHECK_STR("Can't resolve enum value EG2", "arr[EG2]"); out: @@ -230,6 +238,97 @@ out: teardown_fixture(fix); } +/* + * Name filter tests below run veristat on veristat_foo.bpf.o and + * veristat_bar.bpf.o, both defining programs 'foo', 'bar' and 'buz'. + * Every entry describes a single (filters, file, prog) combination and + * tells whether that program is expected in the veristat output: + * 'true' if it is, 'false' if it is not and -1 if veristat is expected + * to reject the filter. + */ +#define FILTER_OBJS "veristat_foo.bpf.o veristat_bar.bpf.o" + +static const struct name_filter_case { + const char *filters; + const char *file; + const char *prog; + int included; +} name_filter_cases[] = { + /* no filters, every program is processed */ + { "", "foo", "foo", true }, + { "", "foo", "bar", true }, + { "", "foo", "buz", true }, + { "", "bar", "foo", true }, + { "", "bar", "bar", true }, + { "", "bar", "buz", true }, + /* deny filters */ + { "-f '!*foo*'", "foo", "bar", false }, + { "-f '!*foo*'", "bar", "foo", false }, + { "-f '!*foo*'", "bar", "bar", true }, + { "-f '!*foo*/bar'", "foo", "bar", false }, + { "-f '!*foo*/bar'", "foo", "buz", true }, + { "-f '!*foo*/bar'", "bar", "bar", true }, + { "-f '!*foo*/'", "foo", "bar", false }, + { "-f '!*foo*/'", "bar", "bar", true }, + { "-f '!/bar'", "foo", "bar", false }, + { "-f '!/bar'", "foo", "foo", true }, + { "-f '!/'", "foo", "bar", -1 }, + { "-f '!'", "foo", "bar", -1 }, + /* allow filters */ + { "-f '*foo*'", "foo", "bar", true }, + { "-f '*foo*'", "bar", "foo", true }, + { "-f '*foo*'", "bar", "bar", false }, + { "-f '*foo*/bar'", "foo", "bar", true }, + { "-f '*foo*/bar'", "foo", "buz", false }, + { "-f '*foo*/bar'", "bar", "bar", false }, + { "-f '*foo*/'", "foo", "bar", true }, + { "-f '*foo*/'", "bar", "bar", false }, + { "-f '/bar'", "foo", "bar", true }, + { "-f '/bar'", "foo", "foo", false }, + { "-f '/'", "foo", "bar", -1 }, + { "-f ''", "foo", "bar", -1 }, + /* allow and deny filters combined */ + { "-f '*foo*/' -f '!/bar'", "foo", "foo", true }, + { "-f '*foo*/' -f '!/bar'", "foo", "bar", false }, + { "-f '*foo*/' -f '!/bar'", "bar", "foo", false }, +}; + +static void test_name_filters(void) +{ + struct fixture *fix = init_fixture(); + const struct name_filter_case *t; + char cmd[512], row[64], name[128]; + int i, err; + + for (i = 0; i < ARRAY_SIZE(name_filter_cases); i++) { + t = &name_filter_cases[i]; + /* stderr is merged with stdout in order to catch error messages */ + snprintf(cmd, sizeof(cmd), "%s " FILTER_OBJS " -q -o csv -e file,prog %s > %s 2>&1", + fix->veristat, t->filters, fix->tmpfile); + err = system(cmd); + read_output(fix); + + snprintf(row, sizeof(row), "veristat_%s.bpf.o,%s", t->file, t->prog); + snprintf(name, sizeof(name), "veristat %s: %s", t->filters, row); + switch (t->included) { + case true: + ASSERT_OK(err, name); + ASSERT_HAS_SUBSTR(fix->output, row, name); + break; + case false: + ASSERT_OK(err, name); + ASSERT_FALSE(!!strstr(fix->output, row), name); + break; + case -1: + ASSERT_NEQ(err, 0, name); + ASSERT_HAS_SUBSTR(fix->output, "Invalid filter", name); + break; + } + } + + teardown_fixture(fix); +} + void test_veristat(void) { if (test__start_subtest("set_global_vars_succeeds")) @@ -256,6 +355,8 @@ void test_veristat(void) if (test__start_subtest("test_no_array_index_for_array")) test_no_array_index_for_array(); + if (test__start_subtest("name_filters")) + test_name_filters(); } #undef __CHECK_STR diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_failure.c b/tools/testing/selftests/bpf/prog_tests/tracing_failure.c index f9f9e1cb87bf..eb585918f0d4 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_failure.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_failure.c @@ -76,6 +76,24 @@ static void test_fexit_noreturns(void) "Attaching fexit/fsession/fmod_ret to __noreturn function 'do_exit' is rejected."); } +static void test_fexit_int128_ret(void) +{ + /* + * __int128 is returned in a register pair on x86_64 and arm64, so + * bpf_testmod_test_int128_ret() is BTF-encoded and attachable and the + * verifier can reject its >8 byte return value. Other architectures + * return a __int128 differently (e.g. s390x returns larger values by + * reference, which makes pahole skip BTF encoding of the function), so + * only exercise this on x86_64 and arm64. + */ +#if defined(__x86_64__) || defined(__aarch64__) + test_tracing_fail_prog("fexit_int128_ret", + "with a >8 byte return value is not supported for this attach type"); +#else + test__skip(); +#endif +} + void test_tracing_failure(void) { if (test__start_subtest("bpf_spin_lock")) @@ -86,4 +104,6 @@ void test_tracing_failure(void) test_tracing_deny(); if (test__start_subtest("fexit_noreturns")) test_fexit_noreturns(); + if (test__start_subtest("fexit_int128_ret")) + test_fexit_int128_ret(); } diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c index f02ffc7f41d7..0aa9532a05cf 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_multi.c @@ -460,6 +460,73 @@ static void test_intersect(void) tracing_multi_intersect__destroy(skel); } +static void test_fentry_after_multi(void) +{ + static const char * const funcs[] = { + "bpf_fentry_test1", + }; + struct bpf_link *fentry_link = NULL, *multi_link = NULL; + struct tracing_multi_intersect *skel = NULL; + LIBBPF_OPTS(bpf_tracing_multi_opts, opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); + __u32 *ids = NULL; + int err; + + skel = tracing_multi_intersect__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_multi_intersect__open_and_load")) + return; + + skel->bss->pid = getpid(); + + ids = get_ids(funcs, ARRAY_SIZE(funcs), NULL); + if (!ASSERT_OK_PTR(ids, "get_ids")) + goto cleanup; + + opts.ids = ids; + opts.cnt = ARRAY_SIZE(funcs); + multi_link = bpf_program__attach_tracing_multi(skel->progs.fentry_1, NULL, &opts); + if (!ASSERT_OK_PTR(multi_link, "attach_multi")) + goto cleanup; + + fentry_link = bpf_program__attach(skel->progs.fentry); + if (!ASSERT_OK_PTR(fentry_link, "attach_fentry")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 1, "multi_fentry"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry"); + + err = bpf_link__destroy(fentry_link); + fentry_link = NULL; + if (!ASSERT_OK(err, "destroy_fentry")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run_multi")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 2, "multi_fentry_only"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry_detached"); + + err = bpf_link__destroy(multi_link); + multi_link = NULL; + if (!ASSERT_OK(err, "destroy_multi")) + goto cleanup; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry_1), &topts); + if (!ASSERT_OK(err, "test_run_detached")) + goto cleanup; + ASSERT_EQ(skel->bss->test_result_fentry_1, 2, "multi_fentry_detached"); + ASSERT_EQ(skel->bss->test_result_fentry, 1, "fentry_still_detached"); + +cleanup: + bpf_link__destroy(fentry_link); + bpf_link__destroy(multi_link); + free(ids); + tracing_multi_intersect__destroy(skel); +} + static void test_session(void) { LIBBPF_OPTS(bpf_test_run_opts, topts); @@ -957,4 +1024,6 @@ void test_tracing_multi_test(void) if (test__start_subtest("attach_api_fails")) test_attach_api_fails(); RUN_TESTS(tracing_multi_verifier); + if (test__start_subtest("fentry_after_multi")) + test_fentry_after_multi(); } diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 6f8c0bfb0415..15b95d0235b5 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -4,6 +4,7 @@ #include <test_progs.h> #include "tracing_struct.skel.h" #include "tracing_struct_many_args.skel.h" +#include "tracing_struct_int128.skel.h" static void test_struct_args(void) { @@ -112,6 +113,39 @@ destroy_skel: tracing_struct_many_args__destroy(skel); } +static void test_int128_args(void) +{ + /* + * __int128 arguments are passed in a register pair on x86_64 and + * arm64, which the trampoline packs into two context slots. Other + * architectures pass a __int128 differently (e.g. s390x passes larger + * arguments by reference), so only exercise this on x86_64 and arm64. + */ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_struct_int128 *skel; + int err; + + skel = tracing_struct_int128__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_struct_int128__open_and_load")) + return; + + err = tracing_struct_int128__attach(skel); + if (!ASSERT_OK(err, "tracing_struct_int128__attach")) + goto destroy_skel; + + ASSERT_OK(trigger_module_test_read(256), "trigger_read"); + + ASSERT_EQ(skel->bss->t_b, 2, "t:b"); + ASSERT_EQ(skel->bss->t_c, 3, "t:c"); + ASSERT_EQ(skel->bss->t_ret, 6, "t ret"); + +destroy_skel: + tracing_struct_int128__destroy(skel); +#else + test__skip(); +#endif +} + static void test_union_args(void) { struct tracing_struct *skel; @@ -145,6 +179,8 @@ void test_tracing_struct(void) test_struct_args(); if (test__start_subtest("struct_many_args")) test_struct_many_args(); + if (test__start_subtest("int128_args")) + test_int128_args(); if (test__start_subtest("union_args")) test_union_args(); } diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index be97f6887f0e..64ac49ad67e6 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -2,6 +2,8 @@ #include <test_progs.h> +#include "arena_kfunc.skel.h" +#include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" #include "verifier_align.skel.h" #include "verifier_and.skel.h" @@ -61,12 +63,14 @@ #include "verifier_loops1.skel.h" #include "verifier_lwt.skel.h" #include "verifier_map_in_map.skel.h" +#include "verifier_map_lookup_refine.skel.h" #include "verifier_map_ptr.skel.h" #include "verifier_map_ptr_mixing.skel.h" #include "verifier_map_ret_val.skel.h" #include "verifier_masking.skel.h" #include "verifier_may_goto_1.skel.h" #include "verifier_may_goto_2.skel.h" +#include "verifier_mem_size_reg.skel.h" #include "verifier_meta_access.skel.h" #include "verifier_movsx.skel.h" #include "verifier_mtu.skel.h" @@ -75,6 +79,7 @@ #include "verifier_netfilter_retcode.skel.h" #include "verifier_bpf_fastcall.skel.h" #include "verifier_or_jmp32_k.skel.h" +#include "verifier_percpu_addr.skel.h" #include "verifier_precision.skel.h" #include "verifier_prevent_map_lookup.skel.h" #include "verifier_private_stack.skel.h" @@ -98,6 +103,7 @@ #include "verifier_stack_arg_order.skel.h" #include "verifier_stack_ptr.skel.h" #include "verifier_store_release.skel.h" +#include "verifier_subprog_insn_stats.skel.h" #include "verifier_subprog_precision.skel.h" #include "verifier_subprog_topo.skel.h" #include "verifier_subreg.skel.h" @@ -124,6 +130,7 @@ #include "verifier_jit_inline.skel.h" #include "irq.skel.h" #include "verifier_ctx_ptr_param.skel.h" +#include "verifier_zext.skel.h" #define MAX_ENTRIES 11 @@ -159,6 +166,10 @@ static void run_tests_aux(const char *skel_name, #define RUN(skel) run_tests_aux(#skel, skel##__elf_bytes, NULL) +void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } + +void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } + void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } @@ -215,12 +226,14 @@ void test_verifier_liveness_exp(void) { RUN(verifier_liveness_exp); } void test_verifier_loops1(void) { RUN(verifier_loops1); } void test_verifier_lwt(void) { RUN(verifier_lwt); } void test_verifier_map_in_map(void) { RUN(verifier_map_in_map); } +void test_verifier_map_lookup_refine(void) { RUN(verifier_map_lookup_refine); } void test_verifier_map_ptr(void) { RUN(verifier_map_ptr); } void test_verifier_map_ptr_mixing(void) { RUN(verifier_map_ptr_mixing); } void test_verifier_map_ret_val(void) { RUN(verifier_map_ret_val); } void test_verifier_masking(void) { RUN(verifier_masking); } void test_verifier_may_goto_1(void) { RUN(verifier_may_goto_1); } void test_verifier_may_goto_2(void) { RUN(verifier_may_goto_2); } +void test_verifier_mem_size_reg(void) { RUN(verifier_mem_size_reg); } void test_verifier_meta_access(void) { RUN(verifier_meta_access); } void test_verifier_movsx(void) { RUN(verifier_movsx); } void test_verifier_mul(void) { RUN(verifier_mul); } @@ -228,6 +241,7 @@ void test_verifier_netfilter_ctx(void) { RUN(verifier_netfilter_ctx); } void test_verifier_netfilter_retcode(void) { RUN(verifier_netfilter_retcode); } void test_verifier_bpf_fastcall(void) { RUN(verifier_bpf_fastcall); } void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); } +void test_verifier_percpu_addr(void) { RUN(verifier_percpu_addr); } void test_verifier_precision(void) { RUN(verifier_precision); } void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); } void test_verifier_private_stack(void) { RUN(verifier_private_stack); } @@ -251,6 +265,7 @@ void test_verifier_stack_arg(void) { RUN(verifier_stack_arg); } void test_verifier_stack_arg_order(void) { RUN(verifier_stack_arg_order); } void test_verifier_stack_ptr(void) { RUN(verifier_stack_ptr); } void test_verifier_store_release(void) { RUN(verifier_store_release); } +void test_verifier_subprog_insn_stats(void) { RUN(verifier_subprog_insn_stats); } void test_verifier_subprog_precision(void) { RUN(verifier_subprog_precision); } void test_verifier_subprog_topo(void) { RUN(verifier_subprog_topo); } void test_verifier_subreg(void) { RUN(verifier_subreg); } @@ -277,6 +292,7 @@ void test_irq(void) { RUN(irq); } void test_verifier_mtu(void) { RUN(verifier_mtu); } void test_verifier_jit_inline(void) { RUN(verifier_jit_inline); } void test_verifier_ctx_ptr_param(void) { RUN(verifier_ctx_ptr_param); } +void test_verifier_zext(void) { RUN_TESTS(verifier_zext); } static int init_test_val_map(struct bpf_object *obj, char *map_name) { diff --git a/tools/testing/selftests/bpf/progs/arena_atomics.c b/tools/testing/selftests/bpf/progs/arena_atomics.c index 2e7751a85399..73bc2b835f3f 100644 --- a/tools/testing/selftests/bpf/progs/arena_atomics.c +++ b/tools/testing/selftests/bpf/progs/arena_atomics.c @@ -28,8 +28,10 @@ bool skip_all_tests = true; #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) bool skip_lacq_srel_tests __attribute((__section__(".data"))) = false; #else bool skip_lacq_srel_tests = true; @@ -315,8 +317,10 @@ int load_acquire(const void *ctx) { #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) #define LOAD_ACQUIRE_ARENA(SIZEOP, SIZE, SRC, DST) \ { asm volatile ( \ @@ -367,8 +371,10 @@ int store_release(const void *ctx) { #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) #define STORE_RELEASE_ARENA(SIZEOP, DST, VAL) \ { asm volatile ( \ diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c new file mode 100644 index 000000000000..bf0d304e0e59 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod_kfunc.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* page 0 hosts the arena global, page 1 is for allocations */ + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* + * Occupies page 0 so no allocation lands at arena offset 0, which the + * nullable tests below must be able to tell apart from NULL. + */ +u64 __arena arena_pad; + +/* volatile to force the scalar reloads below */ +volatile u64 stash; + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_forms(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + u64 ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + /* PTR_TO_ARENA argument */ + *val = 41; + ret = bpf_kfunc_arena_arg_test((u64 *)val); + if (ret != 41 || *val != 42) + return 2; + + /* the low 32 bits as a scalar */ + stash = (u32)(u64)val; + ret = bpf_kfunc_arena_arg_test((u64 *)stash); + if (ret != 42 || *val != 43) + return 3; + + /* the full user address as a scalar */ + stash = (u64)val; + bpf_addr_space_cast(stash, 1, 0); + ret = bpf_kfunc_arena_arg_test((u64 *)stash); + if (ret != 43 || *val != 44) + return 4; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +/* + * Pin the rebase semantics using the capture kfuncs, which return the raw + * argument value: __arena rebases unconditionally, so zero low 32 bits + * arrive as the arena kernel base, while __arena__nullable turns them into + * NULL. + */ +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_rebase(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + u64 base, off; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + base = bpf_kfunc_arena_cap_test(NULL); + if (!base) + return 2; + + /* only the low 32 bits contribute */ + stash = 0xbadc0ffe00000000; + if (bpf_kfunc_arena_cap_test((u64 *)stash) != base) + return 3; + + off = (u32)(u64)val; + if (bpf_kfunc_arena_cap_test((u64 *)val) != base + off) + return 4; + + if (bpf_kfunc_arena_cap_nullable_test(NULL) != 0) + return 5; + + stash = 0xbadc0ffe00000000; + if (bpf_kfunc_arena_cap_nullable_test((u64 *)stash) != 0) + return 6; + + if (bpf_kfunc_arena_cap_nullable_test((u64 *)val) != base + off) + return 7; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_args5(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + val[0] = 1; + val[1] = 2; + val[2] = 4; + val[3] = 8; + val[4] = 16; + + if (bpf_kfunc_arena_args5_test((u64 *)&val[0], (u64 *)&val[1], + (u64 *)&val[2], (u64 *)&val[3], + (u64 *)&val[4]) != 31) + return 2; + if (bpf_kfunc_arena_args5_test((u64 *)&val[0], (u64 *)&val[1], + (u64 *)&val[2], (u64 *)&val[3], NULL) != 15) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_mixed(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + val[0] = 7; + val[1] = 5; + + if (bpf_kfunc_arena_mixed_test((u64 *)&val[0], NULL) != 7) + return 2; + + if (bpf_kfunc_arena_mixed_test((u64 *)&val[0], (u64 *)&val[1]) != 12) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +/* kernel-side faults on unpopulated pages recover via the scratch page */ +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_unpopulated(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + stash = (u64)val + PAGE_SIZE; + bpf_kfunc_arena_arg_test((u64 *)stash); + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("arena pointer requires a program with an associated arena") +int arena_arg_no_arena(void *ctx) +{ + bpf_kfunc_arena_arg_test((u64 *)1); + return 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("is not a pointer to arena or scalar") +int arena_arg_bad_reg(void *ctx) +{ + u64 buf = 0; + + /* use the arena so the program passes the arena presence check */ + bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_arg_test(&buf); + return 0; +} + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("arena pointer cannot be a stack argument") +int arena_arg_stack(void *ctx) +{ + bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_stack_arg_test(1, 2, 3, 4, 5, (u64 *)1); + return 0; +} +#else +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__description("arena_arg_stack: not supported, dummy test") +__success +int arena_arg_stack(void *ctx) +{ + return 0; +} +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c b/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c new file mode 100644 index 000000000000..b5a01cbc33a7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_kfunc_jit.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +/* + * Verify the JIT-emitted rebase sequences for __arena and __arena__nullable + * kfunc arguments. The capture kfuncs take the argument without + * dereferencing it, so these tests pin only the emitted code. + */ +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod_kfunc.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* volatile to force the scalar reloads below */ +volatile u64 stash; + +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + +SEC("syscall") +__arch_x86_64 +__jited("...") +__jited(" movl %edi, %edi") +__jited(" addq %r12, %rdi") +__jited("...") +__jited(" callq {{.*}}") +__arch_arm64 +__jited("...") +__jited(" add x0, x28, w0, uxtw") +__jited(" {{(bl|mov) .*}}") +__success +int arena_arg_jit_rebase(void *ctx) +{ + stash = (u64)bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_cap_test((u64 *)stash); + return 0; +} + +SEC("syscall") +__arch_x86_64 +__jited("...") +__jited(" movl %edi, %edi") +__jited(" testl %edi, %edi") +__jited(" je L0") +__jited(" addq %r12, %rdi") +__jited("L0: callq {{.*}}") +__arch_arm64 +__jited("...") +__jited(" mov w0, w0") +__jited(" cbz w0, L0") +__jited(" add x0, x28, w0, uxtw") +__jited("L0: {{.*}}") +__success +int arena_arg_jit_nullable(void *ctx) +{ + stash = (u64)bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + bpf_kfunc_arena_cap_nullable_test((u64 *)stash); + return 0; +} + +SEC("syscall") +__arch_x86_64 +__jited("...") +__jited(" movl %edi, %edi") +__jited(" addq %r12, %rdi") +__jited(" movl %esi, %esi") +__jited(" addq %r12, %rsi") +__jited(" movl %edx, %edx") +__jited(" addq %r12, %rdx") +__jited(" movl %ecx, %ecx") +__jited(" addq %r12, %rcx") +__jited(" movl %r8d, %r8d") +__jited(" testl %r8d, %r8d") +__jited(" je L0") +__jited(" addq %r12, %r8") +__jited("L0: callq {{.*}}") +__arch_arm64 +__jited("...") +__jited(" add x0, x28, w0, uxtw") +__jited(" add x1, x28, w1, uxtw") +__jited(" add x2, x28, w2, uxtw") +__jited(" add x3, x28, w3, uxtw") +__jited(" mov w4, w4") +__jited(" cbz w4, L0") +__jited(" add x4, x28, w4, uxtw") +__jited("L0: {{.*}}") +__success +int arena_arg_jit_args5(void *ctx) +{ + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + val[0] = 1; + val[1] = 2; + val[2] = 4; + val[3] = 8; + val[4] = 16; + + bpf_kfunc_arena_args5_test((u64 *)&val[0], (u64 *)&val[1], + (u64 *)&val[2], (u64 *)&val[3], + (u64 *)&val[4]); + return 0; +} + +#endif /* __BPF_FEATURE_ADDR_SPACE_CAST */ + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_mem_usage.c b/tools/testing/selftests/bpf/progs/arena_mem_usage.c new file mode 100644 index 000000000000..455ecd669a5a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_mem_usage.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_arena_common.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1000); /* number of pages */ +#ifdef __TARGET_ARCH_arm64 + __ulong(map_extra, 0x1ull << 32); /* start of mmap() region */ +#else + __ulong(map_extra, 0x1ull << 44); /* start of mmap() region */ +#endif +} arena SEC(".maps"); + +void __arena *ptr; +int alloc_cnt; /* in: pages to allocate */ +long free_byte_off; /* in: byte offset within ptr to start freeing */ +int free_cnt; /* in: pages to free */ + +SEC("syscall") +int alloc(void *ctx) +{ + ptr = bpf_arena_alloc_pages(&arena, NULL, alloc_cnt, NUMA_NO_NODE, 0); + /* Success/failure is checked from user space via skel->bss->ptr. */ + return 0; +} + +SEC("syscall") +int free_pages(void *ctx) +{ + if (!ptr) + return 1; + bpf_arena_free_pages(&arena, (char __arena *)ptr + free_byte_off, free_cnt); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/arena_spin_lock.c b/tools/testing/selftests/bpf/progs/arena_spin_lock.c index cf7cda79c16c..92e75ec3844c 100644 --- a/tools/testing/selftests/bpf/progs/arena_spin_lock.c +++ b/tools/testing/selftests/bpf/progs/arena_spin_lock.c @@ -23,6 +23,13 @@ int cs_count; #if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST) arena_spinlock_t __arena lock; int test_skip = 1; + +/* + * Storage for the queue nodes declared by bpf_arena_spin_lock.h. Each program + * linking the arena spinlock provides exactly one definition; libarena's lives + * in libarena/src/common.bpf.c. + */ +struct arena_qnode __arena __hidden qnodes[_Q_MAX_CPUS][_Q_MAX_NODES]; #else int test_skip = 2; #endif diff --git a/tools/testing/selftests/bpf/progs/bpf_for_bench.c b/tools/testing/selftests/bpf/progs/bpf_for_bench.c new file mode 100644 index 000000000000..f9c723051fc7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_for_bench.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +int nr_loops; +long hits; + +static int outer_loop(__u32 index, void *data) +{ + int i; + + /* + * Empty body: the work being measured is the open-coded numeric iterator itself + * (bpf_iter_num_new/next/destroy behind bpf_for()). + */ + bpf_for(i, 0, nr_loops) + ; + __sync_add_and_fetch(&hits, nr_loops); + return 0; +} + +SEC("fentry/" SYS_PREFIX "sys_getpgid") +int benchmark(void *ctx) +{ + bpf_loop(1000, outer_loop, NULL, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index b0c441384f20..5eacf1b43252 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -264,9 +264,12 @@ #endif #if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ - defined(__TARGET_ARCH_powerpc) || defined(__TARGET_ARCH_loongarch)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390) || \ + defined(__TARGET_ARCH_powerpc) || \ + defined(__TARGET_ARCH_loongarch)) #define CAN_USE_LOAD_ACQ_STORE_REL #endif diff --git a/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__untrusted_write.c b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__untrusted_write.c new file mode 100644 index 000000000000..688c2a049ae3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_qdisc_fail__untrusted_write.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include "bpf_experimental.h" +#include "bpf_qdisc_common.h" +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +SEC("struct_ops") +__failure __msg("only read is supported") +int BPF_PROG(untrusted_write, struct sk_buff *skb, struct Qdisc *sch, + struct bpf_sk_buff_ptr *to_free) +{ + struct Qdisc *next = sch->next_sched; + + /* + * sch is trusted, but the walk of next_sched yields a plain + * PTR_TO_BTF_ID which may fault on a dereference. A store through + * it does not get an exception table entry, there is no probed + * store to rewrite it into, hence it has to be rejected before + * bpf_qdisc_btf_struct_access() gets to allow the write to limit. + */ + next->limit = 1000; + + bpf_qdisc_skb_drop(skb, to_free); + return NET_XMIT_DROP; +} + +SEC("struct_ops") +__auxiliary +struct sk_buff *BPF_PROG(bpf_qdisc_test_dequeue, struct Qdisc *sch) +{ + return NULL; +} + +SEC("struct_ops") +__auxiliary +int BPF_PROG(bpf_qdisc_test_init, struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + return 0; +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_reset, struct Qdisc *sch) +{ +} + +SEC("struct_ops") +__auxiliary +void BPF_PROG(bpf_qdisc_test_destroy, struct Qdisc *sch) +{ +} + +SEC(".struct_ops") +struct Qdisc_ops test = { + .enqueue = (void *)untrusted_write, + .dequeue = (void *)bpf_qdisc_test_dequeue, + .init = (void *)bpf_qdisc_test_init, + .reset = (void *)bpf_qdisc_test_reset, + .destroy = (void *)bpf_qdisc_test_destroy, + .id = "bpf_qdisc_test", +}; diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index d8dacef37c16..593b38f90417 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -8,6 +8,9 @@ #define AF_INET 2 #define AF_INET6 10 +/* include/linux/net.h */ +#define SOCK_TYPE_MASK 0xf + #define SOL_SOCKET 1 #define SO_REUSEADDR 2 #define SO_SNDBUF 7 @@ -28,10 +31,12 @@ #define __SO_ACCEPTCON (1 << 16) #define IP_TOS 1 +#define IP_TRANSPARENT 19 #define SOL_IPV6 41 #define IPV6_TCLASS 67 #define IPV6_AUTOFLOWLABEL 70 +#define IPV6_TRANSPARENT 75 #define TC_ACT_UNSPEC (-1) #define TC_ACT_OK 0 diff --git a/tools/testing/selftests/bpf/progs/btf_data.c b/tools/testing/selftests/bpf/progs/btf_data.c index baa525275bde..8082c13490ab 100644 --- a/tools/testing/selftests/bpf/progs/btf_data.c +++ b/tools/testing/selftests/bpf/progs/btf_data.c @@ -48,3 +48,43 @@ int func(struct root_struct *root) { return 0; } + +int kfunc_a(struct root_struct *root) +{ + return 0; +} + +int kfunc_b(struct root_struct *root) +{ + return 0; +} + +struct root_struct *kfunc_c(struct root_struct *a, struct root_struct *b) +{ + return a; +} + +int kfunc_d(struct root_struct *a, struct root_struct *b) +{ + return 0; +} + +int kfunc_e(struct root_struct *a__arena, + struct root_struct *b__arena__nullable, + struct root_struct *c__arena, + struct root_struct *d__arena__nullable, + struct root_struct *e__arena) +{ + return 0; +} + +int kfunc_f(struct root_struct *a, struct root_struct *b__arena, int flags) +{ + return 0; +} + +struct root_struct *kfunc_g(struct root_struct *a__arena, + struct root_struct *b__arena__nullable) +{ + return a__arena; +} diff --git a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c index d0d65d6d450c..efe7bcae70f8 100644 --- a/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/cgrp_kfunc_failure.c @@ -64,7 +64,7 @@ int BPF_PROG(cgrp_kfunc_acquire_no_null_check, struct cgroup *cgrp, const char * } SEC("tp_btf/cgroup_mkdir") -__failure __msg("R1 pointer type STRUCT cgroup must point") +__failure __msg("R1 is fp expected STRUCT cgroup") int BPF_PROG(cgrp_kfunc_acquire_fp, struct cgroup *cgrp, const char *path) { struct cgroup *acquired, *stack_cgrp = (struct cgroup *)&path; diff --git a/tools/testing/selftests/bpf/progs/cpumask_common.h b/tools/testing/selftests/bpf/progs/cpumask_common.h index 86085b79f5ca..8fe01308d210 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_common.h +++ b/tools/testing/selftests/bpf/progs/cpumask_common.h @@ -61,7 +61,7 @@ u32 bpf_cpumask_any_distribute(const struct cpumask *src) __ksym __weak; u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, const struct cpumask *src2) __ksym __weak; u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym __weak; -int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) __ksym __weak; +int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) __ksym __weak; void bpf_rcu_read_lock(void) __ksym __weak; void bpf_rcu_read_unlock(void) __ksym __weak; diff --git a/tools/testing/selftests/bpf/progs/cpumask_failure.c b/tools/testing/selftests/bpf/progs/cpumask_failure.c index 4c45346fe6f7..4628feb53d86 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_failure.c +++ b/tools/testing/selftests/bpf/progs/cpumask_failure.c @@ -116,9 +116,9 @@ int BPF_PROG(test_cpumask_null, struct task_struct *task, u64 clone_flags) return 0; } -SEC("tp_btf/task_newtask") +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("R2 must be a rcu pointer") -int BPF_PROG(test_global_mask_out_of_rcu, struct task_struct *task, u64 clone_flags) +int BPF_PROG(test_global_mask_out_of_rcu) { struct bpf_cpumask *local, *prev; @@ -133,6 +133,10 @@ int BPF_PROG(test_global_mask_out_of_rcu, struct task_struct *task, u64 clone_fl return 0; } + /* + * Use a sleepable program so explicit RCU is the only source of RCU + * protection. + */ bpf_rcu_read_lock(); local = global_mask; if (!local) { @@ -231,7 +235,7 @@ int BPF_PROG(test_populate_invalid_destination, struct task_struct *task, u64 cl u64 bits; int ret; - ret = bpf_cpumask_populate((struct cpumask *)invalid, &bits, sizeof(bits)); + ret = bpf_cpumask_populate(invalid, &bits, sizeof(bits)); if (!ret) err = 2; @@ -252,7 +256,7 @@ int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_f return 0; } - ret = bpf_cpumask_populate((struct cpumask *)local, garbage, 8); + ret = bpf_cpumask_populate(local, garbage, 8); if (!ret) err = 2; @@ -260,3 +264,22 @@ int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_f return 0; } + +SEC("tp_btf/task_newtask") +__failure __msg("expected pointer to STRUCT bpf_cpumask but R1 has a pointer to STRUCT cpumask") +int BPF_PROG(test_populate_borrowed_destination, struct task_struct *task, u64 clone_flags) +{ + u64 bits; + int ret; + + /* + * task->cpus_ptr is a borrowed, read-only struct cpumask *, not an + * owned struct bpf_cpumask *. The verifier must reject it as a + * writable destination for bpf_cpumask_populate(). + */ + ret = bpf_cpumask_populate((struct bpf_cpumask *)task->cpus_ptr, &bits, sizeof(bits)); + if (!ret) + err = 2; + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/cpumask_success.c b/tools/testing/selftests/bpf/progs/cpumask_success.c index 774706e7b058..36f77b9732d4 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_success.c +++ b/tools/testing/selftests/bpf/progs/cpumask_success.c @@ -785,7 +785,7 @@ int BPF_PROG(test_populate_reject_small_mask, struct task_struct *task, u64 clon return 0; /* The kfunc should prevent this operation */ - ret = bpf_cpumask_populate((struct cpumask *)local, &toofewbits, sizeof(toofewbits)); + ret = bpf_cpumask_populate(local, &toofewbits, sizeof(toofewbits)); if (ret != -EACCES) err = 2; @@ -824,7 +824,7 @@ int BPF_PROG(test_populate_reject_unaligned, struct task_struct *task, u64 clone /* Misalign the source array by a byte. */ src = &((char *)bits)[1]; - ret = bpf_cpumask_populate((struct cpumask *)mask, src, CPUMASK_TEST_MASKLEN); + ret = bpf_cpumask_populate(mask, src, CPUMASK_TEST_MASKLEN); if (ret != -EINVAL) err = 2; @@ -855,7 +855,7 @@ int BPF_PROG(test_populate, struct task_struct *task, u64 clone_flags) } /* Pass the entire bits array, the kfunc will only copy the valid bits. */ - ret = bpf_cpumask_populate((struct cpumask *)mask, bits, CPUMASK_TEST_MASKLEN); + ret = bpf_cpumask_populate(mask, bits, CPUMASK_TEST_MASKLEN); if (ret) { err = 2; goto out; diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 29c6361d8820..1cd61d72c166 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -1112,7 +1112,7 @@ int dynptr_overwrite_ref(void *ctx) /* Reject writes to dynptr slot from bpf_dynptr_read */ SEC("?raw_tp") -__failure __msg("potential write to dynptr at off=-16") +__failure __msg("cannot overwrite referenced dynptr") int dynptr_read_into_slot(void *ctx) { union { @@ -1558,7 +1558,7 @@ int BPF_PROG(skb_invalid_ctx_fexit, void *skb) /* Reject writes to dynptr slot for uninit arg */ SEC("?raw_tp") -__failure __msg("potential write to dynptr at off=-16") +__failure __msg("cannot overwrite referenced dynptr") int uninit_write_into_slot(void *ctx) { struct { @@ -1589,7 +1589,8 @@ int xdp_invalid_ctx(void *ctx) __u32 hdr_size = sizeof(struct ethhdr); /* Can't pass in variable-sized len to bpf_dynptr_slice */ SEC("?tc") -__failure __msg("unbounded memory access") +__failure __msg("must be a known constant") +__msg("requires this memory size to be a verifier-known constant") int dynptr_slice_var_len1(struct __sk_buff *skb) { struct bpf_dynptr ptr; @@ -1609,6 +1610,7 @@ int dynptr_slice_var_len1(struct __sk_buff *skb) /* Can't pass in variable-sized len to bpf_dynptr_slice */ SEC("?tc") __failure __msg("must be a known constant") +__msg("requires this memory size to be a verifier-known constant") int dynptr_slice_var_len2(struct __sk_buff *skb) { char buffer[sizeof(struct ethhdr)] = {}; diff --git a/tools/testing/selftests/bpf/progs/fentry_sleepable.c b/tools/testing/selftests/bpf/progs/fentry_sleepable.c new file mode 100644 index 000000000000..8c0fc691d329 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/fentry_sleepable.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char LICENSE[] SEC("license") = "GPL"; + +void *user_ptr; +int retval; + +SEC("fentry.s") +int BPF_PROG(fentry_xdp) +{ + char buff[64]; + + retval = bpf_copy_from_user(buff, sizeof(buff), user_ptr); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/fib_lookup.c b/tools/testing/selftests/bpf/progs/fib_lookup.c index 7b5dd2214ff4..36b7218d9ae2 100644 --- a/tools/testing/selftests/bpf/progs/fib_lookup.c +++ b/tools/testing/selftests/bpf/progs/fib_lookup.c @@ -4,7 +4,11 @@ #include <linux/types.h> #include <linux/bpf.h> #include <linux/pkt_cls.h> +#include <linux/if_ether.h> +#include <linux/ip.h> +#include <linux/in.h> #include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> struct bpf_fib_lookup fib_params = {}; int fib_lookup_ret = 0; @@ -19,4 +23,57 @@ int fib_lookup(struct __sk_buff *skb) return TC_ACT_SHOT; } +SEC("xdp") +int fib_lookup_xdp(struct xdp_md *ctx) +{ + fib_lookup_ret = bpf_fib_lookup(ctx, &fib_params, sizeof(fib_params), + lookup_flags); + + return XDP_DROP; +} + +int redirected = 0; +int passed = 0; +int delivered = 0; + +SEC("xdp") +int fib_lookup_redirect(struct xdp_md *ctx) +{ + struct bpf_fib_lookup params = fib_params; + long ret; + + ret = bpf_fib_lookup(ctx, ¶ms, sizeof(params), lookup_flags); + if (ret == BPF_FIB_LKUP_RET_SUCCESS) { + redirected++; + return bpf_redirect(params.ifindex, 0); + } + + passed++; + return XDP_PASS; +} + +SEC("xdp") +int xdp_count(struct xdp_md *ctx) +{ + void *data = (void *)(long)ctx->data; + void *data_end = (void *)(long)ctx->data_end; + struct ethhdr *eth = data; + struct iphdr *iph; + + /* + * count only the test's TCP frames: the netns has live + * link-local traffic (DAD, MLD) that would satisfy a bare + * counter + */ + if ((void *)(eth + 1) > data_end || + eth->h_proto != bpf_htons(ETH_P_IP)) + return XDP_DROP; + iph = (void *)(eth + 1); + if ((void *)(iph + 1) > data_end || iph->protocol != IPPROTO_TCP) + return XDP_DROP; + + delivered++; + return XDP_DROP; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c b/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c deleted file mode 100644 index 7e94412d47a5..000000000000 --- a/tools/testing/selftests/bpf/progs/freplace_cls_redirect.c +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (c) 2020 Facebook - -#include <linux/stddef.h> -#include <linux/bpf.h> -#include <linux/pkt_cls.h> -#include <bpf/bpf_endian.h> -#include <bpf/bpf_helpers.h> - -struct { - __uint(type, BPF_MAP_TYPE_SOCKMAP); - __type(key, int); - __type(value, int); - __uint(max_entries, 2); -} sock_map SEC(".maps"); - -SEC("freplace/cls_redirect") -int freplace_cls_redirect_test(struct __sk_buff *skb) -{ - int ret = 0; - const int zero = 0; - struct bpf_sock *sk; - - sk = bpf_map_lookup_elem(&sock_map, &zero); - if (!sk) - return TC_ACT_SHOT; - - ret = bpf_map_update_elem(&sock_map, &zero, sk, 0); - bpf_sk_release(sk); - - return ret == 0 ? TC_ACT_OK : TC_ACT_SHOT; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/get_smp_processor_id.c b/tools/testing/selftests/bpf/progs/get_smp_processor_id.c new file mode 100644 index 000000000000..cf4791a5cf07 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/get_smp_processor_id.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +__u64 cpu_nr_result; + +SEC("raw_tp") +void call_bpf_get_smp_processor_id(void) +{ + register __u64 r0 asm("r0") = -1; + asm volatile ("call %[bpf_get_smp_processor_id];" + : "+r"(r0) + : __imm(bpf_get_smp_processor_id) + : "r1", "r2", "r3", "r4", "r5", "memory"); + cpu_nr_result = r0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/icmp_send.c b/tools/testing/selftests/bpf/progs/icmp_send.c new file mode 100644 index 000000000000..c642ccdf9fd5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/icmp_send.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> + +/* 127.0.0.1 in host byte order */ +#define SERVER_IP 0x7F000001 +/* ::1 in host byte order (last 32-bit word) */ +#define SERVER_IP6_LO 0x00000001 + +__u16 server_port = 0; +int unreach_type = 0; +int unreach_code = 0; +int kfunc_ret = -1; +int target_pid = -1; + +unsigned int rec_count = 0; +int rec_kfunc_rets[] = { -1, -1 }; + +SEC("cgroup_skb/egress") +int egress(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct iphdr *iph; + struct ipv6hdr *ip6h; + struct tcphdr *tcph; + __u8 version; + + if (data + 1 > data_end) + return SK_PASS; + + version = (*((__u8 *)data)) >> 4; + + if (version == 4) { + iph = data; + if ((void *)(iph + 1) > data_end || + iph->protocol != IPPROTO_TCP || + iph->daddr != bpf_htonl(SERVER_IP)) + return SK_PASS; + + tcph = (void *)iph + iph->ihl * 4; + if ((void *)(tcph + 1) > data_end || + tcph->dest != bpf_htons(server_port)) + return SK_PASS; + + } else if (version == 6) { + ip6h = data; + if ((void *)(ip6h + 1) > data_end || + ip6h->nexthdr != IPPROTO_TCP) + return SK_PASS; + + if (ip6h->daddr.in6_u.u6_addr32[0] != 0 || + ip6h->daddr.in6_u.u6_addr32[1] != 0 || + ip6h->daddr.in6_u.u6_addr32[2] != 0 || + ip6h->daddr.in6_u.u6_addr32[3] != bpf_htonl(SERVER_IP6_LO)) + return SK_PASS; + + tcph = (void *)(ip6h + 1); + if ((void *)(tcph + 1) > data_end || + tcph->dest != bpf_htons(server_port)) + return SK_PASS; + } else { + return SK_PASS; + } + + kfunc_ret = bpf_icmp_send(skb, unreach_type, unreach_code); + + return SK_DROP; +} + +SEC("cgroup_skb/egress") +int recursion(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_end = (void *)(long)skb->data_end; + struct icmphdr *icmph; + struct tcphdr *tcph; + struct iphdr *iph; + int ret; + + if ((bpf_get_current_pid_tgid() >> 32) != target_pid) + return SK_PASS; + + iph = data; + if ((void *)(iph + 1) > data_end || iph->version != 4) + return SK_PASS; + + if (iph->daddr != bpf_htonl(SERVER_IP)) + return SK_PASS; + + if (iph->protocol == IPPROTO_TCP) { + tcph = (void *)iph + iph->ihl * 4; + if ((void *)(tcph + 1) > data_end || + tcph->dest != bpf_htons(server_port)) + return SK_PASS; + } else if (iph->protocol == IPPROTO_ICMP) { + icmph = (void *)iph + iph->ihl * 4; + if ((void *)(icmph + 1) > data_end || + icmph->type != unreach_type || icmph->code != unreach_code) + return SK_PASS; + } else { + return SK_PASS; + } + + /* + * This call will provoke a recursion: the ICMP packet generated by the + * kfunc will re-trigger this program since we are in the root cgroup in + * which the kernel ICMP socket belongs. However when re-entering the + * kfunc, it should return EBUSY. + */ + ret = bpf_icmp_send(skb, unreach_type, unreach_code); + rec_kfunc_rets[rec_count & 1] = ret; + __sync_fetch_and_add(&rec_count, 1); + + /* Let the first ICMP error message pass */ + if (iph->protocol == IPPROTO_ICMP) + return SK_PASS; + + return SK_DROP; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c index 0fa70b133d93..62d7df9e80be 100644 --- a/tools/testing/selftests/bpf/progs/iters.c +++ b/tools/testing/selftests/bpf/progs/iters.c @@ -88,6 +88,89 @@ int iter_err_unsafe_asm_loop(const void *ctx) return 0; } +/* + * Naked function, so there is no compiler-generated glue and the whole inlined program can be + * matched. Pinned to arches whose JITs zero-extend 32-bit writes implicitly + * (bpf_jit_needs_zext() == false); on arches that need explicit zero-extension the verifier + * interleaves "wN = wN" insns and the fixed shape below would not match. The inlining itself is + * arch independent, so checking it on these arches is sufficient. + * + * bpf_iter_num_new() emits the full range check (distance computation and both the -EINVAL and + * -E2BIG error paths); bpf_iter_num_next() and bpf_iter_num_destroy() are inlined too. + */ +SEC("raw_tp") +__arch_x86_64 +__arch_arm64 +__success +__xlated("r6 = r10") +__xlated("r6 += -8") +__xlated("call unknown") +__xlated("r3 = r0") +__xlated("r3 &= 65535") +__xlated("r1 = r6") +__xlated("r2 = 0") +/* bpf_iter_num_new(&it, 0, <non-const>) with the range check kept */ +__xlated("if w2 s> w3 goto pc+8") +__xlated("w0 = w3") +__xlated("w0 -= w2") +__xlated("if r0 > 0x800000 goto pc+8") +__xlated("w2 += -1") +__xlated("*(u32 *)(r1 +0) = r2") +__xlated("*(u32 *)(r1 +4) = r3") +__xlated("r0 = 0") +__xlated("goto pc+5") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = -22") +__xlated("goto pc+2") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = -7") +__xlated("r1 = r6") +/* bpf_iter_num_next(&it) */ +__xlated("r0 = *(u32 *)(r1 +0)") +__xlated("w0 += 1") +__xlated("r2 = *(u32 *)(r1 +4)") +__xlated("if w0 s>= w2 goto pc+3") +__xlated("*(u32 *)(r1 +0) = r0") +__xlated("r0 = r1") +__xlated("goto pc+2") +__xlated("*(u64 *)(r1 +0) = 0") +__xlated("r0 = 0") +__xlated("if r0 != 0x0 goto pc-11") +__xlated("r1 = r6") +/* bpf_iter_num_destroy(&it) is inlined to a nop */ +__xlated("goto pc+0") +__xlated("r0 = 0") +__xlated("exit") +int __naked iter_num_new_inlined(void) +{ + asm volatile ( + /* r6 points to struct bpf_iter_num on the stack */ + "r6 = r10;" + "r6 += -8;" + /* non-constant end so the range checks are kept */ + "call %[bpf_get_prandom_u32];" + "r3 = r0;" + "r3 &= 0xffff;" + "r1 = r6;" + "r2 = 0;" + "call %[bpf_iter_num_new];" + "1:" + "r1 = r6;" + "call %[bpf_iter_num_next];" + "if r0 != 0 goto 1b;" + "r1 = r6;" + "call %[bpf_iter_num_destroy];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_iter_num_new), + __imm(bpf_iter_num_next), + __imm(bpf_iter_num_destroy) + : __clobber_common, "r6" + ); +} + SEC("raw_tp") __success int iter_while_loop(const void *ctx) diff --git a/tools/testing/selftests/bpf/progs/iters_task_failure.c b/tools/testing/selftests/bpf/progs/iters_task_failure.c index fe3663dedbe1..566a1d3dffea 100644 --- a/tools/testing/selftests/bpf/progs/iters_task_failure.c +++ b/tools/testing/selftests/bpf/progs/iters_task_failure.c @@ -62,6 +62,52 @@ int BPF_PROG(iter_tasks_lock_and_unlock) } SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("expected an RCU CS when using bpf_iter_task_next") +__flag(BPF_F_TEST_STATE_FREQ) +int BPF_PROG(iter_tasks_rcu_state_pruning) +{ + struct bpf_iter_task it; + + asm volatile ( + "call %[bpf_rcu_read_lock];" + "r1 = %[it];" + "r2 = 0;" + "r3 = 0;" /* BPF_TASK_ITER_ALL_PROCS */ + "call %[bpf_iter_task_new];" + + "call %[bpf_get_prandom_u32];" + "if w0 == 0 goto unprotected_%=;" + + /* Keep the outer RCU lock active on the straight-line path. */ + "call %[bpf_rcu_read_lock];" + "call %[bpf_rcu_read_unlock];" + "goto merge_%=;" + + "unprotected_%=:" + /* Create an unprotected gap on the taken path. */ + "call %[bpf_rcu_read_unlock];" + "call %[bpf_rcu_read_lock];" + + "merge_%=: r1 = %[it];" + "call %[bpf_iter_task_next];" + "r1 = %[it];" + "call %[bpf_iter_task_destroy];" + "call %[bpf_rcu_read_unlock];" + : + : __imm_ptr(it), + __imm(bpf_get_prandom_u32), + __imm(bpf_iter_task_new), + __imm(bpf_iter_task_next), + __imm(bpf_iter_task_destroy), + __imm(bpf_rcu_read_lock), + __imm(bpf_rcu_read_unlock) + : __clobber_common + ); + + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("expected an RCU CS when using bpf_iter_css_next") int BPF_PROG(iter_css_lock_and_unlock) { diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c index a1963497f0bf..7e93f7fb1329 100644 --- a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c +++ b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c @@ -4,6 +4,18 @@ #include <bpf/bpf_helpers.h> #include "../test_kmods/bpf_testmod_kfunc.h" +static struct bpf_spin_lock kfunc_call_lock SEC(".data.A"); + +SEC("?tc") +int kfunc_call_test_spin_lock_unsafe(struct __sk_buff *skb) +{ + bpf_spin_lock(&kfunc_call_lock); + bpf_kfunc_trigger_ctx_check(); + bpf_spin_unlock(&kfunc_call_lock); + + return 0; +} + struct syscall_test_args { __u8 data[16]; size_t size; @@ -103,6 +115,66 @@ int kfunc_call_test_get_mem_fail_oob(struct __sk_buff *skb) return ret; } +SEC("?tc") +int kfunc_call_test_get_mem_fail_zero_size(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + /* + * An explicit rdwr_buf_size of 0 gives R0 a zero-sized buffer, + * so any access is out of bounds, hence -EACCES. Previously the + * verifier treated a zero size as "no size argument" and sized + * R0 after the pointed-to return type, wrongly allowing the read. + */ + p = bpf_kfunc_call_test_get_rdwr_mem(pt, 0); + if (p) + ret = p[0]; + else + ret = -1; + + bpf_kfunc_call_test_release(pt); + } + return ret; +} + +SEC("?tc") +int kfunc_call_test_get_mem_fail_oversized(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + /* + * rdwr_buf_size is a const int, so a C literal is narrowed to + * 32 bits before the call. Force the full 64-bit value 2^64 - 192 + * (0xffffffffffffff40, > U32_MAX) into the argument register with + * a 64-bit immediate load. The verifier records r0_size from the + * full register value and must reject it before that value is + * truncated into R0's u32 mem_size. + */ + asm volatile ( + "r1 = %[pt];" + "r2 = %[oversized] ll;" + "call %[get_rdwr_mem];" + "%[p] = r0;" + : [p] "=r"(p) + : [pt] "r"(pt), + [oversized] "i"(0xffffffffffffff40LL), + [get_rdwr_mem] "i"(bpf_kfunc_call_test_get_rdwr_mem) + : "r0", "r1", "r2", "r3", "r4", "r5"); + bpf_kfunc_call_test_release(pt); + } + return ret; +} + int not_const_size = 2 * sizeof(int); SEC("?tc") diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_test.c b/tools/testing/selftests/bpf/progs/kfunc_call_test.c index 5edc51564f71..8e6560c31e78 100644 --- a/tools/testing/selftests/bpf/progs/kfunc_call_test.c +++ b/tools/testing/selftests/bpf/progs/kfunc_call_test.c @@ -5,6 +5,18 @@ #include "bpf_misc.h" #include "../test_kmods/bpf_testmod_kfunc.h" +static struct bpf_spin_lock kfunc_call_lock SEC(".data.A"); + +SEC("tc") +int kfunc_call_test_spin_lock_safe(struct __sk_buff *skb) +{ + bpf_spin_lock(&kfunc_call_lock); + bpf_testmod_test_mod_kfunc(42); + bpf_spin_unlock(&kfunc_call_lock); + + return 0; +} + SEC("tc") int kfunc_call_test5(struct __sk_buff *skb) { diff --git a/tools/testing/selftests/bpf/progs/ksock_common.h b/tools/testing/selftests/bpf/progs/ksock_common.h new file mode 100644 index 000000000000..01edaeb9fdd4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_common.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Isovalent */ + +#ifndef _KSOCK_COMMON_H +#define _KSOCK_COMMON_H + +#include "errno.h" + +#define SOCK_DGRAM 2 +#define IPPROTO_UDP 17 + +struct bpf_ksock *bpf_ksock_create(const struct bpf_ksock_create_opts *opts, + u32 opts__sz, int *err__uninit) __ksym; +int bpf_ksock_connect(struct bpf_ksock *ks, const union bpf_ksock_addr *addr, + u32 addr__sz) __ksym; +struct bpf_ksock *bpf_ksock_acquire(struct bpf_ksock *ks) __ksym; +void bpf_ksock_release(struct bpf_ksock *ks) __ksym; +int bpf_ksock_send(struct bpf_ksock *ks, const void *data, u32 data__sz) __ksym; +void bpf_rcu_read_lock(void) __ksym; +void bpf_rcu_read_unlock(void) __ksym; + +struct __ksock_ctx_value { + struct bpf_ksock __kptr * ctx; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct __ksock_ctx_value); + __uint(max_entries, 1); +} __ksock_ctx_map SEC(".maps"); + +static inline struct __ksock_ctx_value *ksock_ctx_value_lookup(void) +{ + u32 key = 0; + + return bpf_map_lookup_elem(&__ksock_ctx_map, &key); +} + +static inline struct bpf_ksock *ksock_ctx_get(void) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *ks = NULL, *tmp; + + v = ksock_ctx_value_lookup(); + if (!v) + return NULL; + + bpf_rcu_read_lock(); + tmp = v->ctx; + if (tmp) + ks = bpf_ksock_acquire(tmp); + bpf_rcu_read_unlock(); + + return ks; +} + +static inline int ksock_ctx_insert(struct bpf_ksock *ctx) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *old; + + v = ksock_ctx_value_lookup(); + if (!v) { + bpf_ksock_release(ctx); + return -ENOENT; + } + + old = bpf_kptr_xchg(&v->ctx, ctx); + if (old) { + bpf_ksock_release(old); + return -EEXIST; + } + + return 0; +} + +#endif /* _KSOCK_COMMON_H */ diff --git a/tools/testing/selftests/bpf/progs/ksock_lsm.c b/tools/testing/selftests/bpf/progs/ksock_lsm.c new file mode 100644 index 000000000000..9808451098ef --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_lsm.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_endian.h> +#include "bpf_tracing_net.h" +#include "ksock_common.h" + +char send_data[32] = "hello from bpf ksock"; + +__be32 ipv4_remote; +__u16 remote_port; +int target_pid; +int send_ret = -1; + +SEC("syscall") +int ksock_setup(void *ctx) +{ + struct bpf_ksock_create_opts create_opts = {}; + union bpf_ksock_addr addr = {}; + struct bpf_ksock *ks; + int err = 0; + + create_opts.family = AF_INET; + create_opts.type = SOCK_DGRAM; + create_opts.protocol = IPPROTO_UDP; + + ks = bpf_ksock_create(&create_opts, sizeof(create_opts), &err); + if (!ks) + return err; + + addr.sin.sin_family = AF_INET; + addr.sin.sin_port = bpf_htons(remote_port); + addr.sin.sin_addr.s_addr = ipv4_remote; + + err = bpf_ksock_connect(ks, &addr, sizeof(addr)); + if (err) { + bpf_ksock_release(ks); + return err; + } + + err = ksock_ctx_insert(ks); + if (err && err != -EEXIST) + return err; + return 0; +} + +SEC("lsm.s/socket_bind") +int BPF_PROG(ksock_socket_bind, struct socket *sock, struct sockaddr *address, + int addrlen, int ret) +{ + struct bpf_ksock *ks; + u32 pid = bpf_get_current_pid_tgid() >> 32; + + if (ret || pid != target_pid) + return ret; + + ks = ksock_ctx_get(); + if (!ks) { + send_ret = -ENOENT; + return ret; + } + + send_ret = bpf_ksock_send(ks, send_data, sizeof(send_data)); + bpf_ksock_release(ks); + + return ret; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c b/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c new file mode 100644 index 000000000000..fd2ccfdb5802 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_lsm_verifier.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include "bpf_misc.h" +#include "ksock_common.h" + +char send_data[11] = "dummy data"; + +SEC("lsm.s/socket_sendmsg") +__description("bpf_ksock_send is rejected from socket_sendmsg LSM hook") +__failure __msg("calling kernel function bpf_ksock_send is not allowed") +int BPF_PROG(ksock_socket_sendmsg, struct socket *sock, struct msghdr *msg, + int size, int ret) +{ + struct __ksock_ctx_value *v; + struct bpf_ksock *ks; + + v = ksock_ctx_value_lookup(); + if (!v) + return ret; + + ks = bpf_kptr_xchg(&v->ctx, NULL); + if (!ks) + return ret; + + bpf_ksock_send(ks, send_data, sizeof(send_data)); + bpf_ksock_release(ks); + + return ret; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/ksock_wq.c b/tools/testing/selftests/bpf/progs/ksock_wq.c new file mode 100644 index 000000000000..16a1873d132e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/ksock_wq.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Isovalent */ + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include "bpf_experimental.h" +#include "bpf_tracing_net.h" +#include "errno.h" +#include "ksock_common.h" + +struct ksock_wq_value { + struct bpf_wq work; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct ksock_wq_value); +} work_map SEC(".maps"); + +int create_err; +u32 callback_done; + +static int ksock_wq_callback(void *map, int *key, void *value) +{ + struct bpf_ksock_create_opts opts = { + .family = AF_INET, + .type = SOCK_DGRAM, + .protocol = IPPROTO_UDP, + }; + struct bpf_ksock *ks; + int err = 0; + + ks = bpf_ksock_create(&opts, sizeof(opts), &err); + if (ks) + bpf_ksock_release(ks); + create_err = err; + __sync_fetch_and_add(&callback_done, 1); + return 0; +} + +SEC("syscall") +int ksock_wq_start(void *ctx) +{ + struct ksock_wq_value *value; + u32 key = 0; + int err; + + value = bpf_map_lookup_elem(&work_map, &key); + if (!value) + return -ENOENT; + err = bpf_wq_init(&value->work, &work_map, 0); + if (err) + return err; + err = bpf_wq_set_callback(&value->work, ksock_wq_callback, 0); + if (err) + return err; + return bpf_wq_start(&value->work, 0); +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/linked_list_fail.c b/tools/testing/selftests/bpf/progs/linked_list_fail.c index ddd26d1a083f..031e77a288ee 100644 --- a/tools/testing/selftests/bpf/progs/linked_list_fail.c +++ b/tools/testing/selftests/bpf/progs/linked_list_fail.c @@ -167,6 +167,16 @@ CHECK_OP(push_back); #undef CHECK_OP #undef INIT +struct obj_new_flex_elem { + int lo; + int hi; +}; + +struct obj_new_flex { + int hdr; + struct obj_new_flex_elem cells[]; +}; + SEC("?kprobe/xyz") int map_compat_kprobe(void *ctx) { @@ -231,6 +241,19 @@ int obj_new_no_struct(void *ctx) } SEC("?tc") +int obj_new_flex_array(void *ctx) +{ + struct obj_new_flex *p; + + p = bpf_obj_new_impl(bpf_core_type_id_local(struct obj_new_flex), NULL); + if (!p) + return 0; + p->cells[0].hi = 42; + bpf_obj_drop_impl(p, NULL); + return 0; +} + +SEC("?tc") int obj_drop_non_zero_off(void *ctx) { void *f; diff --git a/tools/testing/selftests/bpf/progs/map_kptr.c b/tools/testing/selftests/bpf/progs/map_kptr.c index 3fbefc568e0a..0d87c97dac99 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr.c +++ b/tools/testing/selftests/bpf/progs/map_kptr.c @@ -4,6 +4,18 @@ #include <bpf/bpf_helpers.h> #include "../test_kmods/bpf_testmod_kfunc.h" +struct map_uninit_value { + struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr; + __u32 data; +} __attribute__((packed)); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, int); + __type(value, struct map_uninit_value); + __uint(max_entries, 1); +} pcpu_array SEC(".maps"); + struct map_value { struct prog_test_ref_kfunc __kptr_untrusted *unref_ptr; struct prog_test_ref_kfunc __kptr *ref_ptr; diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c index f11848dfa78f..5e25ca806060 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c @@ -386,7 +386,16 @@ int kptr_xchg_possibly_null(struct __sk_buff *ctx) } SEC("?tc") +/* + * A compiler with BPF_ST folds the constant into a store-immediate, which the + * verifier rejects on a different path (and with a different message) than the + * BPF_STX form. + */ +#ifdef __BPF_FEATURE_ST +__failure __msg("BPF_ST imm must be 0 when storing to kptr at off=8") +#else __failure __msg("invalid kptr access, R") +#endif int reject_scalar_store_to_kptr(struct __sk_buff *ctx) { struct map_value *v; diff --git a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c index f166fff8f217..3e0d4f687aaa 100644 --- a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c +++ b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c @@ -3,6 +3,7 @@ #include <vmlinux.h> #include <bpf/bpf_core_read.h> #include "bpf_misc.h" +#include "bpf_kfuncs.h" #include "../test_kmods/bpf_testmod_kfunc.h" SEC("tp_btf/sys_enter") @@ -137,7 +138,7 @@ int helper_param_not_ok(void *ctx) p = bpf_rdonly_cast(0, 0); /* - * Any helper with ARG_CONST_SIZE_OR_ZERO constraint will do, + * Any helper with ARG_MEM_SIZE_OR_ZERO constraint will do, * the most permissive constraint */ bpf_copy_from_user(p, 0, (void *)42); @@ -164,6 +165,239 @@ int mixed_mem_type(void *ctx) return *p; } +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 4096); +} ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, u64); +} array SEC(".maps"); + +char dynptr_data[8]; + +int zero; + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=untrusted_ptr_sock") +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=ringbuf_mem") +__retval(0) +int mixed_mem_untrusted_btf_id_type(void *ctx) +{ + u64 *p, *q, v; + + p = bpf_ringbuf_reserve(&ringbuf, sizeof(*p), 0); + if (!p) + return 1; + *p = 42; + q = bpf_rdonly_cast(0, bpf_core_type_id_kernel(struct sock)); + /* + * The load below is reached with PTR_TO_MEM | MEM_RINGBUF on one + * path and with PTR_TO_BTF_ID | PTR_UNTRUSTED on the other. The + * merged type has to keep the BPF_PROBE_MEM rewrite, otherwise + * the NULL deref taken at runtime panics the kernel instead of + * returning 0. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] != 0 goto +1;" + "r7 = %[q];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + bpf_ringbuf_discard(p, 0); + return v; +} + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=ptr_nameidata") +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=ringbuf_mem") +__retval(0) +int mixed_mem_btf_id_type(void *ctx) +{ + struct task_struct *task; + u32 *p, *q; + u64 v; + + p = bpf_ringbuf_reserve(&ringbuf, sizeof(*p), 0); + if (!p) + return 1; + *p = 42; + task = bpf_get_current_task_btf(); + /* + * A plain BTF pointer walk yields a bare PTR_TO_BTF_ID, and + * task->nameidata is NULL unless the task currently is in the + * middle of a path lookup. + */ + q = (u32 *)&task->nameidata->flags; + /* + * Same as above, except that the other path yields a bare + * PTR_TO_BTF_ID. Merging it with PTR_TO_MEM used to drop the + * BPF_PROBE_MEM rewrite the bare PTR_TO_BTF_ID would have + * gotten on its own. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] != 0 goto +1;" + "r7 = %[q];" + "r8 = *(u32 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + bpf_ringbuf_discard(p, 0); + return v; +} + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=ptr_nameidata") +__msg("r8 = *(u32 *)(r7 +0){{.*}}R7=rdonly_mem") +__retval(0) +int mixed_rdonly_mem_btf_id_type(void *ctx) +{ + struct task_struct *task; + struct bpf_dynptr dptr; + char buf[sizeof(u32)]; + u32 *p, *q; + u64 v; + + if (bpf_dynptr_from_mem(dynptr_data, sizeof(dynptr_data), 0, &dptr)) + return 1; + p = bpf_dynptr_slice(&dptr, 0, buf, sizeof(buf)); + if (!p) + return 1; + task = bpf_get_current_task_btf(); + q = (u32 *)&task->nameidata->flags; + /* + * Same as above, except that the PTR_TO_MEM side already carries + * MEM_RDONLY. Merging it with a bare PTR_TO_BTF_ID used to yield + * PTR_TO_MEM | MEM_RDONLY, which is not rewritten either since + * only its PTR_UNTRUSTED variant is. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] != 0 goto +1;" + "r7 = %[q];" + "r8 = *(u32 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + return v; +} + +SEC("socket") +__success +__log_level(2) +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=ringbuf_mem") +__msg("r8 = *(u64 *)(r7 +0){{.*}}R7=rdonly_untrusted_mem") +__retval(0) +int mixed_mem_mem_type(void *ctx) +{ + u64 *p, *q, v; + + p = bpf_ringbuf_reserve(&ringbuf, sizeof(*p), 0); + if (!p) + return 1; + *p = 42; + q = bpf_rdonly_cast(0, 0); + /* + * Both paths are PTR_TO_MEM based, so they used to not trip the + * type mismatch check and skipped the merge altogether, leaving + * the insn with the PTR_TO_MEM | MEM_RINGBUF recorded first and + * hence without the BPF_PROBE_MEM rewrite the other path needs. + */ + asm volatile ( + "r7 = %[q];" + "if %[zero] == 0 goto +1;" + "r7 = %[p];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + bpf_ringbuf_discard(p, 0); + return v; +} + +SEC("socket") +__failure +__msg("same insn cannot be used with different pointers") +int mixed_map_value_mem_type(void *ctx) +{ + u64 *p, *q, v; + u32 key = 0; + + p = bpf_map_lookup_elem(&array, &key); + if (!p) + return 1; + q = bpf_rdonly_cast(0, 0); + /* + * PTR_TO_MAP_VALUE is neither PTR_TO_MEM nor PTR_TO_BTF_ID based, + * so it cannot be merged into a type which keeps the BPF_PROBE_MEM + * rewrite the PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED of the other + * path needs. Both bases were mismatch ok, hence the load used to be + * accepted with the PTR_TO_MAP_VALUE recorded and the NULL deref on + * the second path panicked the kernel. + */ + asm volatile ( + "r7 = %[q];" + "if %[zero] == 0 goto +1;" + "r7 = %[p];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [q]"r"(q), + [zero]"r"(zero) + : "r7", "r8"); + return v; +} + +SEC("socket") +__failure +__msg("same insn cannot be used with different pointers") +int mixed_stack_mem_type(void *ctx) +{ + u64 *p = bpf_rdonly_cast(0, 0); + u64 s = 42, v; + + /* + * Same as above, but for a PTR_TO_STACK on the other path. + */ + asm volatile ( + "r7 = %[p];" + "if %[zero] == 0 goto +1;" + "r7 = %[s];" + "r8 = *(u64 *)(r7 + 0);" + "%[v] = r8;" + : [v]"=r"(v) + : [p]"r"(p), + [s]"r"(&s), + [zero]"r"(zero) + : "r7", "r8"); + return v; +} + __attribute__((__aligned__(8))) u8 global[] = { 0x11, 0x22, 0x33, 0x44, diff --git a/tools/testing/selftests/bpf/progs/mptcpify.c b/tools/testing/selftests/bpf/progs/mptcpify.c index cbdc730c3a47..e3f8cb54dbe9 100644 --- a/tools/testing/selftests/bpf/progs/mptcpify.c +++ b/tools/testing/selftests/bpf/progs/mptcpify.c @@ -15,7 +15,7 @@ int BPF_PROG(mptcpify, int family, int type, int protocol) return protocol; if ((family == AF_INET || family == AF_INET6) && - type == SOCK_STREAM && + (type & SOCK_TYPE_MASK) == SOCK_STREAM && (!protocol || protocol == IPPROTO_TCP)) { return IPPROTO_MPTCP; } diff --git a/tools/testing/selftests/bpf/progs/pyperf.h b/tools/testing/selftests/bpf/progs/pyperf.h index 86484f07e1d1..fd93a96e5901 100644 --- a/tools/testing/selftests/bpf/progs/pyperf.h +++ b/tools/testing/selftests/bpf/progs/pyperf.h @@ -85,9 +85,11 @@ static void *get_thread_state(void *tls_base, PidData *pidData) return thread_state; } -static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, - FrameData *frame, Symbol *symbol) +__weak bool __get_frame_data(long frame_ptr_, PidData *pidData __arg_nonnull, + FrameData *frame __arg_nonnull, Symbol *symbol __arg_nonnull) { + void *frame_ptr = (void *)frame_ptr_; + // read data from PyFrameObject bpf_probe_read_user(&frame->f_back, sizeof(frame->f_back), @@ -119,6 +121,12 @@ static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, return true; } +static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData, + FrameData *frame, Symbol *symbol) +{ + return __get_frame_data((long)frame_ptr, pidData, frame, symbol); +} + struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, 1); diff --git a/tools/testing/selftests/bpf/progs/rcu_read_lock.c b/tools/testing/selftests/bpf/progs/rcu_read_lock.c index b4e073168fb1..31d4081c3a9f 100644 --- a/tools/testing/selftests/bpf/progs/rcu_read_lock.c +++ b/tools/testing/selftests/bpf/progs/rcu_read_lock.c @@ -549,3 +549,79 @@ int rcu_read_lock_sleepable_global_subprog_indirect(void *ctx) bpf_rcu_read_unlock(); return 0; } + +struct rcu_node_data { + long key; + struct bpf_rb_node node; +}; + +struct rcu_node_stash { + struct rcu_node_data __kptr *node; +}; + +/* + * Necessary so that LLVM emits BTF for rcu_node_data rather than just a + * fwd reference to it, same as in progs/local_kptr_stash.c. + */ +struct rcu_node_data *just_here_because_btf_bug; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct rcu_node_stash); +} node_stash SEC(".maps"); + +long non_own_ref_key; + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +int non_own_ref_untrusted_ld(void *ctx) +{ + struct rcu_node_stash *stash; + struct rcu_node_data *node; + int key = 0; + + stash = bpf_map_lookup_elem(&node_stash, &key); + if (!stash) + return 0; + bpf_rcu_read_lock(); + node = stash->node; + if (!node) { + bpf_rcu_read_unlock(); + return 0; + } + bpf_rcu_read_unlock(); + /* + * The unlock leaves node as PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED + * | NON_OWN_REF, and the load below has to get the BPF_PROBE_MEM + * rewrite for it, otherwise a bad address panics the kernel. + */ + non_own_ref_key = node->key; + return 0; +} + +long rcu_untrusted_wq_flags; + +SEC("?tp_btf/tcp_probe") +int BPF_PROG(rcu_untrusted_union_ld, struct sock *sk) +{ + struct socket_wq *wq; + + /* + * sk_wq sits in a two member union, so btf_struct_walk() marks the + * pointer PTR_UNTRUSTED, and the __rcu tag on the member adds MEM_RCU + * on top of it. struct sock is not on the __safe_rcu_or_null allow + * list, hence the two stay combined and the load below has to get the + * BPF_PROBE_MEM rewrite for PTR_TO_BTF_ID | PTR_UNTRUSTED | MEM_RCU, + * otherwise a bad address panics the kernel. + * + * The __rcu tag only reaches BTF on a clang built kernel, that is, one + * with CONFIG_PAHOLE_HAS_BTF_TAG. On a gcc built kernel the walk yields + * a plain untrusted pointer, which is rewritten either way. + */ + wq = sk->sk_wq; + if (!wq) + return 0; + rcu_untrusted_wq_flags = wq->flags; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c index 024ef2aae200..eaaed0859f94 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c @@ -63,6 +63,7 @@ long rbtree_refcounted_node_ref_escapes(void *ctx) SEC("?tc") __failure __msg("Possibly NULL pointer passed to trusted R1") +__msg("requires a non-NULL value of type (void *)") long refcount_acquire_maybe_null(void *ctx) { struct node_acquire *n, *m; @@ -81,6 +82,14 @@ long refcount_acquire_maybe_null(void *ctx) } SEC("?tc") +__failure __msg("R1 is neither owning or non-owning ref") +__msg("expects a pointer to a BPF-managed refcounted object, but R1 is a context pointer") +long refcount_acquire_non_object(void *ctx) +{ + return bpf_refcount_acquire(ctx) != NULL; +} + +SEC("?tc") __failure __msg("Unreleased reference id=3 alloc_insn={{[0-9]+}}") long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/setget_sockopt.c b/tools/testing/selftests/bpf/progs/setget_sockopt.c index 636a7cd8e2fa..d96e99b67aeb 100644 --- a/tools/testing/selftests/bpf/progs/setget_sockopt.c +++ b/tools/testing/selftests/bpf/progs/setget_sockopt.c @@ -69,12 +69,14 @@ static const struct sockopt_test sol_tcp_tests[] = { static const struct sockopt_test sol_ip_tests[] = { { .opt = IP_TOS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, }, + { .opt = IP_TRANSPARENT, .flip = 1, }, { .opt = 0, }, }; static const struct sockopt_test sol_ipv6_tests[] = { { .opt = IPV6_TCLASS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, }, { .opt = IPV6_AUTOFLOWLABEL, .flip = 1, }, + { .opt = IPV6_TRANSPARENT, .flip = 1, }, { .opt = 0, }, }; diff --git a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c index 0660f29dca95..3177bc5b733a 100644 --- a/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c +++ b/tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c @@ -44,8 +44,18 @@ int bpf_prog2(struct __sk_buff *skb) __sink(lport); __sink(rport); - if (data + 8 > data_end) - return SK_DROP; + if (data + 8 > data_end) { + if (bpf_skb_pull_data(skb, 8)) + return SK_DROP; + + data = (void *)(long)skb->data; + data_end = (void *)(long)skb->data_end; + + if (data + 8 > data_end) + return SK_DROP; + + d = data; + } map = d[0]; sk = d[1]; diff --git a/tools/testing/selftests/bpf/progs/stream.c b/tools/testing/selftests/bpf/progs/stream.c index 92ba1d72e0ec..8e8e1339dc74 100644 --- a/tools/testing/selftests/bpf/progs/stream.c +++ b/tools/testing/selftests/bpf/progs/stream.c @@ -64,6 +64,8 @@ SEC("syscall") __arch_x86_64 __arch_arm64 __arch_s390x +__arch_riscv64 +__arch_loongarch __success __retval(0) __stderr("ERROR: Timeout detected for may_goto instruction") __stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") @@ -184,6 +186,151 @@ int stream_arena_read_fault(void *ctx) return 0; } +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +__stderr("ERROR: Arena READ access at unmapped address 0x{{.*}}") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +int stream_arena_load_acquire_fault(void *ctx) +{ + static const struct bpf_insn load_acquire_insn = { + .code = 0xc3, /* BPF_STX | BPF_ATOMIC | BPF_W */ + .dst_reg = 0, /* BPF_REG_0 */ + .src_reg = 1, /* BPF_REG_1 */ + .off = 0x7fff, + .imm = 0x100, /* BPF_LOAD_ACQ */ + }; + struct bpf_arena *ptr = (void *)&arena; + u64 user_vm_start, val; + + /* + * Prevent GCC bounds warning: casting &arena to struct bpf_arena * + * triggers bounds checking since the map definition is smaller than + * struct bpf_arena. barrier_var() makes the pointer opaque to GCC, + * preventing the bounds analysis. + */ + barrier_var(ptr); + user_vm_start = ptr->user_vm_start; + fault_addr = user_vm_start + 0x7fff; + bpf_addr_space_cast(user_vm_start, 0, 1); + asm volatile ( + "r1 = %[user_vm_start];" + "r0 = 1;" + ".8byte %[load_acquire_insn];" /* r0 = load_acquire((u32 *)(r1 + 0x7fff)) */ + "%[val] = r0;" + : [val] "=r" (val) + : [user_vm_start] "r" (user_vm_start), + __imm_insn(load_acquire_insn, load_acquire_insn) + : "r0", "r1" + ); + return val; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +__stderr("ERROR: Arena WRITE access at unmapped address 0x{{.*}}") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +int stream_arena_xchg_fault(void *ctx) +{ + static const struct bpf_insn xchg_insn = { + .code = 0xc3, /* BPF_STX | BPF_ATOMIC | BPF_W */ + .dst_reg = 1, /* BPF_REG_1 */ + .src_reg = 2, /* BPF_REG_2 */ + .off = 0x7fff, + .imm = 0xe1, /* BPF_XCHG */ + }; + struct bpf_arena *ptr = (void *)&arena; + u64 user_vm_start, val; + + /* + * Prevent GCC bounds warning: casting &arena to struct bpf_arena * + * triggers bounds checking since the map definition is smaller than + * struct bpf_arena. barrier_var() makes the pointer opaque to GCC, + * preventing the bounds analysis. + */ + barrier_var(ptr); + user_vm_start = ptr->user_vm_start; + fault_addr = user_vm_start + 0x7fff; + bpf_addr_space_cast(user_vm_start, 0, 1); + /* + * A read-modify-write carrying BPF_FETCH writes to memory, so the fault + * has to be reported as a WRITE from the dst_reg address, but it also + * reads the old value into src_reg, so the exception handler has to + * clear src_reg. Poison it up front, the returned value must be 0. + */ + asm volatile ( + "r1 = %[user_vm_start];" + "r2 = 1;" + ".8byte %[xchg_insn];" /* r2 = xchg((u32 *)(r1 + 0x7fff), r2) */ + "%[val] = r2;" + : [val] "=r" (val) + : [user_vm_start] "r" (user_vm_start), + __imm_insn(xchg_insn, xchg_insn) + : "r1", "r2" + ); + return val; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +__stderr("ERROR: Arena WRITE access at unmapped address 0x{{.*}}") +__stderr("CPU: {{[0-9]+}} UID: 0 PID: {{[0-9]+}} Comm: {{.*}}") +__stderr("Call trace:\n" +"{{([a-zA-Z_][a-zA-Z0-9_]*\\+0x[0-9a-fA-F]+/0x[0-9a-fA-F]+\n" +"|[ \t]+[^\n]+\n)*}}") +int stream_arena_cmpxchg_fault(void *ctx) +{ + static const struct bpf_insn cmpxchg_insn = { + .code = 0xc3, /* BPF_STX | BPF_ATOMIC | BPF_W */ + .dst_reg = 1, /* BPF_REG_1 */ + .src_reg = 2, /* BPF_REG_2 */ + .off = 0x7fff, + .imm = 0xf1, /* BPF_CMPXCHG */ + }; + struct bpf_arena *ptr = (void *)&arena; + u64 user_vm_start, val; + + /* + * Prevent GCC bounds warning: casting &arena to struct bpf_arena * + * triggers bounds checking since the map definition is smaller than + * struct bpf_arena. barrier_var() makes the pointer opaque to GCC, + * preventing the bounds analysis. + */ + barrier_var(ptr); + user_vm_start = ptr->user_vm_start; + fault_addr = user_vm_start + 0x7fff; + bpf_addr_space_cast(user_vm_start, 0, 1); + /* + * Same as the exchange above, except that a BPF_CMPXCHG reads the old + * value into r0 rather than into src_reg, so r0 is the register the + * exception handler has to clear. It doubles as the compare value, but + * the comparison never happens since the access faults first. + */ + asm volatile ( + "r1 = %[user_vm_start];" + "r0 = 1;" + "r2 = 2;" + ".8byte %[cmpxchg_insn];" /* r0 = cmpxchg((u32 *)(r1 + 0x7fff), r0, r2) */ + "%[val] = r0;" + : [val] "=r" (val) + : [user_vm_start] "r" (user_vm_start), + __imm_insn(cmpxchg_insn, cmpxchg_insn) + : "r0", "r1", "r2" + ); + return val; +} + static __noinline void subprog(void) { int __arena *addr = (int __arena *)0xdeadbeef; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena.c b/tools/testing/selftests/bpf/progs/struct_ops_arena.c new file mode 100644 index 000000000000..8aa8639df91f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_experimental.h" +#include <bpf_arena_common.h> +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* page 0 hosts the arena globals, page 1 is for allocations */ + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callbacks with the arena */ +u64 __arena arena_touch; +/* raw value of the last __arena ctx argument, captured by test_arena_cb */ +u64 __arena cb_ptr_val; + +SEC("struct_ops/test_arena") +int test_arena_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[0]; + + arena_touch++; + cb_ptr_val = ctx[0]; + *ptr += 1; + return 0; +} + +SEC("struct_ops/test_arena_nullable") +int test_arena_nullable_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[0]; + + arena_touch++; + if (!ptr) + return 0xbee; + *ptr += 1; + return 0; +} + +SEC("struct_ops/test_arena_stack") +int test_arena_stack_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[8]; + + arena_touch++; + /* pin the slot layout: the leading args fill ctx[0]..ctx[7] */ + if (ctx[0] != 1 || ctx[7] != 8) + return 0xbad; + *ptr += 1; + return 0; +} + +SEC("struct_ops/test_arena_multislot") +int test_arena_multislot_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[2]; + + arena_touch++; + /* + * The 16-byte struct occupies ctx[0] and ctx[1], so @ptr is argument + * one but slot two. Getting that wrong hands the callback a scalar. + */ + if (ctx[0] != 11 || ctx[1] != 22) + return 0xbad; + *ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena = { + .test_arena = (void *)test_arena_cb, + .test_arena_nullable = (void *)test_arena_nullable_cb, + .test_arena_stack = (void *)test_arena_stack_cb, + .test_arena_multislot = (void *)test_arena_multislot_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena((u64 *)val); + if (ret) + return 2; + if (*val != 42) + return 3; + + /* + * The callback must have seen exactly (u32)(kaddr - kern_vm_start), + * which is the arena offset of val with the upper 32 bits clear. + */ + if (cb_ptr_val != (u32)(u64)val) + return 4; + + ret = bpf_testmod_ops3_call_test_arena_nullable((u64 *)val); + if (ret) + return 5; + if (*val != 43) + return 6; + + /* NULL survives the nullable kfunc and the trampoline as NULL */ + ret = bpf_testmod_ops3_call_test_arena_nullable(NULL); + if (ret != 0xbee) + return 7; + + /* the arena pointer is stack-passed into the trampoline here */ + ret = bpf_testmod_ops3_call_test_arena_stack((u64 *)val); + if (ret) + return 8; + if (*val != 44) + return 9; + + /* a multi-slot arg precedes the arena pointer here */ + ret = bpf_testmod_ops3_call_test_arena_multislot((u64 *)val); + if (ret) + return 10; + if (*val != 45) + return 11; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_attach.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_attach.c new file mode 100644 index 000000000000..081a770307e5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_attach.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +SEC("fentry") +int BPF_PROG(fentry_test_arena, unsigned long long *st_ops_ctx) +{ + return 0; +} + +SEC("fexit") +int BPF_PROG(fexit_test_arena, unsigned long long *st_ops_ctx, int ret) +{ + return 0; +} + +SEC("freplace") +int freplace_test_arena(unsigned long long *st_ops_ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_fail.c new file mode 100644 index 000000000000..1c0ec727d637 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_fail.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* No arena in the program: attaching to test_arena must be rejected. */ +SEC("struct_ops/test_arena") +int test_arena_no_arena(unsigned long long *ctx) +{ + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_fail = { + .test_arena = (void *)test_arena_no_arena, +}; diff --git a/tools/testing/selftests/bpf/progs/tailcall_callback.c b/tools/testing/selftests/bpf/progs/tailcall_callback.c new file mode 100644 index 000000000000..c41632cf423b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_callback.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_test_utils.h" + +int classifier_0(struct __sk_buff *skb); + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __array(values, void (void)); +} jmp_table SEC(".maps") = { + .values = { + [0] = (void *) &classifier_0, + }, +}; + +__auxiliary +SEC("tc") +int classifier_0(struct __sk_buff *skb) +{ + return 0; +} + +static __noinline +int subprog_tail0(struct __sk_buff *skb) +{ + int ret = 0; + + bpf_tail_call_static(skb, &jmp_table, 0); + barrier_var(ret); + return ret; +} + +static __noinline +int callback_loop(int index, void **cb_ctx) +{ + int ret; + + ret = subprog_tail0(*cb_ctx); + barrier_var(ret); + return ret ? 1 : 0; +} + +static __noinline +int callback_empty(int index, void *data) +{ + return 0; +} + +/* callback involving subprog with tail call is rejected */ +SEC("tc") +__failure __msg("cannot tail call within callback") +int tailcall_callback_1(struct __sk_buff *skb) +{ + clobber_regs_stack(); + + bpf_loop(1, callback_loop, &skb, 0); + return 0; +} + +/* subprogs with tailcall do not affect no-tailcall callback */ +SEC("tc") +__success +__retval(0) +int tailcall_callback_2(struct __sk_buff *skb) +{ + int ret; + + clobber_regs_stack(); + + ret = subprog_tail0(skb); + __sink(ret); + + bpf_loop(1, callback_empty, NULL, 0); + return 0; +} + +char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_common.h b/tools/testing/selftests/bpf/progs/task_kfunc_common.h index e9c4fea7a4bb..a0c599b58c29 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_common.h +++ b/tools/testing/selftests/bpf/progs/task_kfunc_common.h @@ -20,12 +20,26 @@ struct { __uint(max_entries, 1); } __tasks_kfunc_map SEC(".maps"); +struct task_kptr_lock_value { + struct bpf_spin_lock lock; + struct task_struct __kptr * task; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct task_kptr_lock_value); + __uint(max_entries, 1); +} task_kptr_lock_map SEC(".maps"); + struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; void bpf_task_release(struct task_struct *p) __ksym; struct task_struct *bpf_task_from_pid(s32 pid) __ksym; struct task_struct *bpf_task_from_vpid(s32 vpid) __ksym; void bpf_rcu_read_lock(void) __ksym; void bpf_rcu_read_unlock(void) __ksym; +void bpf_local_irq_save(unsigned long *flags) __weak __ksym; +void bpf_local_irq_restore(unsigned long *flags) __weak __ksym; static inline struct __tasks_kfunc_map_value *tasks_kfunc_map_value_lookup(struct task_struct *p) { diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c index 8942b5478129..f96b0c13ed1a 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_failure.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_failure.c @@ -50,7 +50,7 @@ int BPF_PROG(task_kfunc_acquire_untrusted, struct task_struct *task, u64 clone_f } SEC("tp_btf/task_newtask") -__failure __msg("R1 pointer type STRUCT task_struct must point") +__failure __msg("R1 is fp expected STRUCT task_struct") int BPF_PROG(task_kfunc_acquire_fp, struct task_struct *task, u64 clone_flags) { struct task_struct *acquired, *stack_task = (struct task_struct *)&clone_flags; @@ -378,3 +378,76 @@ int BPF_PROG(task_kfunc_release_in_map, struct task_struct *task, u64 clone_flag return 0; } + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("R1 must be a rcu pointer") +int BPF_PROG(task_kfunc_acquire_after_final_spin_unlock) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("R1 must be a rcu pointer") +int BPF_PROG(task_kfunc_acquire_after_preempt_enable) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + task = v->task; + bpf_preempt_enable(); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__failure __msg("R1 must be a rcu pointer") +int BPF_PROG(task_kfunc_acquire_after_irq_restore) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + task = v->task; + bpf_local_irq_restore(&flags); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/task_kfunc_success.c b/tools/testing/selftests/bpf/progs/task_kfunc_success.c index d63a79ee33dc..6545b124dee1 100644 --- a/tools/testing/selftests/bpf/progs/task_kfunc_success.c +++ b/tools/testing/selftests/bpf/progs/task_kfunc_success.c @@ -6,6 +6,7 @@ #include <bpf/bpf_helpers.h> #include "../bpf_experimental.h" +#include "bpf_misc.h" #include "task_kfunc_common.h" char _license[] SEC("license") = "GPL"; @@ -366,6 +367,200 @@ int BPF_PROG(task_kfunc_acquire_trusted_walked, struct task_struct *task, u64 cl return 0; } +SEC("fentry/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_non_sleepable) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (!task) + return 0; + + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_explicit_rcu) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_rcu_read_lock(); + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_rcu_read_unlock(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_preempt_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_preempt_enable(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_spin_unlock_irq_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + bpf_spin_lock(&v->lock); + task = v->task; + bpf_spin_unlock(&v->lock); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_local_irq_restore(&flags); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_rcu_unlock_preempt_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + bpf_rcu_read_lock(); + task = v->task; + bpf_rcu_read_unlock(); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_preempt_enable(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_rcu_unlock_irq_disabled) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + bpf_rcu_read_lock(); + task = v->task; + bpf_rcu_read_unlock(); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_local_irq_restore(&flags); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_preempt_enable_explicit_rcu) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_preempt_disable(); + task = v->task; + bpf_rcu_read_lock(); + bpf_preempt_enable(); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_rcu_read_unlock(); + return 0; +} + +SEC("fentry.s/" SYS_PREFIX "sys_getpgid") +int BPF_PROG(task_kfunc_acquire_after_irq_restore_explicit_rcu) +{ + struct task_kptr_lock_value *v; + struct task_struct *task, *acquired; + unsigned long flags; + int key = 0; + + v = bpf_map_lookup_elem(&task_kptr_lock_map, &key); + if (!v) + return 0; + + bpf_local_irq_save(&flags); + task = v->task; + bpf_rcu_read_lock(); + bpf_local_irq_restore(&flags); + if (task) { + acquired = bpf_task_acquire(task); + if (acquired) + bpf_task_release(acquired); + } + bpf_rcu_read_unlock(); + return 0; +} + SEC("syscall") int test_task_from_vpid_current(const void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c b/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c new file mode 100644 index 000000000000..eda4697aac80 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tcp_ca_untrusted_btf_write.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "bpf_tracing_net.h" +#include <bpf/bpf_core_read.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +SEC("struct_ops") +void BPF_PROG(untrusted_btf_write_init, struct sock *sk) +{ + struct tcp_sock *tp; + int v = 1; + void *p; + + p = bpf_rdonly_cast(&v, 0); + tp = bpf_rdonly_cast(p, bpf_core_type_id_kernel(struct tcp_sock)); + tp->snd_cwnd = 1; +} + +SEC(".struct_ops") +struct tcp_congestion_ops untrusted_btf_write = { + .init = (void *)untrusted_btf_write_init, + .name = "bpf_ro_btf", +}; diff --git a/tools/testing/selftests/bpf/progs/test_fill_link_info.c b/tools/testing/selftests/bpf/progs/test_fill_link_info.c index 137bd6292163..c85081538e93 100644 --- a/tools/testing/selftests/bpf/progs/test_fill_link_info.c +++ b/tools/testing/selftests/bpf/progs/test_fill_link_info.c @@ -58,4 +58,10 @@ int BPF_PROG(umulti_run) return 0; } +SEC("fentry.multi") +int BPF_PROG(tmulti_run) +{ + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c new file mode 100644 index 000000000000..5dc21b3b4cb5 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +/* Used for testing map name. */ +int loong SEC(".percpu.looooooooong"); +int data3 SEC(".data.percpu"); +int data2 SEC(".percpu.data"); + +int run; +/* cpu_id as array to verify map value resizing. */ +int cpu_id[1] SEC(".percpu"); +int data SEC(".percpu") = -1; +int nums[7] SEC(".percpu"); +bool set SEC(".percpu") = false; +struct { + char set; + int i; + int nums[7]; +} struct_data SEC(".percpu") = { + .set = 0, + .i = -1, +}; + +SEC("raw_tp/task_rename") +__auxiliary +int update_percpu_data(void *ctx) +{ + struct_data.nums[6] = 0xc0de; + struct_data.set = 1; + struct_data.i = 1; + nums[6] = 0xc0de; + data = 1; + run++; + set = true; + cpu_id[0] = bpf_get_smp_processor_id(); + return 0; +} + +static const char fmt[] SEC(".percpu.fmt") = "data %d\n"; + +SEC("?kprobe") +__failure __msg("R{{[0-9]+}} points to percpu_array map which cannot be used as const string") +int verifier_strncmp(void *ctx) +{ + return bpf_strncmp("test", 5, fmt); +} + +SEC("?kprobe") +__failure __msg("R{{[0-9]+}} points to percpu_array map which cannot be used as const string") +int verifier_snprintf(void *ctx) +{ + u64 args[] = { data }; + char buf[128]; + int len; + + len = bpf_snprintf(buf, sizeof(buf), fmt, args, sizeof(args)); + if (len > 0) + bpf_printk("snprintf: %s\n", buf); + return 0; +} + +volatile const __u32 num_cpus = 0; +volatile const int num_off; +volatile const int elem_sz; +__u32 sum = 0; +bool run_iter = false; + +SEC("iter/bpf_map_elem") +__auxiliary +int dump_percpu_data(struct bpf_iter__bpf_map_elem *ctx) +{ + void *pptr = ctx->value; + int i; + + if (!pptr) + return 0; + + run_iter = true; + + for (i = 0; i < num_cpus; i++) { + sum += *(int *) (pptr + num_off); + pptr += elem_sz; + } + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_signed_loader.c b/tools/testing/selftests/bpf/progs/test_signed_loader.c index d9a4b85f9391..50451a69b99a 100644 --- a/tools/testing/selftests/bpf/progs/test_signed_loader.c +++ b/tools/testing/selftests/bpf/progs/test_signed_loader.c @@ -4,10 +4,11 @@ /* * Minimal, map-less program. Driven through libbpf's gen_loader (gen_hash) - * by prog_tests/signed_loader.c so the generated light-skeleton loader (with - * the emit_signature_match metadata check) can be exercised against good - * and tampered metadata. A socket filter needs no load-time attach resolution, - * and having no maps keeps the generated loader's ctx trivial (0 maps, 1 prog). + * by prog_tests/signed_loader.c so the generated light-skeleton loader can be + * exercised against good and tampered metadata, which the kernel now verifies + * at load time via the insns||metadata signature. A socket filter needs no + * load-time attach resolution, and having no maps keeps the generated loader's + * ctx trivial (0 maps, 1 prog). */ SEC("socket") int probe(void *ctx) diff --git a/tools/testing/selftests/bpf/progs/test_sockmap_update.c b/tools/testing/selftests/bpf/progs/test_sockmap_update.c deleted file mode 100644 index 6d64ea536e3d..000000000000 --- a/tools/testing/selftests/bpf/progs/test_sockmap_update.c +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// Copyright (c) 2020 Cloudflare -#include "vmlinux.h" -#include <bpf/bpf_helpers.h> - -struct { - __uint(type, BPF_MAP_TYPE_SOCKMAP); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, __u64); -} src SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_SOCKMAP); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, __u64); -} dst_sock_map SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_SOCKHASH); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, __u64); -} dst_sock_hash SEC(".maps"); - -SEC("tc") -int copy_sock_map(void *ctx) -{ - struct bpf_sock *sk; - bool failed = false; - __u32 key = 0; - - sk = bpf_map_lookup_elem(&src, &key); - if (!sk) - return SK_DROP; - - if (bpf_map_update_elem(&dst_sock_map, &key, sk, 0)) - failed = true; - - if (bpf_map_update_elem(&dst_sock_hash, &key, sk, 0)) - failed = true; - - bpf_sk_release(sk); - return failed ? SK_DROP : SK_PASS; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_tc_peer.c b/tools/testing/selftests/bpf/progs/test_tc_peer.c index 365eacb5dc34..cfb9ef7f467c 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_peer.c +++ b/tools/testing/selftests/bpf/progs/test_tc_peer.c @@ -35,6 +35,28 @@ int tc_src(struct __sk_buff *skb) } SEC("tc") +int tc_dst_ing(struct __sk_buff *skb) +{ + if (!skb->mark) { + skb->mark = 0x1; + return bpf_redirect_peer(IFINDEX_SRC, BPF_F_EGRESS); + } + + return bpf_redirect(IFINDEX_DST, 0); +} + +SEC("tc") +int tc_src_ing(struct __sk_buff *skb) +{ + if (!skb->mark) { + skb->mark = 0x1; + return bpf_redirect_peer(IFINDEX_DST, BPF_F_EGRESS); + } + + return bpf_redirect(IFINDEX_SRC, 0); +} + +SEC("tc") int tc_dst_l3(struct __sk_buff *skb) { return bpf_redirect(IFINDEX_SRC, 0); diff --git a/tools/testing/selftests/bpf/progs/test_tc_tunnel.c b/tools/testing/selftests/bpf/progs/test_tc_tunnel.c index 7376df405a6b..853bca962910 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_tunnel.c +++ b/tools/testing/selftests/bpf/progs/test_tc_tunnel.c @@ -6,6 +6,7 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_endian.h> +#include <bpf/bpf_core_read.h> #include "bpf_tracing_net.h" #include "bpf_compiler.h" @@ -37,6 +38,22 @@ struct vxlanhdr___local { #define EXTPROTO_VXLAN 0x1 +#define SKB_GSO_UDP_TUNNEL_MASK (SKB_GSO_UDP_TUNNEL | \ + SKB_GSO_UDP_TUNNEL_CSUM) + +#define SKB_GSO_TUNNEL_MASK (SKB_GSO_UDP_TUNNEL_MASK | \ + SKB_GSO_GRE | \ + SKB_GSO_GRE_CSUM | \ + SKB_GSO_IPXIP4 | \ + SKB_GSO_IPXIP6 | \ + SKB_GSO_ESP) + +#define BPF_F_ADJ_ROOM_DECAP_L4_MASK (BPF_F_ADJ_ROOM_DECAP_L4_UDP | \ + BPF_F_ADJ_ROOM_DECAP_L4_GRE) + +#define BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK (BPF_F_ADJ_ROOM_DECAP_IPXIP4 | \ + BPF_F_ADJ_ROOM_DECAP_IPXIP6) + #define VXLAN_FLAGS bpf_htonl(1<<27) #define VNI_ID 1 #define VXLAN_VNI bpf_htonl(VNI_ID << 8) @@ -589,9 +606,12 @@ int __encap_ip6vxlan_eth(struct __sk_buff *skb) return TC_ACT_OK; } -static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) +static int decap_internal(struct __sk_buff *skb, int off, int len, char proto, + __u64 ipxip_flag) { __u64 flags = BPF_F_ADJ_ROOM_FIXED_GSO; + struct sk_buff *kskb; + struct skb_shared_info *shinfo; struct ipv6_opt_hdr ip6_opt_hdr; struct gre_hdr greh; struct udphdr udph; @@ -599,10 +619,12 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) switch (proto) { case IPPROTO_IPIP: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4 | + ipxip_flag; break; case IPPROTO_IPV6: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6 | + ipxip_flag; break; case NEXTHDR_DEST: if (bpf_skb_load_bytes(skb, off + len, &ip6_opt_hdr, @@ -610,10 +632,12 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) return TC_ACT_OK; switch (ip6_opt_hdr.nexthdr) { case IPPROTO_IPIP: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV4 | + ipxip_flag; break; case IPPROTO_IPV6: - flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6; + flags |= BPF_F_ADJ_ROOM_DECAP_L3_IPV6 | + ipxip_flag; break; default: return TC_ACT_OK; @@ -621,6 +645,11 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) break; case IPPROTO_GRE: olen += sizeof(struct gre_hdr); + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_L4_GRE)) + return TC_ACT_SHOT; + flags |= BPF_F_ADJ_ROOM_DECAP_L4_GRE; + if (bpf_skb_load_bytes(skb, off + len, &greh, sizeof(greh)) < 0) return TC_ACT_OK; switch (bpf_ntohs(greh.protocol)) { @@ -634,6 +663,10 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) break; case IPPROTO_UDP: olen += sizeof(struct udphdr); + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_L4_UDP)) + return TC_ACT_SHOT; + flags |= BPF_F_ADJ_ROOM_DECAP_L4_UDP; if (bpf_skb_load_bytes(skb, off + len, &udph, sizeof(udph)) < 0) return TC_ACT_OK; switch (bpf_ntohs(udph.dest)) { @@ -655,6 +688,40 @@ static int decap_internal(struct __sk_buff *skb, int off, int len, char proto) if (bpf_skb_adjust_room(skb, -olen, BPF_ADJ_ROOM_MAC, flags)) return TC_ACT_SHOT; + kskb = bpf_cast_to_kern_ctx(skb); + shinfo = bpf_core_cast(kskb->head + kskb->end, struct skb_shared_info); + if (shinfo->gso_size) { + if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_UDP) && + (shinfo->gso_type & SKB_GSO_UDP_TUNNEL_MASK)) + return TC_ACT_SHOT; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_GRE) && + (shinfo->gso_type & (SKB_GSO_GRE | SKB_GSO_GRE_CSUM))) + return TC_ACT_SHOT; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP4) && + (shinfo->gso_type & SKB_GSO_IPXIP4)) + return TC_ACT_SHOT; + + if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP6) && + (shinfo->gso_type & SKB_GSO_IPXIP6)) + return TC_ACT_SHOT; + + if (flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK | + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) { + if ((shinfo->gso_type & SKB_GSO_TUNNEL_MASK) && + !kskb->encapsulation) + return TC_ACT_SHOT; + if (!(shinfo->gso_type & SKB_GSO_TUNNEL_MASK) && + kskb->encapsulation) + return TC_ACT_SHOT; + } + } else if ((flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK | + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) && + kskb->encapsulation) { + return TC_ACT_SHOT; + } + return TC_ACT_OK; } @@ -662,6 +729,10 @@ static int decap_ipv4(struct __sk_buff *skb) { struct iphdr iph_outer; + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_IPXIP4)) + return TC_ACT_SHOT; + if (bpf_skb_load_bytes(skb, ETH_HLEN, &iph_outer, sizeof(iph_outer)) < 0) return TC_ACT_OK; @@ -670,19 +741,25 @@ static int decap_ipv4(struct __sk_buff *skb) return TC_ACT_OK; return decap_internal(skb, ETH_HLEN, sizeof(iph_outer), - iph_outer.protocol); + iph_outer.protocol, + BPF_F_ADJ_ROOM_DECAP_IPXIP4); } static int decap_ipv6(struct __sk_buff *skb) { struct ipv6hdr iph_outer; + if (!bpf_core_enum_value_exists(enum bpf_adj_room_flags, + BPF_F_ADJ_ROOM_DECAP_IPXIP6)) + return TC_ACT_SHOT; + if (bpf_skb_load_bytes(skb, ETH_HLEN, &iph_outer, sizeof(iph_outer)) < 0) return TC_ACT_OK; return decap_internal(skb, ETH_HLEN, sizeof(iph_outer), - iph_outer.nexthdr); + iph_outer.nexthdr, + BPF_F_ADJ_ROOM_DECAP_IPXIP6); } SEC("tc") diff --git a/tools/testing/selftests/bpf/progs/tracing_failure.c b/tools/testing/selftests/bpf/progs/tracing_failure.c index 65e485c4468c..f7a095767679 100644 --- a/tools/testing/selftests/bpf/progs/tracing_failure.c +++ b/tools/testing/selftests/bpf/progs/tracing_failure.c @@ -30,3 +30,9 @@ int BPF_PROG(fexit_noreturns) { return 0; } + +SEC("?fexit/bpf_testmod_test_int128_ret") +int BPF_PROG(fexit_int128_ret) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c b/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c index cd5be0bb6ffd..5b0af8f4c62f 100644 --- a/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c +++ b/tools/testing/selftests/bpf/progs/tracing_multi_intersect_attach.c @@ -11,6 +11,14 @@ __u64 test_result_fentry_1 = 0; __u64 test_result_fentry_2 = 0; __u64 test_result_fexit_1 = 0; __u64 test_result_fexit_2 = 0; +__u64 test_result_fentry = 0; + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG(fentry) +{ + tracing_multi_arg_check(ctx, &test_result_fentry, false); + return 0; +} SEC("fentry.multi") int BPF_PROG(fentry_1) diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c new file mode 100644 index 000000000000..4638dfec1f38 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <vmlinux.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> + +long t_b, t_c, t_ret; + +SEC("fexit/bpf_testmod_test_int128_arg") +int test_int128_arg_fexit(unsigned long long *ctx) +{ + t_b = (int)ctx[2]; + t_c = (long)ctx[3]; + t_ret = (long)ctx[4]; + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index df0e22d1a29b..815f342eb4b0 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -12,15 +12,17 @@ #define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8))) +#ifdef __TARGET_ARCH_arm64 +#define ARENA_VM_START ((1ull << 32) | (~0u - __PAGE_SIZE * 2 + 1)) +#else +#define ARENA_VM_START ((1ull << 44) | (~0u - __PAGE_SIZE * 2 + 1)) +#endif + struct { __uint(type, BPF_MAP_TYPE_ARENA); __uint(map_flags, BPF_F_MMAPABLE); __uint(max_entries, 2); /* arena of two pages close to 32-bit boundary*/ -#ifdef __TARGET_ARCH_arm64 - __ulong(map_extra, (1ull << 32) | (~0u - __PAGE_SIZE * 2 + 1)); /* start of mmap() region */ -#else - __ulong(map_extra, (1ull << 44) | (~0u - __PAGE_SIZE * 2 + 1)); /* start of mmap() region */ -#endif + __ulong(map_extra, ARENA_VM_START); /* start of mmap() region */ } arena SEC(".maps"); SEC("socket") @@ -93,6 +95,34 @@ int basic_alloc1(void *ctx) return 0; } +SEC("syscall") +__success __retval(0) +int free_scalar_below_arena(void *ctx) +{ + void __arena *page1, *page2, *page3; + __u64 bad_addr = ARENA_VM_START - __PAGE_SIZE; + + page1 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page1) + return 1; + + page2 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!page2) + return 2; + + page3 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (page3) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)bad_addr, 1); + + page3 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (page3) + return 4; + + return 0; +} + SEC("socket") __success __retval(0) int basic_alloc2_nosleep(void *ctx) @@ -605,7 +635,37 @@ int non_arena_ptr_add_to_arena_ptr(void *ctx) return 0; } -#endif +SEC("socket") +__description("arena and stack atomic at the same instruction") +__failure __msg("same insn cannot be used with different pointers") +__arch_x86_64 +__load_if_JITed() +__naked void mixed_arena_stack_atomic(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r6 = r10; \ + r6 += -8; \ + r9 = 0; \ + *(u64 *)(r6 + 0) = r9; \ + r7 = 8192; \ + r7 = addr_space_cast(r7, 0, 1); \ + call %[bpf_get_prandom_u32]; \ + if w0 != 0 goto 1f; \ + r8 = r6; \ + goto 2f; \ +1: r8 = r7; \ +2: r9 = 1; \ + lock *(u64 *)(r8 + 0) += r9; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(arena), + __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#endif /* defined(__BPF_FEATURE_ADDR_SPACE_CAST) */ static __noinline u32 __arena *check_arena_arg_nonglobal(u32 __arena *arg) diff --git a/tools/testing/selftests/bpf/progs/verifier_basic_stack.c b/tools/testing/selftests/bpf/progs/verifier_basic_stack.c index d3df7a9f1d8c..0eb495ce85c1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_basic_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_basic_stack.c @@ -27,7 +27,8 @@ __naked void stack_out_of_bounds(void) SEC("socket") __description("uninitialized stack1") -__success __log_level(4) __msg("stack depth 8") +__success __log_level(4) +__msg("subprog 0 (uninitialized_stack1) main {{.*}} stack 8") __failure_unpriv __msg_unpriv("invalid read from stack") __naked void uninitialized_stack1(void) { @@ -45,7 +46,8 @@ __naked void uninitialized_stack1(void) SEC("socket") __description("uninitialized stack2") -__success __log_level(4) __msg("stack depth 8") +__success __log_level(4) +__msg("subprog 0 (uninitialized_stack2) main insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 8") __failure_unpriv __msg_unpriv("invalid read from stack") __naked void uninitialized_stack2(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index bc038ac2df98..1a273e416fed 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -1195,7 +1195,7 @@ l0_%=: r1 = r6; \ r3 += -8; \ r5 = 0; \ /* The 4th argument of bpf_skb_store_bytes is defined as \ - * ARG_CONST_SIZE, so 0 is not allowed. The 'r4 != 0' \ + * ARG_MEM_SIZE, so 0 is not allowed. The 'r4 != 0' \ * is providing us this exclusion of zero from initial \ * [0, 7] range. \ */ \ diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index 8d7ff38e4c06..328cf630210a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c @@ -10,7 +10,8 @@ SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 8") +__log_level(4) +__msg("subprog 0 (simple) main insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 8") __xlated("4: r5 = 5") __xlated("5: r0 = ") __xlated("6: r0 = &(void __percpu *)(r0)") @@ -96,7 +97,8 @@ __naked void canary_zero_spills(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 16") +__log_level(4) +__msg("subprog 0 (wrong_reg_in_pattern1) main {{.*}} stack 16") __xlated("1: *(u64 *)(r10 -16) = r1") __xlated("...") __xlated("3: r0 = &(void __percpu *)(r0)") @@ -598,7 +600,8 @@ __naked static void subprogs_use_independent_offsets_aux(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 8") +__log_level(4) +__msg("subprog 0 (helper_call_does_not_prevent_bpf_fastcall) main {{.*}} stack 8") __xlated("2: r0 = &(void __percpu *)(r0)") __success __naked void helper_call_does_not_prevent_bpf_fastcall(void) @@ -620,7 +623,8 @@ __naked void helper_call_does_not_prevent_bpf_fastcall(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 24") +__log_level(4) +__msg("subprog 0 (may_goto_interaction_x86_64) main {{.*}} stack 24") /* may_goto counter at -24 */ __xlated("0: *(u64 *)(r10 -24) =") /* may_goto timestamp at -16 */ @@ -660,7 +664,10 @@ __naked void may_goto_interaction_x86_64(void) SEC("raw_tp") __arch_arm64 -__log_level(4) __msg("stack depth 24") +__arch_riscv64 +__arch_loongarch +__log_level(4) +__msg("subprog 0 (may_goto_interaction) main {{.*}} stack 24") /* may_goto counter at -24 */ __xlated("0: *(u64 *)(r10 -24) =") /* may_goto timestamp at -16 */ @@ -679,7 +686,7 @@ __xlated("10: *(u64 *)(r10 -24) = r12") __xlated("11: *(u64 *)(r10 -8) = r1") __xlated("12: exit") __success -__naked void may_goto_interaction_arm64(void) +__naked void may_goto_interaction(void) { asm volatile ( "r1 = 1;" @@ -707,7 +714,9 @@ __naked static void dummy_loop_callback(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 32+0") +__log_level(4) +__msg("subprog 0 (bpf_loop_interaction1) main {{.*}} stack 32") +__msg("subprog 1 (dummy_loop_callback) static {{.*}} stack 0") __xlated("2: r1 = 1") __xlated("3: r0 =") __xlated("4: r0 = &(void __percpu *)(r0)") @@ -755,7 +764,9 @@ __naked int bpf_loop_interaction1(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 40+0") +__log_level(4) +__msg("subprog 0 (bpf_loop_interaction2) main {{.*}} stack 40") +__msg("subprog 1 (dummy_loop_callback) static {{.*}} stack 0") /* call bpf_get_smp_processor_id */ __xlated("2: r1 = 42") __xlated("3: r0 =") @@ -799,7 +810,10 @@ __naked int bpf_loop_interaction2(void) SEC("raw_tp") __arch_x86_64 -__log_level(4) __msg("stack depth 512+0 max 512") +__log_level(4) +__msg("stack depth max 512") +__msg("subprog 0 (cumulative_stack_depth) main {{.*}} stack 512") +__msg("subprog 1 (cumulative_stack_depth_subprog) static {{.*}} stack 0") /* just to print xlated version when debugging */ __xlated("r0 = &(void __percpu *)(r0)") __success diff --git a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c index 67dc352addfd..966f49348787 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c @@ -52,7 +52,15 @@ __msg("('global_calls_good_only') is global and assumed valid.") /* eventually global_good() is transitively validated as well */ __msg("Validating global_good() func") __msg("('global_good') is safe for any args that match its prototype") -__msg("insns processed {{[0-9]+\\+[0-9]+\\+[0-9]+$}}") +__msg("subprog 0 (chained_global_func_calls_success) main insns_self 7 insns_total 7 stack") +__msg("subprog {{[0-9]+}} (global_calls_good_only) global insns_self 2 insns_total 2 stack") +#if defined(__BPF_CPU_VERSION__) && __BPF_CPU_VERSION__ >= 4 +__msg("subprog {{[0-9]+}} (global_good) global insns_self 3 insns_total 3 stack") +__msg("processed 12 insns") +#else +__msg("subprog {{[0-9]+}} (global_good) global insns_self 5 insns_total 5 stack") +__msg("processed 14 insns") +#endif int chained_global_func_calls_success(void) { int sum = 0; diff --git a/tools/testing/selftests/bpf/progs/verifier_gotox.c b/tools/testing/selftests/bpf/progs/verifier_gotox.c index f88aa4cdb279..5b18c9a27717 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotox.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotox.c @@ -384,6 +384,31 @@ jt0_%=: \ : __clobber_all); } +/* check valid spill/fill, ptr to insn */ +SEC("socket") +__success +__naked void spill_fill_ptr_to_insn(void) +{ + asm volatile ( + ".pushsection .jumptables,\"\",@progbits;" + "jt0_%=:" + ".quad ret0_%= - socket;" + ".size jt0_%=, 8;" + ".global jt0_%=;" + ".popsection;" + "r0 = jt0_%= ll;" + "r0 = *(u64 *)(r0 + 0);" + "*(u64 *)(r10 - 8) = r0;" + "r0 = *(u64 *)(r10 - 8);" + ".8byte %[gotox_r0];" + "ret0_%=:" + "r0 = 0;" + "exit;" + : + : __imm_insn(gotox_r0, BPF_RAW_INSN(BPF_JMP | BPF_JA | BPF_X, BPF_REG_0, 0, 0, 0)) + : __clobber_all); +} + #endif /* __TARGET_ARCH_x86 || __TARGET_ARCH_arm64 || __TARGET_ARCH_powerpc*/ char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c index f2c54e4d89eb..343fc08d9747 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_access_var_len.c @@ -85,7 +85,7 @@ __naked void stack_bitwise_and_zero_included(void) r2 += -64; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory at &fp[-64] is\ * not initialized. \ */ \ @@ -278,7 +278,7 @@ __naked void stack_jmp_no_min_check(void) r2 += -64; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory at &fp[-64] is\ * not initialized. \ */ \ @@ -778,7 +778,7 @@ __naked void variable_memory_8_bytes_leak(void) r3 += 1; \ r4 = 0; \ /* Call bpf_ringbuf_output(), it is one of a few helper functions with\ - * ARG_CONST_SIZE_OR_ZERO parameter allowed in unpriv mode.\ + * ARG_MEM_SIZE_OR_ZERO parameter allowed in unpriv mode.\ * For unpriv this should signal an error, because memory region [1, 64]\ * at &fp[-64] is not fully initialized. \ */ \ diff --git a/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c b/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c index 6d2a38597c34..c6603a118fdc 100644 --- a/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_helper_value_access.c @@ -91,7 +91,7 @@ l0_%=: exit; \ /* Call a function taking a pointer and a size which doesn't allow the size to * be zero (i.e. bpf_trace_printk() declares the second argument to be - * ARG_CONST_SIZE, not ARG_CONST_SIZE_OR_ZERO). We attempt to pass zero for the + * ARG_MEM_SIZE, not ARG_MEM_SIZE_OR_ZERO). We attempt to pass zero for the * size and expect to fail. */ SEC("tracepoint") diff --git a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c index 74f4f19c10b8..d17026d7480d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c +++ b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c @@ -3,6 +3,7 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include <bpf/bpf_core_read.h> #include "../../../include/linux/filter.h" #include "bpf_misc.h" @@ -148,6 +149,22 @@ __naked void load_acquire_from_ctx_pointer(void) : __clobber_all); } +SEC("socket") +__description("load-acquire from ctx pointer, same dst and src register") +__failure __failure_unpriv __msg("BPF_ATOMIC loads from R6 ctx is not allowed") +__naked void load_acquire_ctx_same_dst_src(void) +{ + asm volatile ( + "r6 = r1;" + ".8byte %[load_acquire_insn];" // w6 = load_acquire((u32 *)(r6 + 0)); + "r0 = 0;" + "exit;" + : + : __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_6, BPF_REG_6, 0)) + : __clobber_all); +} + SEC("xdp") __description("load-acquire from pkt pointer") __failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed") @@ -206,6 +223,33 @@ __naked void load_acquire_from_sock_pointer(void) } SEC("socket") +__description("load-acquire from rdonly_untrusted_mem pointer") +__failure __msg("BPF_ATOMIC loads from R{{[0-9]+}} rdonly_untrusted_mem is not allowed") +int load_acquire_from_rdonly_untrusted_mem(void *ctx) +{ + __u64 val = 0; + void *p; + + /* + * bpf_rdonly_cast(x, 0) yields PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED. + * A regular BPF_LDX from it is rewritten to BPF_PROBE_MEM, but a + * load-acquire is not, so it must be rejected, otherwise the JIT emits + * a plain load with no exception table entry and a fault would crash + * the kernel. + */ + p = bpf_rdonly_cast(&val, 0); + asm volatile ( + "r1 = %[p];" + ".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0)); + : + : [p] "r" (p), + __imm_insn(load_acquire_insn, + BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0)) + : "r0", "r1"); + return 0; +} + +SEC("socket") __description("load-acquire with invalid register R15") __failure __failure_unpriv __msg("R15 is invalid") __naked void load_acquire_with_invalid_reg(void) diff --git a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c index b606b5dca734..d3be69a9a755 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c @@ -154,7 +154,8 @@ l0_%=: r0 = 0; \ SEC("socket") __description("forgot null checking on the inner map pointer") -__failure __msg("R1 type=map_value_or_null expected=map_ptr") +__failure __msg("R1 type=map_ptr_or_null expected=map_ptr") +__msg("map_ptr_or_null, but this argument accepts map_ptr") __failure_unpriv __naked void on_the_inner_map_pointer(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c new file mode 100644 index 000000000000..c01abf54923d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_map_lookup_refine.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +struct inner_map { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); +} inner_map SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS); + __uint(max_entries, 1); + __type(key, int); + __array(values, struct inner_map); +} outer_map SEC(".maps") = { + .values = { [0] = &inner_map }, +}; + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_kfunc_mem_buf(struct __sk_buff *skb) +{ + struct bpf_dynptr dptr; + __u32 key = 0; + void *inner; + char *p; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + bpf_dynptr_from_skb(skb, 0, &dptr); + /* arg3 is mem+size */ + p = bpf_dynptr_slice(&dptr, 0, inner, 4); + if (p) + return p[0]; + return 0; +} + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_helper_mem_buf(struct __sk_buff *skb) +{ + __u32 key = 0; + void *inner; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + /* arg1 is mem+size */ + return bpf_csum_diff(inner, 4, NULL, 0, 0) + skb->len; +} + +SEC("?tc") +__failure __msg("type=map_ptr_or_null expected=fp") +int mapofmaps_value_as_helper_fixed_mem(struct __sk_buff *skb) +{ + char th[sizeof(struct tcphdr)] = {}; + __u32 key = 0; + void *inner; + + inner = bpf_map_lookup_elem(&outer_map, &key); + /* intentionally NOT NULL-checked: type is map_ptr_or_null */ + + /* arg1 is fixed-sized mem */ + return bpf_tcp_raw_check_syncookie_ipv4(inner, (void *)th); +} diff --git a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c index 166193659870..e0a65835c861 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c @@ -72,14 +72,15 @@ __naked void bpf_map_ptr_write_rejected(void) /* * struct bpf_map starts with the SHA256 hash sha[32] at offset 0 (a readable - * byte array), the u32 excl field at offset 32, and the ops pointer at offset - * 40. Reading a u32 at offset 41 reaches into the middle of the ops pointer, - * i.e. a partial pointer access, which is rejected. + * byte array), followed by the ops pointer at offset 32 and the inner_map_meta + * pointer at offset 40. Reading a u32 at offset 41 reaches into the middle of + * the inner_map_meta pointer, i.e. a partial pointer access, which is + * rejected. */ SEC("socket") __description("bpf_map_ptr: read non-existent field rejected") __failure -__msg("cannot access ptr member ops with moff 40 in struct bpf_map with off 41 size 4") +__msg("cannot access ptr member inner_map_meta with moff 40 in struct bpf_map with off 41 size 4") __failure_unpriv __msg_unpriv("access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") __flag(BPF_F_ANY_ALIGNMENT) @@ -97,23 +98,23 @@ __naked void read_non_existent_field_rejected(void) } /* - * The u32 excl field spans offsets 32..35 (mend 36). Reading a u32 at offset - * 33 starts inside excl but extends past its end, which the verifier rejects + * The sha byte array spans offsets 0..31 (mend 32). Reading a u32 at offset + * 30 starts inside sha but extends past its end, which the verifier rejects * as an out-of-bounds scalar access. */ SEC("socket") -__description("bpf_map_ptr: read beyond excl field rejected") +__description("bpf_map_ptr: read beyond sha field rejected") __failure -__msg("access beyond the end of member excl (mend:36) in struct bpf_map with off 33 size 4") +__msg("access beyond the end of member sha (mend:32) in struct bpf_map with off 30 size 4") __failure_unpriv __msg_unpriv("access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") __flag(BPF_F_ANY_ALIGNMENT) -__naked void read_beyond_excl_field_rejected(void) +__naked void read_beyond_sha_field_rejected(void) { asm volatile (" \ r6 = 0; \ r1 = %[map_array_48b] ll; \ - r6 = *(u32*)(r1 + 33); \ + r6 = *(u32*)(r1 + 30); \ r0 = 1; \ exit; \ " : @@ -131,7 +132,7 @@ __naked void ptr_read_ops_field_accepted(void) asm volatile (" \ r6 = 0; \ r1 = %[map_array_48b] ll; \ - r6 = *(u64*)(r1 + 40); \ + r6 = *(u64*)(r1 + 32); \ r0 = 1; \ exit; \ " : diff --git a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c index 4bdf4256a41e..db7e30da234f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +++ b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c @@ -11,6 +11,8 @@ __description("may_goto 0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: r0 = 1") __xlated("1: exit") __success @@ -31,6 +33,8 @@ __description("batch 2 of may_goto 0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: r0 = 1") __xlated("1: exit") __success @@ -53,6 +57,8 @@ __description("may_goto batch with offsets 2/1/0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: r0 = 1") __xlated("1: exit") __success @@ -79,6 +85,8 @@ __description("may_goto batch with offsets 2/0") __arch_x86_64 __arch_s390x __arch_arm64 +__arch_riscv64 +__arch_loongarch __xlated("0: *(u64 *)(r10 -16) = 65535") __xlated("1: *(u64 *)(r10 -8) = 0") __xlated("2: r12 = *(u64 *)(r10 -16)") @@ -106,4 +114,63 @@ __naked void may_goto_batch_2(void) : __clobber_all); } +/* + * Use bpf_get_prandom_u32() to prevent DCE from removing the checks. + * retval: 0=all ok, 1-6=R0-R5 clobbered. + */ +SEC("syscall") +__description("timed may_goto preserves R0-R5") +__arch_x86_64 +__arch_s390x +__arch_arm64 +__arch_riscv64 +__arch_loongarch +__success +__retval(0) +__naked void timed_may_goto_preserves_regs(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r6 = r0;" + "r0 = 0x1111;" + "r0 += r6;" + "r1 = 0x2222;" + "r1 += r6;" + "r2 = 0x3333;" + "r2 += r6;" + "r3 = 0x4444;" + "r3 += r6;" + "r4 = 0x5555;" + "r4 += r6;" + "r5 = 0x6666;" + "r5 += r6;" + ".8byte %[may_goto];" + ".8byte %[loop];" + "r0 -= r6;" + "r1 -= r6;" + "r2 -= r6;" + "r3 -= r6;" + "r4 -= r6;" + "r5 -= r6;" + "if r0 != 0x1111 goto 1f;" + "if r1 != 0x2222 goto 2f;" + "if r2 != 0x3333 goto 3f;" + "if r3 != 0x4444 goto 4f;" + "if r4 != 0x5555 goto 5f;" + "if r5 != 0x6666 goto 6f;" + "r0 = 0;" + "exit;" + "1: r0 = 1; exit;" + "2: r0 = 2; exit;" + "3: r0 = 3; exit;" + "4: r0 = 4; exit;" + "5: r0 = 5; exit;" + "6: r0 = 6; exit;" + : + : __imm(bpf_get_prandom_u32), + __imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP | BPF_JCOND, 0, 0, 1, 0)), + __imm_insn(loop, BPF_RAW_INSN(BPF_JMP | BPF_JA, 0, 0, -2, 0)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c b/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c new file mode 100644 index 000000000000..7e24706a764e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_mem_size_reg.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "bpf_kfuncs.h" + +char _license[] SEC("license") = "GPL"; + +/* + * The __szk size of a kfunc memory/size pair must be marked precise even when + * the nullable buffer is passed as NULL. + */ +SEC("?tc") +__success __log_level(2) +__msg("mark_precise: frame0: regs=r4 stack= before") +int dynptr_slice_null_buf_size_precise(struct __sk_buff *skb) +{ + struct bpf_dynptr dptr; + char *p; + + bpf_dynptr_from_skb(skb, 0, &dptr); + + p = bpf_dynptr_slice(&dptr, 0, NULL, 8); + if (p) + return p[0]; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c new file mode 100644 index 000000000000..967f4e6e3a49 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +#if defined(__TARGET_ARCH_x86) + +int percpu_data SEC(".percpu"); + +/* + * An ld_imm64 of a per-CPU map value is followed by a mov_percpu_addr that + * reuses the same register, so check that the add resolves into the register + * the address was loaded into, for every register. + */ +SEC("raw_tp") +__description("per-CPU address resolution") +__success +__arch_x86_64 +__jited(" movabsq $0x{{.*}}, %rax") +__jited(" addq %gs:{{.*}}, %rax") +__jited(" movabsq $0x{{.*}}, %rdi") +__jited(" addq %gs:{{.*}}, %rdi") +__jited(" movabsq $0x{{.*}}, %rsi") +__jited(" addq %gs:{{.*}}, %rsi") +__jited(" movabsq $0x{{.*}}, %rdx") +__jited(" addq %gs:{{.*}}, %rdx") +__jited(" movabsq $0x{{.*}}, %rcx") +__jited(" addq %gs:{{.*}}, %rcx") +__jited(" movabsq $0x{{.*}}, %r8") +__jited(" addq %gs:{{.*}}, %r8") +__jited(" movabsq $0x{{.*}}, %rbx") +__jited(" addq %gs:{{.*}}, %rbx") +__jited(" movabsq $0x{{.*}}, %r13") +__jited(" addq %gs:{{.*}}, %r13") +__jited(" movabsq $0x{{.*}}, %r14") +__jited(" addq %gs:{{.*}}, %r14") +__jited(" movabsq $0x{{.*}}, %r15") +__jited(" addq %gs:{{.*}}, %r15") +__naked void percpu_addr(void) +{ + asm volatile (" \ + r0 = %[percpu_data] ll; \ + r1 = %[percpu_data] ll; \ + r2 = %[percpu_data] ll; \ + r3 = %[percpu_data] ll; \ + r4 = %[percpu_data] ll; \ + r5 = %[percpu_data] ll; \ + r6 = %[percpu_data] ll; \ + r7 = %[percpu_data] ll; \ + r8 = %[percpu_data] ll; \ + r9 = %[percpu_data] ll; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(percpu_data) + : __clobber_all); +} + +#else + +SEC("raw_tp") +__description("percpu addr dummy") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index bb8206e10880..ea0a7e73331d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -86,7 +86,9 @@ __naked static void cumulative_stack_depth_subprog(void) SEC("kprobe") __description("Private stack, subtree > MAX_BPF_STACK") __success -__log_level(4) __msg("stack depth 512+32 max 512") +__log_level(4) __msg("stack depth max 512") +__msg("subprog 0 (private_stack_nested_1) main {{.*}} stack 512") +__msg("subprog 1 (cumulative_stack_depth_subprog) static {{.*}} stack 32") __arch_x86_64 /* private stack fp for the main prog */ __jited(" movabsq $0x{{.*}}, %r9") @@ -331,7 +333,11 @@ SEC("fentry/bpf_fentry_test9") __description("Private stack, async callback, potential nesting") __success __retval(0) __load_if_JITed() -__log_level(4) __msg("stack depth 8+0+256+0 max 272") +__log_level(4) __msg("stack depth max 272") +__msg("subprog 0 (private_stack_async_callback_2) main {{.*}} stack 8") +__msg("subprog 1 (timer_cb1) static {{.*}} stack 0") +__msg("subprog 2 (subprog1) static {{.*}} stack 256") +__msg("subprog 3 (subprog2) static {{.*}} stack 0") __arch_x86_64 __jited(" subq $0x100, %rsp") __arch_arm64 @@ -355,7 +361,10 @@ int private_stack_async_callback_2(void) SEC("fentry/bpf_fentry_test9") __description("private stack, max stack depth is private stack") __success -__log_level(4) __msg("stack depth 8+256+0 max 256") +__log_level(4) __msg("stack depth max 256") +__msg("subprog 0 (private_stack_max_depth) main {{.*}} stack 8") +__msg("subprog 1 (subprog1) static insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 256") +__msg("subprog 2 (subprog2) static insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 0") int private_stack_max_depth(void) { int x = 0; diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c index e38f102da45f..663d15fc5fd2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c +++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c @@ -4,6 +4,13 @@ #include <bpf/bpf_helpers.h> #include "bpf_misc.h" +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, long long); + __type(value, long long); +} map_hash_8b SEC(".maps"); + /* Check that precision marks propagate through scalar IDs. * Registers r{0,1,2} have the same scalar ID. * Range information is propagated for scalars sharing same ID. @@ -915,4 +922,53 @@ __naked void linked_regs_and_subreg_def(void) : __clobber_all); } +/* + * A scalar is spilled to the stack and then filled twice: once via a + * sign-extending load (BPF_MEMSX) into r4 and once via a zero-extending + * load (BPF_MEM) into r5. coerce_reg_to_size_sx() gives r4 a different + * value than the spilled/zero-extended siblings, so r4 must not keep the + * shared scalar id. Otherwise the later 'if r5 == 0x80000000' refines r4 + * through sync_linked_regs() to a known 0x80000000, while at runtime r4 + * is the sign-extended 0xffffffff80000000. The test turns that discrepancy + * into an out-of-bounds map value access (r4 >> 63 is believed 0 but is 1 + * at runtime), which must be rejected. + */ +SEC("socket") +__failure __msg("R0 max value is outside of the allowed memory range") +__naked void ldsx_fill_scalar_id_not_shared(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + /* r7 = unknown u32, keep only bit 31 */ \ + r7 = *(u32*)(r0 + 0); \ + r2 = 0x80000000 ll; \ + r7 &= r2; \ + /* link r6 and r7 via a fresh scalar id */ \ + r6 = r7; \ + /* spill r7 (u32) to the stack */ \ + *(u32*)(r10 - 8) = r7; \ + /* sign-extending fill: must drop the id */ \ + r4 = *(s32*)(r10 - 8); \ + /* zero-extending fill: keeps the id */ \ + r5 = *(u32*)(r10 - 8); \ + /* r5 becomes known 0x80000000 on fall-through */\ + if r5 != r2 goto l0_%=; \ + /* verifier believes r4 == 0 here, runtime is 1 */\ + r4 >>= 63; \ + r0 += r4; \ + r0 = *(u8*)(r0 + 7); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c b/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c index fe4b123187b8..20332a731d4e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c +++ b/tools/testing/selftests/bpf/progs/verifier_sockmap_mutate.c @@ -74,7 +74,7 @@ static __always_inline void test_sockmap_lookup_and_mutate(void) } SEC("action") -__success +__failure __msg("cannot update sockmap in this context") int test_sched_act(struct __sk_buff *skb) { test_sockmap_mutate(skb->sk); @@ -82,7 +82,7 @@ int test_sched_act(struct __sk_buff *skb) } SEC("classifier") -__success +__failure __msg("cannot update sockmap in this context") int test_sched_cls(struct __sk_buff *skb) { test_sockmap_mutate(skb->sk); @@ -90,7 +90,7 @@ int test_sched_cls(struct __sk_buff *skb) } SEC("flow_dissector") -__success +__failure __msg("cannot update sockmap in this context") int test_flow_dissector_delete(struct __sk_buff *skb __always_unused) { test_sockmap_delete(); @@ -98,7 +98,7 @@ int test_flow_dissector_delete(struct __sk_buff *skb __always_unused) } SEC("flow_dissector") -__failure __msg("program of this type cannot use helper bpf_sk_release") +__failure __msg("cannot update sockmap in this context") int test_flow_dissector_update(struct __sk_buff *skb __always_unused) { test_sockmap_lookup_and_update(); /* no access to skb->sk */ @@ -146,7 +146,7 @@ int test_sk_reuseport(struct sk_reuseport_md *ctx) } SEC("socket") -__success +__failure __msg("cannot update sockmap in this context") int test_socket_filter(struct __sk_buff *skb) { test_sockmap_mutate(skb->sk); @@ -179,7 +179,7 @@ int test_sockops_update_dedicated(struct bpf_sock_ops *ctx) } SEC("xdp") -__success +__failure __msg("cannot update sockmap in this context") int test_xdp(struct xdp_md *ctx __always_unused) { test_sockmap_lookup_and_mutate(); diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 0174887e28f5..8b166c42c4e0 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -634,6 +634,32 @@ __naked void partial_stack_load_preserves_partial_zeros(void) : __clobber_common); } +SEC("raw_tp") +__log_level(2) +__success +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R2=0") +__naked void stack_load_preserves_mixed_zero_and_zero_spill(void) +{ + asm volatile ( + /* fp-8 has scalar const-zero spill bytes and STACK_ZERO bytes. */ + ".8byte %[fp4_st_zero];" /* LLVM-18+: *(u32 *)(r10 -4) = 0; */ + "r0 = 0;" + "*(u32 *)(r10 -8) = r0;" + + "r1 = %[single_byte_buf];" + "r2 = *(u64 *)(r10 -8);" + "r1 += r2;" + "*(u8 *)(r1 + 0) = r2;" /* this should be fine */ + + "r0 = 0;" + "exit;" + : + : __imm_ptr(single_byte_buf), + __imm_insn(fp4_st_zero, BPF_ST_MEM(BPF_W, BPF_REG_FP, -4, 0)) + : __clobber_common); +} + char two_byte_buf[2] SEC(".data.two_byte_buf"); SEC("raw_tp") @@ -1377,4 +1403,46 @@ __naked void partial_fill_from_cleaned_pointer_spill(void) ::: __clobber_all); } +/* check valid spill/fill, ptr to tp buffer */ +SEC("raw_tracepoint.w") +__success +__naked void spill_fill_ptr_to_tp_buffer(void) +{ + asm volatile ( + "r6 = *(u64*)(r1 + 0);" /* r6 is the writable tracepoint buffer */ + "*(u64*)(r10 - 8) = r6;" + "r7 = *(u64*)(r10 - 8);" + "r0 = 0;" + "*(u64*)(r7 + 0) = r0;" /* should be able to write through the buffer */ + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +__noinline int spill_fill_dynptr_subprog(struct bpf_dynptr *dptr) +{ + long *p; + + asm volatile ("*(u64 *)(r10 - 8) = %[dptr];" /* spill the CONST_PTR_TO_DYNPTR argument */ + "%[dptr] = *(u64 *)(r10 - 8);" + : [dptr] "+r"(dptr) :: "memory"); + p = bpf_dynptr_data(dptr, 0, sizeof(*p)); + if (!p) + return 0; + return 0; +} + +static char dptr_mem_buf[16]; + +/* check valid spill/fill, const ptr to dynptr */ +SEC("socket") +__success +int spill_fill_const_ptr_to_dynptr(void) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_mem(dptr_mem_buf, sizeof(dptr_mem_buf), 0, &ptr); + return spill_fill_dynptr_subprog(&ptr); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c b/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c new file mode 100644 index 000000000000..8f6082fdb5c8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_subprog_insn_stats.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +struct timer_value { + struct bpf_timer timer; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct timer_value); +} timer_map SEC(".maps"); + +SEC("?raw_tp") +__success __log_level(4) +__msg("subprog 0 (stats_main_only) main insns_self 2 insns_total 2 stack 0") +__msg("processed 2 insns") +__naked int stats_main_only(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked __noinline __used +static int stats_chain_leaf(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked __noinline __used +static int stats_chain_parent(void) +{ + asm volatile ( + "call stats_chain_leaf;" + "exit;" + ); +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * self: 2 + 2 + 2 = 6 + * totals: leaf 2, parent 2 + 2 = 4, main 2 + 4 = 6 + */ +__msg("subprog 0 (stats_static_chain) main insns_self 2 insns_total 6 stack 0") +__msg("subprog {{[0-9]+}} (stats_chain_parent) static insns_self 2 insns_total 4 stack 0") +__msg("subprog {{[0-9]+}} (stats_chain_leaf) static insns_self 2 insns_total 2 stack 0") +__msg("processed 6 insns") +__naked int stats_static_chain(void) +{ + asm volatile ( + "call stats_chain_parent;" + "exit;" + ); +} + +__naked __noinline __used +static int stats_shared_leaf(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked __noinline __used +int stats_global_root(void) +{ + asm volatile ( + "call stats_shared_leaf;" + "exit;" + ); +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * stats_shared_leaf is explored once under each independent root. + * self: main 3 + leaf 4 + global 2 = 9 + * root totals: main 5 + global 4 = 9 + */ +__msg("subprog 0 (stats_shared_roots) main insns_self 3 insns_total 5 stack 0") +__msg("subprog {{[0-9]+}} (stats_shared_leaf) static insns_self 4 insns_total 4 stack 0") +__msg("subprog {{[0-9]+}} (stats_global_root) global insns_self 2 insns_total 4 stack 0") +__msg("processed 9 insns") +__naked int stats_shared_roots(void) +{ + asm volatile ( + "call stats_shared_leaf;" + "call stats_global_root;" + "exit;" + ); +} + +__noinline __used +static int stats_async_leaf(void *map, __u32 *key, struct bpf_timer *timer) +{ + return 0; +} + +__noinline __used +static __u64 stats_async_schedule(struct bpf_map *map, __u32 *key, + struct timer_value *value, void *ctx) +{ + asm volatile ( + "r1 = %[timer];" + "r2 = %[stats_async_leaf];" + "call %[bpf_timer_set_callback];" + : + : [timer] "r" (value), + __imm_ptr(stats_async_leaf), + __imm(bpf_timer_set_callback) + : __clobber_common + ); + return 0; +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * self: 9 + 7 + 2 = 18 + * totals: leaf 2, scheduler 7, main root 18 + */ +__msg("subprog 0 (stats_async_direct) main insns_self 9 insns_total 18 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_schedule) static insns_self 7 insns_total 7 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_leaf) static insns_self 2 insns_total 2 stack 0") +__msg("processed 18 insns") +__naked int stats_async_direct(void) +{ + asm volatile ( + "r1 = %[timer_map] ll;" + "r2 = %[stats_async_schedule];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_for_each_map_elem];" + "r0 = 0;" + "exit;" + : + : __imm_addr(timer_map), + __imm_ptr(stats_async_schedule), + __imm(bpf_for_each_map_elem) + : __clobber_common + ); +} + +__noinline __used +static int stats_async_nested_leaf(void *map, __u32 *key, struct bpf_timer *timer) +{ + return 0; +} + +__noinline __used +static int stats_async_outer(void *map, __u32 *key, struct bpf_timer *timer) +{ + asm volatile ( + "r1 = %[timer];" + "r2 = %[stats_async_nested_leaf];" + "call %[bpf_timer_set_callback];" + : + : [timer] "r" (timer), + __imm_ptr(stats_async_nested_leaf), + __imm(bpf_timer_set_callback) + : __clobber_common + ); + return 0; +} + +__noinline __used +static __u64 stats_async_nested_schedule(struct bpf_map *map, __u32 *key, + struct timer_value *value, void *ctx) +{ + asm volatile ( + "r1 = %[timer];" + "r2 = %[stats_async_outer];" + "call %[bpf_timer_set_callback];" + : + : [timer] "r" (value), + __imm_ptr(stats_async_outer), + __imm(bpf_timer_set_callback) + : __clobber_common + ); + return 0; +} + +SEC("?raw_tp") +__success __log_level(4) +/* + * self: 9 + 7 + 7 + 2 = 25 + * totals: leaf 2, outer 7, scheduler 7, main root 25 + */ +__msg("subprog 0 (stats_async_nested) main insns_self 9 insns_total 25 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_nested_schedule) static insns_self 7 insns_total 7 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_outer) static insns_self 7 insns_total 7 stack 0") +__msg("subprog {{[0-9]+}} (stats_async_nested_leaf) static insns_self 2 insns_total 2 stack 0") +__msg("processed 25 insns") +__naked int stats_async_nested(void) +{ + asm volatile ( + "r1 = %[timer_map] ll;" + "r2 = %[stats_async_nested_schedule];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_for_each_map_elem];" + "r0 = 0;" + "exit;" + : + : __imm_addr(timer_map), + __imm_ptr(stats_async_nested_schedule), + __imm(bpf_for_each_map_elem) + : __clobber_common + ); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_uninit.c b/tools/testing/selftests/bpf/progs/verifier_uninit.c index 7718cd7d19ce..691018a46049 100644 --- a/tools/testing/selftests/bpf/progs/verifier_uninit.c +++ b/tools/testing/selftests/bpf/progs/verifier_uninit.c @@ -9,6 +9,7 @@ SEC("socket") __description("read uninitialized register") __failure __msg("R2 !read_ok") +__msg("R2 has never been initialized on this path") __failure_unpriv __naked void read_uninitialized_register(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_unpriv.c b/tools/testing/selftests/bpf/progs/verifier_unpriv.c index 42de5cff7e52..3069e70fbcbd 100644 --- a/tools/testing/selftests/bpf/progs/verifier_unpriv.c +++ b/tools/testing/selftests/bpf/progs/verifier_unpriv.c @@ -96,6 +96,24 @@ __naked void pseudo_btf_id_log_masks_address(void) : __clobber_all); } +static int pseudo_func_callback(__u32 index, void *ctx) +{ + return 0; +} + +SEC("socket") +__description("unpriv: pseudo function policy diagnostic") +__success __failure_unpriv +__msg_unpriv("loading/calling other bpf or kernel functions") +__not_msg_unpriv("BPF-to-BPF function call") +__msg_unpriv("policy check failed for BPF function reference") +__msg_unpriv("avoid BPF function references in unprivileged") +int unpriv_pseudo_func_policy(void *ctx) +{ + bpf_loop(1, pseudo_func_callback, NULL, 0); + return 0; +} + SEC("socket") __description("unpriv: return pointer") __success __failure_unpriv __msg_unpriv("R0 leaks addr") diff --git a/tools/testing/selftests/bpf/progs/verifier_var_off.c b/tools/testing/selftests/bpf/progs/verifier_var_off.c index f345466bca68..a63e33675091 100644 --- a/tools/testing/selftests/bpf/progs/verifier_var_off.c +++ b/tools/testing/selftests/bpf/progs/verifier_var_off.c @@ -60,6 +60,116 @@ __naked void stack_read_priv_vs_unpriv(void) } SEC("cgroup/skb") +__description("variable-offset stack read preserves spilled zero") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_spilled_zero(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 7; \ + r2 -= 8; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read preserves spilled zero across slots") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8,-16") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_spilled_zero_across_slots(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64*)(r10 - 8) = r0; \ + *(u64*)(r10 - 16) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 15; \ + r2 -= 16; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read preserves partial spilled zero") +__success +__log_level(2) +__msg("mark_precise: frame0: regs= stack=-8") +__msg("R3=0") +__retval(0) +__naked void stack_read_var_off_preserves_partial_spilled_zero(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u8*)(r10 - 9) = r0; \ + *(u8*)(r10 - 10) = r0; \ + *(u8*)(r10 - 11) = r0; \ + *(u8*)(r10 - 12) = r0; \ + *(u8*)(r10 - 13) = r0; \ + *(u8*)(r10 - 14) = r0; \ + *(u8*)(r10 - 15) = r0; \ + *(u32*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 15; \ + if r2 > 10 goto l0_%=; \ + r2 -= 15; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = r3; \ +l0_%=: r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") +__description("variable-offset stack read partial spill with misc data") +__failure +__msg("invalid variable-offset write to stack R1") +__naked void stack_read_var_off_partial_spill_with_misc_data(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u32*)(r10 - 8) = r0; \ + r2 = *(u32*)(r1 + 0); \ + r2 &= 7; \ + r2 -= 8; \ + r2 += r10; \ + r3 = *(u8*)(r2 + 0); \ + r1 = r10; \ + r1 += -1; \ + r1 += r3; \ + *(u8*)(r1 + 0) = 0; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("cgroup/skb") __description("variable-offset stack read, uninitialized") __success __failure_unpriv __msg_unpriv("R2 variable stack access prohibited for !root") @@ -88,7 +198,8 @@ __success /* Check that the maximum stack depth is correctly maintained according to the * maximum possible variable offset. */ -__log_level(4) __msg("stack depth 16") +__log_level(4) +__msg("subprog 0 (stack_write_priv_vs_unpriv) main {{.*}} stack 16") __failure_unpriv /* Variable stack access is rejected for unprivileged. */ @@ -128,7 +239,8 @@ __success /* Check that the maximum stack depth is correctly maintained according to the * maximum possible variable offset. */ -__log_level(4) __msg("stack depth 16") +__log_level(4) +__msg("subprog 0 (stack_write_followed_by_read) main {{.*}} stack 16") __failure_unpriv __msg_unpriv("R2 variable stack access prohibited for !root") __retval(0) diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index 2870738d93f7..8f0c45421f89 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -28,7 +28,7 @@ int BPF_PROG(get_task_exe_file_kfunc_null) } SEC("lsm.s/inode_getxattr") -__failure __msg("R1 pointer type STRUCT task_struct must point to scalar, or struct with scalar") +__failure __msg("R1 is fp expected STRUCT task_struct") int BPF_PROG(get_task_exe_file_kfunc_fp) { u64 x; @@ -98,7 +98,7 @@ int BPF_PROG(path_d_path_kfunc_null) } SEC("lsm.s/task_alloc") -__failure __msg("R1 must be referenced or trusted") +__failure __msg("dereference of modified untrusted_ptr_") int BPF_PROG(path_d_path_kfunc_untrusted_from_argument, struct task_struct *task) { struct path *root; @@ -112,7 +112,7 @@ int BPF_PROG(path_d_path_kfunc_untrusted_from_argument, struct task_struct *task } SEC("lsm.s/file_open") -__failure __msg("R1 must be referenced or trusted") +__failure __msg("dereference of modified untrusted_ptr_") int BPF_PROG(path_d_path_kfunc_untrusted_from_current) { struct path *pwd; diff --git a/tools/testing/selftests/bpf/progs/verifier_zext.c b/tools/testing/selftests/bpf/progs/verifier_zext.c new file mode 100644 index 000000000000..8f2362da91d6 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_zext.c @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" +#include <bpf_arena_common.h> +#include <bpf/bpf_core_read.h> +#include "bpf_misc.h" + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE | BPF_F_NO_USER_CONV); + __uint(max_entries, 1); +} arena SEC(".maps"); + +extern long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym; + +/* to retain debug info for BTF generation */ +void __kfunc_btf_root(void) +{ + bpf_kfunc_call_test4(0, 0, 0, 0); + bpf_arena_alloc_pages(0, 0, 0, 0, 0); + bpf_rdonly_cast(0, 0); +} + +SEC("socket") +__flag(BPF_F_TEST_STATE_FREQ) +__flag(BPF_F_TEST_RND_HI32) +__success __retval(0) +__naked void zext_lost_across_checkpoint(void) +{ + asm volatile (" \ + call %[bpf_ktime_get_ns]; \ + r8 = r0; \ + r6 = 0xdeadbeefcafebabe ll; /* inject some value for r6's upper half */ \ + if r8 != 0 goto 1f; /* fall-through cached first, branch pruned */ \ + r6 = 32; /* full 64-bit def */ \ + goto 2f; \ +1: w6 = 32; /* 32-bit def, zext mark lost */ \ +2: r0 = r6; /* buggy verifier believed upper 32 bits are 0 */ \ + /* and thus did not zero extended w6 = 32. */ \ + r0 >>= 32; \ + exit; \ +" : + : __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* 32-bit ALU result read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("w1 = w0{{ +}}; zext") +__naked void zext_alu32_hi_used(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + r0 = r1; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* 32-bit ALU result read only as 32-bit -> no zext */ +SEC("socket") +__success __log_level(2) +__not_msg("; zext") +__naked void no_zext_alu32_hi_unused(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; /* MOV */ \ + w2 = w1; \ + w2 += w1; /* ALU32, BPF_X */ \ + w2 += 1; /* ALU32, BPF_K */ \ + w2 = w2; /* keep w2 alive for previous instruction */ \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* 64-bit definition is never zero extended */ +SEC("socket") +__success __log_level(2) +__not_msg("r1 = r0{{.*}}; zext") +__naked void no_zext_mov64(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = r0; \ + r0 = r1; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Narrow load result read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("r1 = *(u32 *)(r10 -8){{ +}}; zext") +__naked void zext_narrow_load_hi_used(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u64 *)(r10 - 8) = r0; \ + r1 = *(u32 *)(r10 - 8); \ + r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +/* 32-bit atomic fetch result read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1){{ +}}; zext") +__naked void zext_atomic_fetch32_hi_used(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + w1 = 1; \ + .8byte %[fetch_add32]; \ + r0 = r1; \ + exit; \ +" : + : __imm_insn(fetch_add32, + BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +/* 32-bit atomic cmpxchg result (r0) read as 64-bit -> zext */ +SEC("socket") +__success __log_level(2) +__msg("r0 = atomic_cmpxchg((u32 *)(r10 -8), r0, r1){{ +}}; zext") +__naked void zext_cmpxchg32_hi_used(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64 *)(r10 - 8) = r1; \ + w0 = 0; \ + w1 = 1; \ + .8byte %[cmpxchg32]; \ + r2 = r0; \ + r0 = r2; \ + exit; \ +" : + : __imm_insn(cmpxchg32, + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_10, BPF_REG_1, -8)) + : __clobber_all); +} + +/* 32-bit def before a branch, upper half used on one branch -> zext */ +SEC("socket") +__success __log_level(2) +__msg("w6 = 32{{ +}}; zext") +__naked void zext_cfg_hi_used_one_branch(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w6 = 32; \ + if r0 == 0 goto 1f; \ + r0 = r6; \ + exit; \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* r1's upper half is dead, so 'w1 = 1' must NOT be marked for zero extension. */ +SEC("socket") +__success __log_level(2) +__not_msg("w1 = 1{{.*}}; zext") +__naked void no_zext_other_reg_hi_used(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r6 = r0; \ + r6 <<= 32; \ + w1 = 1; \ + r0 = r6; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* LD_ABS defines r0; when r0 is read as 64-bit it must be zero extended */ +SEC("socket") +__success __log_level(2) +__msg("r0 = *(u8 *)skb[0]{{.*}}; zext") +__naked void zext_ld_abs_hi_used(void) +{ + asm volatile (" \ + r6 = r1; \ + r0 = *(u8 *)skb[0]; \ + r7 = r0; \ + r0 = r7; \ + exit; \ +" ::: __clobber_all); +} + +/* Helper parameters are read as 64-bit (call_use_mask() fallback) */ +SEC("socket") +__success __log_level(2) +__msg("w2 = 1{{ +}}; zext") +__naked void helper_param_read_as_64bit(void) +{ + asm volatile (" \ + r1 = r10; \ + r1 += -8; \ + w2 = 1; \ + call %[bpf_trace_printk]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_trace_printk) + : __clobber_all); +} + +static __used __naked int subprog_reads_arg_as_64bit(void) +{ + asm volatile (" \ + r0 = r1; \ + exit; \ +" ::: __clobber_all); +} + +/* subprogram parameters are conservatively read as 64-bit */ +SEC("socket") +__success __log_level(2) +__msg("w1 = w0{{ +}}; zext") +__naked void subprog_param_read_as_64bit(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + call subprog_reads_arg_as_64bit; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* kfunc parameters are zero extended */ +SEC("tc") +__success __log_level(2) +__msg("w1 = 1{{ +}}; zext") +__msg("w2 = 1{{ +}}; zext") +__msg("w3 = 1{{ +}}; zext") +__msg("w4 = 1{{ +}}; zext") +__naked void kfunc_param_read_per_btf(void) +{ + asm volatile (" \ + w1 = 1; \ + w2 = 1; \ + w3 = 1; \ + w4 = 1; \ + call bpf_kfunc_call_test4; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +SEC("socket") +__success __log_level(2) +__not_msg("; zext") +__naked void alu32_and_32bit_conditional(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + if w1 > 42 goto 1f; /* BPF_K */ \ + w2 = 28; \ + if w2 > w1 goto 1f; /* BPF_X */ \ + r0 = 0; \ +1: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success __log_level(2) +__msg("w1 = w0{{ +}}; zext") +__naked void alu32_and_64bit_conditional(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + w1 = w0; \ + if r1 > 42 goto 1f; /* BPF_K */ \ + r2 = 28; \ + if r2 > r1 goto 1f; /* BPF_X */ \ + r0 = 0; \ +1: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +SEC("socket") +__success __log_level(2) +__not_msg("; zext") +__naked void alu64_and_conditionals(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + r1 = r0; \ + if w1 > 42 goto 1f; /* BPF_K */ \ + if r1 > 42 goto 1f; /* BPF_K */ \ + r2 = 28; \ + if w2 > w1 goto 1f; /* BPF_X */ \ + if r2 > r1 goto 1f; /* BPF_X */ \ + r0 = 0; \ +1: exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") +__arch_s390x +__xlated("7: w1 = w0") +__xlated("8: w1 = w1") +__xlated("9: w1 += 8") +__xlated("10: w1 = w1") +__xlated("11: w2 = w1") +__xlated("12: w2 = w2") +__xlated("13: *(u64 *)(r1 +0) = r2") +__naked void arena_ptr(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r2 = 0; \ + r3 = 1; \ + r4 = 0; \ + r5 = 0; \ + call %[bpf_arena_alloc_pages]; \ + r1 = addr_space_cast(r0, 0, 1); /* needs zext */ \ + r1 += 8; /* needs zext */ \ + r2 = addr_space_cast(r1, 1, 0); /* needs zext because of BPF_F_NO_USER_CONV */ \ + *(u64 *)(r1 +0) = r2; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_arena_alloc_pages), + __imm_addr(arena) + : __clobber_all); +} + +#endif + +/* Check if probe mem loads keep their zero extension. */ +SEC("socket") +__success __log_level(2) +__arch_s390x +__xlated("3: r1 = *(u64 *)(r0 +0)") +__xlated("4: r2 = *(u32 *)(r0 +0)") +__xlated("5: w2 = w2") +__xlated("6: r3 = *(u16 *)(r0 +0)") +__xlated("7: w3 = w3") +__xlated("8: r4 = *(u8 *)(r0 +0)") +__xlated("9: w4 = w4") +__naked void probe_mem(void) +{ + asm volatile (" \ + r1 = 0; \ + r2 = 0; \ + call %[bpf_rdonly_cast]; \ + r1 = *(u64 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r2 = *(u32 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r3 = *(u16 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r4 = *(u8 *)(r0 + 0); /* BPF_PROBE_MEM */ \ + r0 = r1; /* make the registers used */ \ + r0 += r2; \ + r0 += r3; \ + r0 += r4; \ +1: exit; \ +" : + : __imm(bpf_rdonly_cast) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/veristat_bar.c b/tools/testing/selftests/bpf/progs/veristat_bar.c new file mode 100644 index 000000000000..83d2a2a1dfc9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/veristat_bar.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include "veristat_foo.c" diff --git a/tools/testing/selftests/bpf/progs/veristat_foo.c b/tools/testing/selftests/bpf/progs/veristat_foo.c new file mode 100644 index 000000000000..bd24b97664b4 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/veristat_foo.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +/* + * Programs below exist only to exercise veristat's -f name filters, + * their bodies are irrelevant, only the names matter. + * This file is also included by veristat_bar.c, so that the same set of + * program names is available in two differently named object files. + */ + +SEC("socket") +int foo(void *ctx) +{ + return 0; +} + +SEC("socket") +int bar(void *ctx) +{ + return 0; +} + +SEC("socket") +int buz(void *ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/xdp_dummy.c b/tools/testing/selftests/bpf/progs/xdp_dummy.c index d988b2e0cee8..5f1e0771021d 100644 --- a/tools/testing/selftests/bpf/progs/xdp_dummy.c +++ b/tools/testing/selftests/bpf/progs/xdp_dummy.c @@ -10,4 +10,10 @@ int xdp_dummy_prog(struct xdp_md *ctx) return XDP_PASS; } +SEC("xdp") +int __x64_sys_nop(struct xdp_md *ctx) +{ + return XDP_PASS; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 30f1cd23093c..9366a3c578f1 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -161,6 +161,33 @@ bpf_testmod_test_arg_ptr_to_struct(struct bpf_testmod_struct_arg_1 *a) { return bpf_testmod_test_struct_arg_result; } +#ifdef __SIZEOF_INT128__ +noinline __int128 +bpf_testmod_test_int128_ret(int a) +{ + bpf_testmod_test_struct_arg_result = a; + return (__int128)a; +} + +/* + * The __int128 'a' is the first argument on purpose. On arm64 a 16-byte + * argument must start in an even-numbered register pair, so placing it + * after a single-register scalar would leave a padding register (x1) + * unused. pahole maps parameters to registers positionally and would then + * see the following argument in an "unexpected" register and skip BTF + * encoding of the whole function, making it unattachable. Keeping the + * __int128 first (x0:x1) avoids the padding while still exercising the + * trampoline packing of a 128-bit argument together with the trailing + * int and long arguments. + */ +noinline long +bpf_testmod_test_int128_arg(__int128 a, int b, long c) +{ + bpf_testmod_test_struct_arg_result = (long)a + b + c; + return bpf_testmod_test_struct_arg_result; +} +#endif + __weak noinline void bpf_testmod_looooooooooooooooooooooooooooooong_name(void) { } @@ -210,6 +237,44 @@ __bpf_kfunc void bpf_kfunc_common_test(void) { } +__bpf_kfunc u64 bpf_kfunc_arena_arg_test(u64 *val__arena) +{ + u64 old; + + old = *val__arena; + *val__arena = old + 1; + return old; +} + +__bpf_kfunc u64 bpf_kfunc_arena_cap_test(u64 *val__arena) +{ + return (u64)val__arena; +} + +__bpf_kfunc u64 bpf_kfunc_arena_cap_nullable_test(u64 *val__arena__nullable) +{ + return (u64)val__arena__nullable; +} + +__bpf_kfunc u64 bpf_kfunc_arena_args5_test(u64 *a__arena, u64 *b__arena, + u64 *c__arena, u64 *d__arena, + u64 *e__arena__nullable) +{ + return *a__arena + *b__arena + *c__arena + *d__arena + + (e__arena__nullable ? *e__arena__nullable : 0); +} + +__bpf_kfunc u64 bpf_kfunc_arena_stack_arg_test(u64 a, u64 b, u64 c, u64 d, u64 e, + u64 *f__arena) +{ + return a + b + c + d + e + *f__arena; +} + +__bpf_kfunc u64 bpf_kfunc_arena_mixed_test(u64 *a__arena, u64 *b__arena__nullable) +{ + return *a__arena + (b__arena__nullable ? *b__arena__nullable : 0); +} + __bpf_kfunc void bpf_kfunc_dynptr_test(struct bpf_dynptr *ptr, struct bpf_dynptr *ptr__nullable) { @@ -320,9 +385,36 @@ static int bpf_testmod_test_4(void) return 0; } +static int bpf_testmod_ops3__test_arena(u64 *ptr__arena) +{ + return 0; +} + +static int bpf_testmod_ops3__test_arena_nullable(u64 *ptr__arena__nullable) +{ + return 0; +} + +static int bpf_testmod_ops3__test_arena_stack(u64 a, u64 b, u64 c, u64 d, + u64 e, u64 f, u64 g, u64 h, + u64 *ptr__arena) +{ + return 0; +} + +static int bpf_testmod_ops3__test_arena_multislot(struct bpf_testmod_arena_pair p, + u64 *ptr__arena) +{ + return 0; +} + static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_1 = bpf_testmod_test_3, .test_2 = bpf_testmod_test_4, + .test_arena = bpf_testmod_ops3__test_arena, + .test_arena_nullable = bpf_testmod_ops3__test_arena_nullable, + .test_arena_stack = bpf_testmod_ops3__test_arena_stack, + .test_arena_multislot = bpf_testmod_ops3__test_arena_multislot, }; static void bpf_testmod_test_struct_ops3(void) @@ -341,6 +433,28 @@ __bpf_kfunc void bpf_testmod_ops3_call_test_2(void) st_ops3->test_2(); } +__bpf_kfunc int bpf_testmod_ops3_call_test_arena(u64 *ptr__arena) +{ + return st_ops3->test_arena(ptr__arena); +} + +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_nullable(u64 *ptr__arena__nullable) +{ + return st_ops3->test_arena_nullable(ptr__arena__nullable); +} + +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_stack(u64 *ptr__arena) +{ + return st_ops3->test_arena_stack(1, 2, 3, 4, 5, 6, 7, 8, ptr__arena); +} + +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_multislot(u64 *ptr__arena) +{ + struct bpf_testmod_arena_pair p = { .a = 11, .b = 22 }; + + return st_ops3->test_arena_multislot(p, ptr__arena); +} + struct bpf_testmod_btf_type_tag_1 { int a; }; @@ -514,6 +628,11 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, (void)bpf_testmod_test_arg_ptr_to_struct(&struct_arg1_2); +#ifdef __SIZEOF_INT128__ + (void)bpf_testmod_test_int128_ret(i); + (void)bpf_testmod_test_int128_arg((__int128)1, 2, 3); +#endif + (void)trace_bpf_testmod_test_raw_tp_null_tp(NULL); bpf_testmod_test_struct_ops3(); @@ -723,6 +842,12 @@ BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_value) BTF_ID_FLAGS(func, bpf_kfunc_common_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_arg_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_nullable_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_args5_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_stack_arg_test) +BTF_ID_FLAGS(func, bpf_kfunc_arena_mixed_test) BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1) BTF_ID_FLAGS(func, bpf_kfunc_dynptr_test) BTF_ID_FLAGS(func, bpf_kfunc_nested_acquire_nonzero_offset_test, KF_ACQUIRE) @@ -738,6 +863,10 @@ BTF_ID_FLAGS(func, bpf_testmod_ctx_create, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_testmod_ctx_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_1) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_2) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_nullable) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids) @@ -1352,7 +1481,7 @@ __bpf_kfunc void bpf_kfunc_trigger_ctx_check(void) } BTF_KFUNCS_START(bpf_testmod_check_kfunc_ids) -BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc) +BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc, KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_kfunc_call_test1) BTF_ID_FLAGS(func, bpf_kfunc_call_test2) BTF_ID_FLAGS(func, bpf_kfunc_call_test3) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 863fd10f1619..210b919290cc 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -103,9 +103,23 @@ struct bpf_testmod_ops2 { int (*test_1)(void); }; +/* 16 bytes, so it takes two argument slots when passed by value */ +struct bpf_testmod_arena_pair { + u64 a; + u64 b; +}; + struct bpf_testmod_ops3 { int (*test_1)(void); int (*test_2)(void); + /* Used to test arena pointer arguments. */ + int (*test_arena)(u64 *ptr); + int (*test_arena_nullable)(u64 *ptr); + /* enough leading args to force @ptr onto the stack on x86 and arm64 */ + int (*test_arena_stack)(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, + u64 g, u64 h, u64 *ptr); + /* a multi-slot leading arg, so @ptr is not at the slot its arg index suggests */ + int (*test_arena_multislot)(struct bpf_testmod_arena_pair p, u64 *ptr); }; struct st_ops_args { diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index c36bb911defa..7d81070eefe7 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -98,6 +98,15 @@ void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) __ksym; void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p) __ksym; void bpf_kfunc_call_test_mem_len_pass1(void *mem, int len) __ksym; +__u64 bpf_kfunc_arena_arg_test(__u64 *val__arena) __ksym; +__u64 bpf_kfunc_arena_cap_test(__u64 *val__arena) __ksym; +__u64 bpf_kfunc_arena_cap_nullable_test(__u64 *val__arena__nullable) __ksym; +__u64 bpf_kfunc_arena_args5_test(__u64 *a__arena, __u64 *b__arena, + __u64 *c__arena, __u64 *d__arena, + __u64 *e__arena__nullable) __ksym; +__u64 bpf_kfunc_arena_stack_arg_test(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, + __u64 *f__arena) __ksym; +__u64 bpf_kfunc_arena_mixed_test(__u64 *a__arena, __u64 *b__arena__nullable) __ksym; int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size) __ksym; int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; @@ -111,6 +120,10 @@ u32 bpf_kfunc_call_test_static_unused_arg(u32 arg, u32 unused) __ksym; #endif void bpf_testmod_test_mod_kfunc(int i) __ksym; +int bpf_testmod_ops3_call_test_arena(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_arena_nullable(__u64 *ptr__arena__nullable) __ksym; +int bpf_testmod_ops3_call_test_arena_stack(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_arena_multislot(__u64 *ptr__arena) __ksym; __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b, __u32 c, __u64 d) __ksym; diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 3ce32d134e2c..07807757b518 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -807,7 +807,7 @@ static void verify_stderr(int prog_fd, struct expected_msgs *msgs) return; buf = malloc(TEST_LOADER_LOG_BUF_SZ); - if (!ASSERT_OK_PTR(buf, "malloc")) + if (!ASSERT_NEQ(buf, NULL, "malloc")) return; ret = bpf_prog_stream_read(prog_fd, 2, buf, TEST_LOADER_LOG_BUF_SZ - 1, diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 7ba82974ee78..46eb201b96a3 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -424,10 +424,12 @@ static void jsonw_write_log_message(json_writer_t *w, char *log_buf, size_t log_ } } +/* @quiet elides the human readable output, the JSON report is unaffected */ static void dump_test_log(const struct prog_test_def *test, const struct test_state *test_state, bool skip_ok_subtests, bool par_exec_result, + bool quiet, json_writer_t *w) { bool test_failed = test_state->error_cnt > 0; @@ -449,7 +451,7 @@ static void dump_test_log(const struct prog_test_def *test, if (verbose() && !par_exec_result) return; - if (test_state->log_cnt && print_test) + if (test_state->log_cnt && print_test && !quiet) print_test_log(test_state->log_buf, test_state->log_cnt); if (w && print_test) { @@ -471,15 +473,16 @@ static void dump_test_log(const struct prog_test_def *test, if ((skip_ok_subtests && !subtest_failed) || subtest_filtered) continue; - if (subtest_state->log_cnt && print_subtest) { + if (subtest_state->log_cnt && print_subtest && !quiet) { print_test_log(subtest_state->log_buf, subtest_state->log_cnt); } - print_subtest_name(test->test_num, i + 1, - test->test_name, subtest_state->name, - test_result(subtest_state->error_cnt, - subtest_state->skipped)); + if (!quiet) + print_subtest_name(test->test_num, i + 1, + test->test_name, subtest_state->name, + test_result(subtest_state->error_cnt, + subtest_state->skipped)); if (w && print_subtest) { jsonw_start_object(w); @@ -496,7 +499,8 @@ static void dump_test_log(const struct prog_test_def *test, jsonw_end_object(w); } - print_test_result(test, test_state); + if (!quiet) + print_test_result(test, test_state); } /* A bunch of tests set custom affinity per-thread and/or per-process. Reset @@ -573,18 +577,19 @@ bool test__start_subtest_with_desc(const char *subtest_name, const char *subtest struct subtest_state *subtest_state; const char *subtest_display_name; size_t sub_state_size = sizeof(*subtest_state); + void *tmp; if (env.subtest_state) test__end_subtest(); state->subtest_num++; - state->subtest_states = - realloc(state->subtest_states, - state->subtest_num * sub_state_size); - if (!state->subtest_states) { + tmp = realloc(state->subtest_states, state->subtest_num * sub_state_size); + if (!tmp) { + state->subtest_num--; fprintf(stderr, "Not enough memory to allocate subtest result\n"); return false; } + state->subtest_states = tmp; subtest_state = &state->subtest_states[state->subtest_num - 1]; @@ -730,11 +735,14 @@ int compare_map_keys(int map1_fd, int map2_fd) int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len) { __u32 key, next_key, *cur_key_p, *next_key_p; - char *val_buf1, *val_buf2; - int i, err = 0; + char *val_buf1 = NULL, *val_buf2 = NULL; + int i, err = -ENOMEM; val_buf1 = malloc(stack_trace_len); val_buf2 = malloc(stack_trace_len); + if (!val_buf1 || !val_buf2) + goto out; + err = 0; cur_key_p = NULL; next_key_p = &key; while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) { @@ -895,6 +903,7 @@ enum ARG_KEYS { ARG_JSON_SUMMARY = 'J', ARG_TRAFFIC_MONITOR = 'm', ARG_WATCHDOG_TIMEOUT = 'w', + ARG_NO_ERROR_SUMMARY = -2, }; static const struct argp_option opts[] = { @@ -927,6 +936,8 @@ static const struct argp_option opts[] = { #endif { "watchdog-timeout", ARG_WATCHDOG_TIMEOUT, "SECONDS", 0, "Kill the process if tests are not making progress for specified number of seconds." }, + { "no-error-summary", ARG_NO_ERROR_SUMMARY, NULL, 0, + "Do not re-print the aggregated error logs of failed tests at the end of the run." }, {}, }; @@ -1128,6 +1139,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case ARG_DEBUG: env->debug = true; break; + case ARG_NO_ERROR_SUMMARY: + env->error_summary = false; + break; case ARG_JSON_SUMMARY: env->json = fopen(arg, "w"); if (env->json == NULL) { @@ -1300,7 +1314,7 @@ static void dump_crash_log(void) if (env.test) { env.test_state->error_cnt++; - dump_test_log(env.test, env.test_state, true, false, NULL); + dump_test_log(env.test, env.test_state, true, false, false, NULL); } } @@ -1458,7 +1472,7 @@ static void run_one_test(int test_num) free(stop_libbpf_log_capture()); - dump_test_log(test, state, false, false, NULL); + dump_test_log(test, state, false, false, false, NULL); } struct dispatch_data { @@ -1513,13 +1527,15 @@ static int dispatch_thread_send_subtests(int sock_fd, struct test_state *state) struct subtest_state *subtest_state; int subtest_num = state->subtest_num; - state->subtest_states = malloc(subtest_num * sizeof(*subtest_state)); + state->subtest_states = calloc(subtest_num, sizeof(*subtest_state)); + if (!state->subtest_states) { + state->subtest_num = 0; + return -ENOMEM; + } for (int i = 0; i < subtest_num; i++) { subtest_state = &state->subtest_states[i]; - memset(subtest_state, 0, sizeof(*subtest_state)); - if (read_prog_test_msg(sock_fd, &msg, MSG_SUBTEST_DONE)) return 1; @@ -1617,7 +1633,7 @@ static void *dispatch_thread(void *ctx) } while (false); pthread_mutex_lock(&stdout_output_lock); - dump_test_log(test, state, false, true, NULL); + dump_test_log(test, state, false, true, false, NULL); pthread_mutex_unlock(&stdout_output_lock); } /* while (true) */ error: @@ -1640,8 +1656,8 @@ done: static void calculate_summary_and_print_errors(struct test_env *env) { - int i; - int succ_cnt = 0, fail_cnt = 0, sub_succ_cnt = 0, skip_cnt = 0; + int i, j; + int succ_cnt = 0, fail_cnt = 0, sub_succ_cnt = 0, sub_fail_cnt = 0, skip_cnt = 0; json_writer_t *w = NULL; for (i = 0; i < prog_test_cnt; i++) { @@ -1654,10 +1670,14 @@ static void calculate_summary_and_print_errors(struct test_env *env) sub_succ_cnt += state->sub_succ_cnt; skip_cnt += state->skip_cnt; - if (state->error_cnt) + if (state->error_cnt) { fail_cnt++; - else if (!test->not_built) + for (j = 0; j < state->subtest_num; j++) + if (state->subtest_states[j].error_cnt) + sub_fail_cnt++; + } else if (!test->not_built) { succ_cnt++; + } } if (env->json) { @@ -1672,6 +1692,7 @@ static void calculate_summary_and_print_errors(struct test_env *env) jsonw_uint_field(w, "success_subtest", sub_succ_cnt); jsonw_uint_field(w, "skipped", skip_cnt); jsonw_uint_field(w, "failed", fail_cnt); + jsonw_uint_field(w, "failed_subtest", sub_fail_cnt); jsonw_name(w, "results"); jsonw_start_array(w); } @@ -1680,9 +1701,14 @@ static void calculate_summary_and_print_errors(struct test_env *env) * We only print error logs summary when there are failed tests and * verbose mode is not enabled. Otherwise, results may be inconsistent. * + * --no-error-summary elides the human readable dump. The walk still + * happens when a JSON report was requested, so the JSON output keeps + * its per-test results; with no JSON report there is nothing left to + * do and the whole loop is skipped. */ - if (!verbose() && fail_cnt) { - printf("\nAll error logs:\n"); + if (!verbose() && fail_cnt && (env->error_summary || w)) { + if (env->error_summary) + printf("\nAll error logs:\n"); /* print error logs again */ for (i = 0; i < prog_test_cnt; i++) { @@ -1692,7 +1718,8 @@ static void calculate_summary_and_print_errors(struct test_env *env) if (!state->tested || !state->error_cnt) continue; - dump_test_log(test, state, true, true, w); + dump_test_log(test, state, true, true, + !env->error_summary, w); } } @@ -1706,12 +1733,12 @@ static void calculate_summary_and_print_errors(struct test_env *env) fclose(env->json); if (env->not_built_cnt) - printf("Summary: %d/%d PASSED, %d SKIPPED (%d not built), %d FAILED\n", + printf("Summary: %d/%d PASSED, %d SKIPPED (%d not built), %d/%d FAILED\n", succ_cnt, sub_succ_cnt, skip_cnt, env->not_built_cnt, - fail_cnt); + fail_cnt, sub_fail_cnt); else - printf("Summary: %d/%d PASSED, %d SKIPPED, %d FAILED\n", - succ_cnt, sub_succ_cnt, skip_cnt, fail_cnt); + printf("Summary: %d/%d PASSED, %d SKIPPED, %d/%d FAILED\n", + succ_cnt, sub_succ_cnt, skip_cnt, fail_cnt, sub_fail_cnt); env->succ_cnt = succ_cnt; env->sub_succ_cnt = sub_succ_cnt; @@ -1741,7 +1768,7 @@ static void server_main(void) data[i].worker_id = i; data[i].sock_fd = env.worker_socks[i]; rc = pthread_create(&dispatcher_threads[i], NULL, dispatch_thread, &data[i]); - if (rc < 0) { + if (rc) { perror("Failed to launch dispatcher thread"); exit(EXIT_ERR_SETUP_INFRA); } @@ -1886,7 +1913,6 @@ static int worker_main_send_subtests(int sock, struct test_state *state) worker_main_send_log(sock, subtest_state->log_buf, subtest_state->log_cnt); free_subtest_state(subtest_state); - free(subtest_state->name); } out: @@ -2023,6 +2049,7 @@ int main(int argc, char **argv) env.secs_till_notify = 10; env.secs_till_kill = 120; + env.error_summary = true; err = argp_parse(&argp, argc, argv, 0, NULL, &env); if (err) return err; diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index 2cf950afcd85..ea493c477fbd 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -105,6 +105,7 @@ struct test_env { struct test_selector tmon_selector; bool verifier_stats; bool debug; + bool error_summary; enum verbosity verbosity; bool jit_enabled; @@ -123,7 +124,7 @@ struct test_env { int succ_cnt; /* successful tests */ int sub_succ_cnt; /* successful sub-tests */ - int fail_cnt; /* total failed tests + sub-tests */ + int fail_cnt; /* failed tests */ int skip_cnt; /* skipped tests */ int not_built_cnt; /* tests not built */ diff --git a/tools/testing/selftests/bpf/test_sockmap.c b/tools/testing/selftests/bpf/test_sockmap.c index ac814eb63edb..aaf2050e8845 100644 --- a/tools/testing/selftests/bpf/test_sockmap.c +++ b/tools/testing/selftests/bpf/test_sockmap.c @@ -435,7 +435,8 @@ static int msg_alloc_iov(struct msghdr *msg, return 0; unwind_iov: for (i--; i >= 0 ; i--) - free(msg->msg_iov[i].iov_base); + free(iov[i].iov_base); + free(iov); return -ENOMEM; } diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index a8ae03c57bba..bffb7360434c 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -1560,7 +1560,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv, opts.expected_attach_type = test->expected_attach_type; if (expected_ret == VERBOSE_ACCEPT) - opts.log_level = 2; + opts.log_level = 2 | 4; else if (verbose) opts.log_level = verif_log_level | 4; /* force stats */ else diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index 302d712e0d7e..eb6e3baef412 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -31,7 +31,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 pointer type STRUCT prog_test_fail1 must point to scalar", + .errstr = "R1 is fp expected STRUCT prog_test_fail1", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail1", 2 }, }, @@ -46,7 +46,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "max struct nesting depth exceeded\nR1 pointer type STRUCT prog_test_fail2", + .errstr = "max struct nesting depth exceeded\nR1 is fp expected STRUCT prog_test_fail2", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail2", 2 }, }, @@ -61,7 +61,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, - .errstr = "R1 pointer type STRUCT prog_test_fail3 must point to scalar", + .errstr = "R1 is fp expected STRUCT prog_test_fail3", .fixup_kfunc_btf_id = { { "bpf_kfunc_call_test_fail3", 2 }, }, @@ -1091,7 +1091,17 @@ /* stack_main=32, stack_A=256, stack_B=64 * and max(main+A, main+A+B) < 512 */ - .result = ACCEPT, + .result = VERBOSE_ACCEPT, + .errstr = "stack depth max 352\t" + "subprog 0 (<unknown>) main insns_self \t" + " insns_total \t" + " stack 32\t" + "subprog 1 (<unknown>) static insns_self \t" + " insns_total \t" + " stack 256\t" + "subprog 2 (<unknown>) static insns_self \t" + " insns_total \t" + " stack 64", }, { "calls: stack depth check using three frames. test2", diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index a7db6f04f7e1..e70741c6b9b7 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -514,6 +514,40 @@ cleanup: return err == 0; } +/* Exact filter match */ +static bool name_filter_matches(struct filter *f, const char *filename, const char *prog_name) +{ + if (f->any_glob) + return glob_matches(filename, f->any_glob) || + (prog_name && glob_matches(prog_name, f->any_glob)); + if (f->file_glob && f->prog_glob) + return prog_name && + glob_matches(filename, f->file_glob) && + glob_matches(prog_name, f->prog_glob); + if (f->file_glob) + return glob_matches(filename, f->file_glob); + if (f->prog_glob) + return prog_name && glob_matches(prog_name, f->prog_glob); + return false; +} + +/* Check if the filter does not outright reject the file name */ +static bool name_filter_may_match(struct filter *f, const char *filename) +{ + if (f->file_glob) + return glob_matches(filename, f->file_glob); + /* + * If we don't know program name yet, any_glob filter + * has to assume that current BPF object file might be + * relevant; we'll check again later on after opening + * BPF object file, at which point program name will + * be known finally. + */ + if (f->any_glob || f->prog_glob) + return true; + return false; +} + static bool should_process_file_prog(const char *filename, const char *prog_name) { struct filter *f; @@ -521,16 +555,7 @@ static bool should_process_file_prog(const char *filename, const char *prog_name for (i = 0; i < env.deny_filter_cnt; i++) { f = &env.deny_filters[i]; - if (f->kind != FILTER_NAME) - continue; - - if (f->any_glob && glob_matches(filename, f->any_glob)) - return false; - if (f->any_glob && prog_name && glob_matches(prog_name, f->any_glob)) - return false; - if (f->file_glob && glob_matches(filename, f->file_glob)) - return false; - if (f->prog_glob && prog_name && glob_matches(prog_name, f->prog_glob)) + if (f->kind == FILTER_NAME && name_filter_matches(f, filename, prog_name)) return false; } @@ -540,24 +565,15 @@ static bool should_process_file_prog(const char *filename, const char *prog_name continue; allow_cnt++; - if (f->any_glob) { - if (glob_matches(filename, f->any_glob)) - return true; - /* If we don't know program name yet, any_glob filter - * has to assume that current BPF object file might be - * relevant; we'll check again later on after opening - * BPF object file, at which point program name will - * be known finally. - */ - if (!prog_name || glob_matches(prog_name, f->any_glob)) - return true; - } else { - if (f->file_glob && !glob_matches(filename, f->file_glob)) - continue; - if (f->prog_glob && prog_name && !glob_matches(prog_name, f->prog_glob)) - continue; + if (prog_name && name_filter_matches(f, filename, prog_name)) + return true; + /* + * If there is no prog_name and the file name is not blocked by + * the filter, allow to open the file. Afterwards there would be + * a second refining query with prog_name set. + */ + if (!prog_name && name_filter_may_match(f, filename)) return true; - } } /* if there are no file/prog name allow filters, allow all progs, @@ -703,6 +719,12 @@ static int append_filter(struct filter **filters, int *cnt, const char *str) } } + if ((!f->any_glob && !f->file_glob && !f->prog_glob) || + (f->any_glob && strcmp(f->any_glob, "") == 0)) { + fprintf(stderr, "Invalid filter: '%s'\n", str); + return -EINVAL; + } + *cnt += 1; return 0; } @@ -993,13 +1015,15 @@ static void free_verif_stats(struct verif_stats *stats, size_t stat_cnt) static char verif_log_buf[64 * 1024]; -#define MAX_PARSED_LOG_LINES 100 +/* Keep room for all 256 subprogram records and trailing statistics. */ +#define MAX_PARSED_LOG_LINES 300 static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s) { const char *cur; - int pos, lines, sub_stack, cnt = 0; - char *state = NULL, *token, stack[512]; + long sub_stack; + int pos, lines, cnt = 0; + char *state = NULL, *token, stack[512] = {}; buf[buf_sz - 1] = '\0'; @@ -1025,11 +1049,24 @@ static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats * &s->stats[MARK_READ_MAX_LEN])) continue; + /* + * New kernels emit one "subprog <id> (<name>) <kind>" record + * per subprogram with the stack depth at the end, while old + * kernels emit a single "stack depth <a+...+n> max <max>" + * line. Match both formats so veristat works against either + * kernel. + */ + if (sscanf(cur, "stack depth max %ld", &s->stats[MAX_STACK]) == 1) + continue; + if (sscanf(cur, "subprog %*d %*s %*s insns_self %*d insns_total %*d stack %ld", &sub_stack) == 1) { + s->stats[STACK] += sub_stack; + continue; + } if (2 == sscanf(cur, "stack depth %511s max %ld", stack, &s->stats[MAX_STACK])) continue; } while ((token = strtok_r(cnt++ ? NULL : stack, "+", &state))) { - if (sscanf(token, "%d", &sub_stack) == 0) + if (sscanf(token, "%ld", &sub_stack) == 0) break; s->stats[STACK] += sub_stack; } @@ -1248,6 +1285,29 @@ static void fixup_obj_maps(struct bpf_object *obj) /* fix up map size, if necessary */ switch (bpf_map__type(map)) { + /* + * if the verifier doesn't use max_entries + * then set to 1 to avoid -ENOMEM + */ + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_SOCKHASH: + case BPF_MAP_TYPE_DEVMAP_HASH: + case BPF_MAP_TYPE_QUEUE: + case BPF_MAP_TYPE_STACK: + case BPF_MAP_TYPE_BLOOM_FILTER: + case BPF_MAP_TYPE_STACK_TRACE: + bpf_map__set_max_entries(map, 1); + break; + + /* ringbufs must be page-aligned */ + case BPF_MAP_TYPE_RINGBUF: + case BPF_MAP_TYPE_USER_RINGBUF: + bpf_map__set_max_entries(map, sysconf(_SC_PAGESIZE)); + break; + case BPF_MAP_TYPE_SK_STORAGE: case BPF_MAP_TYPE_TASK_STORAGE: case BPF_MAP_TYPE_INODE_STORAGE: diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 9ca802285393..6a3d026d76bd 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -428,8 +428,17 @@ main() if [[ $# -eq 0 && "${debug_shell}" == "no" ]]; then echo "No command specified, will run ${DEFAULT_COMMAND} in the vm" - else - command="$@" + elif [[ $# -gt 0 ]]; then + # Quote each argument so the command survives into the guest: the + # host expands ${command} into the generated init script, which + # the guest bash then parses as shell source. Without the %q + # escapes an argument with a space or a glob (e.g. -a 'verifier_*') + # is re-split and expanded against /root/bpf there. + # + # Skip this when there is no command: printf '%q ' would still + # apply the format once and emit '', which is not the empty + # command that -s (debug shell) expects. + command=$(printf '%q ' "$@") fi local kconfig_file="${OUTPUT_DIR}/latest.config" |