From a6f0643e4f63cfaa0d5d4a69de4f132eac4b8fe4 Mon Sep 17 00:00:00 2001 From: Matt Bobrowski Date: Sun, 28 Jun 2026 20:11:03 +0000 Subject: bpf: Reject BPF_MAP_TYPE_INODE_STORAGE creation if BPF LSM is uninitialized When CONFIG_BPF_LSM=y is set, BPF inode storage maps (BPF_MAP_TYPE_INODE_STORAGE) are compiled into the kernel. However, if the BPF LSM is not explicitly enabled at boot time (e.g. omitted from the "lsm=" boot parameter), lsm_prepare() is never executed for the BPF LSM. Consequently, the BPF inode security blob offset (bpf_lsm_blob_sizes.lbs_inode) is never initialized and remains at its default compiled size of 8 bytes instead of being updated to a valid offset past the reserved struct rcu_head (typically 16 bytes or more). When a privileged user creates and updates a BPF_MAP_TYPE_INODE_STORAGE map, bpf_inode() evaluates inode->i_security + 8. This erroneously aliases the struct rcu_head.func callback pointer at the beginning of the inode->i_security blob. During subsequent map element cleanup or inode destruction, writing NULL to owner_storage clears the queued RCU callback pointer. When rcu_do_batch() later executes the queued callback, it attempts an instruction fetch at address 0x0, triggering an immediate kernel panic. Fix this by introducing a global bpf_lsm_initialized boolean flag marked with __ro_after_init. Set this flag to true inside bpf_lsm_init() when the LSM framework successfully registers the BPF LSM. Gate map allocation in inode_storage_map_alloc() on this flag, returning -EOPNOTSUPP if the BPF LSM is in turn uninitialized. This fail-fast approach prevents userspace from allocating inode storage maps when the supporting BPF LSM infrastructure is absent, avoiding zombie map states. Fixes: 8ea636848aca ("bpf: Implement bpf_local_storage for inodes") Reported-by: oxsignal Signed-off-by: Matt Bobrowski Signed-off-by: Daniel Borkmann Reviewed-by: Emil Tsalapatis Reviewed-by: Amery Hung Link: https://lore.kernel.org/bpf/20260628201103.3624525-1-mattbobrowski@google.com --- include/linux/bpf_lsm.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include') diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index 143775a27a2a..dda272d78f01 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -14,6 +14,8 @@ #ifdef CONFIG_BPF_LSM +extern bool bpf_lsm_initialized __ro_after_init; + #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ RET bpf_lsm_##NAME(__VA_ARGS__); #include @@ -56,6 +58,8 @@ bool bpf_lsm_hook_returns_errno(u32 btf_id); #else /* !CONFIG_BPF_LSM */ +#define bpf_lsm_initialized false + static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id) { return false; -- cgit From 96cce16e26dd02a8678f1e87f88a4b5cdb63b995 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:37:52 -0700 Subject: bpf: Support for hardening against JIT spraying The BPF JIT allocator packs many small programs into larger executable allocations and reuses space within those allocations as programs are loaded and freed. When fresh code is written into space that a previous program occupied, an indirect jump into the new program can reuse a branch prediction left behind by the old one. Flush the indirect branch predictors before reusing JIT memory so that indirect jumps into a newly written program don't reuse predictions from an old program that occupied the same space. Introduce bpf_arch_pred_flush_enabled static key and bpf_arch_pred_flush static call for flushing the branch predictors on JIT memory reuse. Architectures that need a flush, can update it to a predictor flush function. By default, its a NOP and does not emit any CALL. Allocations larger than a pack are not covered by this flush. That is safe because cBPF programs (the unprivileged attack surface) are bounded well below a pack size. Issue a warning if this assumption is ever violated while the flush is active. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- include/linux/filter.h | 10 ++++++++++ kernel/bpf/core.c | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+) (limited to 'include') diff --git a/include/linux/filter.h b/include/linux/filter.h index 67d337ede91b..f68694f94ee7 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -1314,6 +1315,15 @@ extern long bpf_jit_limit_max; typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size); +/* + * Flush the indirect branch predictors before reusing JIT memory, so that + * indirect jumps into a newly written program don't reuse predictions left + * behind by an old program that occupied the same space. + */ +void bpf_arch_pred_flush(void); +DECLARE_STATIC_CALL(bpf_arch_pred_flush, bpf_arch_pred_flush); +DECLARE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + void bpf_jit_fill_hole_with_zero(void *area, unsigned int size); struct bpf_binary_header * diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 649cce41e13f..7f0a17f128d4 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -883,6 +884,15 @@ void bpf_jit_fill_hole_with_zero(void *area, unsigned int size) memset(area, 0, size); } +DEFINE_STATIC_CALL_NULL(bpf_arch_pred_flush, bpf_arch_pred_flush); + +/* + * Enabled once bpf_arch_pred_flush points at a real flush routine. Lets the + * pack allocator test "is a predictor flush wired up at all" with a cheap + * static branch instead of repeatedly querying the static call target. + */ +DEFINE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) static DEFINE_MUTEX(pack_mutex); @@ -941,6 +951,14 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) mutex_lock(&pack_mutex); if (size > BPF_PROG_PACK_SIZE) { + /* + * Allocations larger than a pack get their own pages, and + * predictors are not flushed for such allocation. This is only + * safe because cBPF programs (the unprivileged attack surface) + * are bounded well below a pack size. + */ + if (static_branch_unlikely(&bpf_pred_flush_enabled)) + pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); if (ptr) { @@ -971,6 +989,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) pos = 0; found_free_area: + static_call_cond(bpf_arch_pred_flush)(); bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); -- cgit From 0bb99f2cfaae6822d734d69722de30af823efdf3 Mon Sep 17 00:00:00 2001 From: Pawan Gupta Date: Mon, 29 Jun 2026 22:38:23 -0700 Subject: bpf: Restrict JIT predictor flush to cBPF Currently predictor flush on memory reuse is done for all BPF JIT allocations, but only cBPF programs can be loaded by an unprivileged user. eBPF is privileged by default, and flushing predictors for all CPUs on every eBPF reuse penalizes the common case for no security benefit. eBPF allocations can be frequent on busy systems, only flush predictors for cBPF programs. Trampoline and dispatcher allocations also skip the flush as they are eBPF-only. Signed-off-by: Pawan Gupta Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- arch/arm64/net/bpf_jit_comp.c | 4 ++-- arch/loongarch/net/bpf_jit.c | 5 +++-- arch/powerpc/net/bpf_jit_comp.c | 4 ++-- arch/riscv/net/bpf_jit_comp64.c | 2 +- arch/riscv/net/bpf_jit_core.c | 3 ++- arch/x86/net/bpf_jit_comp.c | 5 +++-- include/linux/filter.h | 5 +++-- kernel/bpf/core.c | 13 ++++++++----- kernel/bpf/dispatcher.c | 2 +- 9 files changed, 25 insertions(+), 18 deletions(-) (limited to 'include') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index f6bcc0e1a950..b0075ece4a6e 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2177,7 +2177,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr image_size = extable_offset + extable_size; ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u64), &header, &image_ptr, - jit_fill_hole); + jit_fill_hole, was_classic); if (!ro_header) goto out_off; @@ -2870,7 +2870,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, jit_fill_hole); + return bpf_prog_pack_alloc(size, jit_fill_hole, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 058ffbbaad85..3f3f0335d63c 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1762,7 +1762,7 @@ static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, jit_fill_hole); + return bpf_prog_pack_alloc(size, jit_fill_hole, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) @@ -2228,7 +2228,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr image_size = prog_size + extable_size; /* Now we know the size of the structure to make */ ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32), - &header, &image_ptr, jit_fill_hole); + &header, &image_ptr, jit_fill_hole, + bpf_prog_was_classic(prog)); if (!ro_header) goto out_offset; diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index d4a17e18c9fb..7b07b43575f1 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -295,7 +295,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, - bpf_jit_fill_ill_insns); + bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp)); if (!fhdr) goto out_err; @@ -588,7 +588,7 @@ bool bpf_jit_inlines_helper_call(s32 imm) void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns); + return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index c03c1de16b79..f9d5347ba966 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1321,7 +1321,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, bpf_fill_ill_insns); + return bpf_prog_pack_alloc(size, bpf_fill_ill_insns, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 4365d07aaf54..ce3bd3762e08 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -109,7 +109,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr bpf_jit_binary_pack_alloc(prog_size + extable_size, &jit_data->ro_image, sizeof(u32), &jit_data->header, &jit_data->image, - bpf_fill_ill_insns); + bpf_fill_ill_insns, + bpf_prog_was_classic(prog)); if (!jit_data->ro_header) goto out_offset; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 054e043ffcd2..de7515ea1bea 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3653,7 +3653,7 @@ cleanup: void *arch_alloc_bpf_trampoline(unsigned int size) { - return bpf_prog_pack_alloc(size, jit_fill_hole); + return bpf_prog_pack_alloc(size, jit_fill_hole, false); } void arch_free_bpf_trampoline(void *image, unsigned int size) @@ -3965,7 +3965,8 @@ out_image: /* allocate module memory for x86 insns and extable */ header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size, &image, align, &rw_header, &rw_image, - jit_fill_hole); + jit_fill_hole, + bpf_prog_was_classic(prog)); if (!header) goto out_addrs; prog->aux->extable = (void *) image + roundup(proglen, align); diff --git a/include/linux/filter.h b/include/linux/filter.h index f68694f94ee7..14acb2455746 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1338,7 +1338,7 @@ void bpf_jit_free(struct bpf_prog *fp); struct bpf_binary_header * bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns); +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic); void bpf_prog_pack_free(void *ptr, u32 size); static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) @@ -1352,7 +1352,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image, unsigned int alignment, struct bpf_binary_header **rw_hdr, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns); + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic); int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header, struct bpf_binary_header *rw_header); void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 7f0a17f128d4..1614ccc3f111 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -942,7 +942,7 @@ out: return NULL; } -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic) { unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); struct bpf_prog_pack *pack; @@ -957,7 +957,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) * safe because cBPF programs (the unprivileged attack surface) * are bounded well below a pack size. */ - if (static_branch_unlikely(&bpf_pred_flush_enabled)) + if (was_classic && static_branch_unlikely(&bpf_pred_flush_enabled)) pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n"); size = round_up(size, PAGE_SIZE); ptr = bpf_jit_alloc_exec(size); @@ -989,7 +989,9 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) pos = 0; found_free_area: - static_call_cond(bpf_arch_pred_flush)(); + /* Flush only for cBPF as it may contain a crafted gadget */ + if (static_branch_unlikely(&bpf_pred_flush_enabled) && was_classic) + static_call_cond(bpf_arch_pred_flush)(); bitmap_set(pack->bitmap, pos, nbits); ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT); @@ -1149,7 +1151,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, unsigned int alignment, struct bpf_binary_header **rw_header, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns) + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic) { struct bpf_binary_header *ro_header; u32 size, hole, start; @@ -1162,7 +1165,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, if (bpf_jit_charge_modmem(size)) return NULL; - ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns); + ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns, was_classic); if (!ro_header) { bpf_jit_uncharge_modmem(size); return NULL; diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index b77db7413f8c..ea2d60dc1fee 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -145,7 +145,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, mutex_lock(&d->mutex); if (!d->image) { - d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero); + d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false); if (!d->image) goto out; d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); -- cgit