mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2024-12-28 16:53:49 +00:00
BPF fixes:
- Fix a bug in the BPF verifier to track changes to packet data property for global functions (Eduard Zingerman) - Fix a theoretical BPF prog_array use-after-free in RCU handling of __uprobe_perf_func (Jann Horn) - Fix BPF tracing to have an explicit list of tracepoints and their arguments which need to be annotated as PTR_MAYBE_NULL (Kumar Kartikeya Dwivedi) - Fix a logic bug in the bpf_remove_insns code where a potential error would have been wrongly propagated (Anton Protopopov) - Avoid deadlock scenarios caused by nested kprobe and fentry BPF programs (Priya Bala Govindasamy) - Fix a bug in BPF verifier which was missing a size check for BTF-based context access (Kumar Kartikeya Dwivedi) - Fix a crash found by syzbot through an invalid BPF prog_array access in perf_event_detach_bpf_prog (Jiri Olsa) - Fix several BPF sockmap bugs including a race causing a refcount imbalance upon element replace (Michal Luczaj) - Fix a use-after-free from mismatching BPF program/attachment RCU flavors (Jann Horn) Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> -----BEGIN PGP SIGNATURE----- iIsEABYKADMWIQTFp0I1jqZrAX+hPRXbK58LschIgwUCZ13rdhUcZGFuaWVsQGlv Z2VhcmJveC5uZXQACgkQ2yufC7HISINfqAD7B2vX6EgTFrgy7QDepQnZsmu2qjdW fFUzPatFXXp2S3MA/16vOEoHJ4rRhBkcUK/vw3gyY5j5bYZNUTTaam5l4BcM =gkfb -----END PGP SIGNATURE----- Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf Pull bpf fixes from Daniel Borkmann: - Fix a bug in the BPF verifier to track changes to packet data property for global functions (Eduard Zingerman) - Fix a theoretical BPF prog_array use-after-free in RCU handling of __uprobe_perf_func (Jann Horn) - Fix BPF tracing to have an explicit list of tracepoints and their arguments which need to be annotated as PTR_MAYBE_NULL (Kumar Kartikeya Dwivedi) - Fix a logic bug in the bpf_remove_insns code where a potential error would have been wrongly propagated (Anton Protopopov) - Avoid deadlock scenarios caused by nested kprobe and fentry BPF programs (Priya Bala Govindasamy) - Fix a bug in BPF verifier which was missing a size check for BTF-based context access (Kumar Kartikeya Dwivedi) - Fix a crash found by syzbot through an invalid BPF prog_array access in perf_event_detach_bpf_prog (Jiri Olsa) - Fix several BPF sockmap bugs including a race causing a refcount imbalance upon element replace (Michal Luczaj) - Fix a use-after-free from mismatching BPF program/attachment RCU flavors (Jann Horn) * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (23 commits) bpf: Avoid deadlock caused by nested kprobe and fentry bpf programs selftests/bpf: Add tests for raw_tp NULL args bpf: Augment raw_tp arguments with PTR_MAYBE_NULL bpf: Revert "bpf: Mark raw_tp arguments with PTR_MAYBE_NULL" selftests/bpf: Add test for narrow ctx load for pointer args bpf: Check size for BTF-based ctx access of pointer members selftests/bpf: extend changes_pkt_data with cases w/o subprograms bpf: fix null dereference when computing changes_pkt_data of prog w/o subprogs bpf: Fix theoretical prog_array UAF in __uprobe_perf_func() bpf: fix potential error return selftests/bpf: validate that tail call invalidates packet pointers bpf: consider that tail calls invalidate packet pointers selftests/bpf: freplace tests for tracking of changes_packet_data bpf: check changes_pkt_data property for extension programs selftests/bpf: test for changing packet data from global functions bpf: track changes_pkt_data property for global functions bpf: refactor bpf_helper_changes_pkt_data to use helper number bpf: add find_containing_subprog() utility function bpf,perf: Fix invalid prog_array access in perf_event_detach_bpf_prog bpf: Fix UAF via mismatching bpf_prog/attachment RCU flavors ...
This commit is contained in:
Linus Torvalds
commit
35f301dd45
@ -1527,6 +1527,7 @@ struct bpf_prog_aux {
|
||||
bool is_extended; /* true if extended by freplace program */
|
||||
bool jits_use_priv_stack;
|
||||
bool priv_stack_requested;
|
||||
bool changes_pkt_data;
|
||||
u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
|
||||
struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
|
||||
struct bpf_arena *arena;
|
||||
@ -2193,26 +2194,25 @@ bpf_prog_run_array(const struct bpf_prog_array *array,
|
||||
* rcu-protected dynamically sized maps.
|
||||
*/
|
||||
static __always_inline u32
|
||||
bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu,
|
||||
bpf_prog_run_array_uprobe(const struct bpf_prog_array *array,
|
||||
const void *ctx, bpf_prog_run_fn run_prog)
|
||||
{
|
||||
const struct bpf_prog_array_item *item;
|
||||
const struct bpf_prog *prog;
|
||||
const struct bpf_prog_array *array;
|
||||
struct bpf_run_ctx *old_run_ctx;
|
||||
struct bpf_trace_run_ctx run_ctx;
|
||||
u32 ret = 1;
|
||||
|
||||
might_fault();
|
||||
RCU_LOCKDEP_WARN(!rcu_read_lock_trace_held(), "no rcu lock held");
|
||||
|
||||
if (unlikely(!array))
|
||||
return ret;
|
||||
|
||||
rcu_read_lock_trace();
|
||||
migrate_disable();
|
||||
|
||||
run_ctx.is_uprobe = true;
|
||||
|
||||
array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held());
|
||||
if (unlikely(!array))
|
||||
goto out;
|
||||
old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
|
||||
item = &array->items[0];
|
||||
while ((prog = READ_ONCE(item->prog))) {
|
||||
@ -2227,9 +2227,7 @@ bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu,
|
||||
rcu_read_unlock();
|
||||
}
|
||||
bpf_reset_run_ctx(old_run_ctx);
|
||||
out:
|
||||
migrate_enable();
|
||||
rcu_read_unlock_trace();
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -3516,10 +3514,4 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog)
|
||||
return prog->aux->func_idx != 0;
|
||||
}
|
||||
|
||||
static inline bool bpf_prog_is_raw_tp(const struct bpf_prog *prog)
|
||||
{
|
||||
return prog->type == BPF_PROG_TYPE_TRACING &&
|
||||
prog->expected_attach_type == BPF_TRACE_RAW_TP;
|
||||
}
|
||||
|
||||
#endif /* _LINUX_BPF_H */
|
||||
|
||||
@ -659,6 +659,7 @@ struct bpf_subprog_info {
|
||||
bool args_cached: 1;
|
||||
/* true if bpf_fastcall stack region is used by functions that can't be inlined */
|
||||
bool keep_fastcall_stack: 1;
|
||||
bool changes_pkt_data: 1;
|
||||
|
||||
enum priv_stack_mode priv_stack_mode;
|
||||
u8 arg_cnt;
|
||||
|
||||
@ -1122,7 +1122,7 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
|
||||
bool bpf_jit_supports_private_stack(void);
|
||||
u64 bpf_arch_uaddress_limit(void);
|
||||
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
|
||||
bool bpf_helper_changes_pkt_data(void *func);
|
||||
bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id);
|
||||
|
||||
static inline bool bpf_dump_raw_ok(const struct cred *cred)
|
||||
{
|
||||
|
||||
@ -53,3 +53,9 @@ obj-$(CONFIG_BPF_SYSCALL) += relo_core.o
|
||||
obj-$(CONFIG_BPF_SYSCALL) += btf_iter.o
|
||||
obj-$(CONFIG_BPF_SYSCALL) += btf_relocate.o
|
||||
obj-$(CONFIG_BPF_SYSCALL) += kmem_cache_iter.o
|
||||
|
||||
CFLAGS_REMOVE_percpu_freelist.o = $(CC_FLAGS_FTRACE)
|
||||
CFLAGS_REMOVE_bpf_lru_list.o = $(CC_FLAGS_FTRACE)
|
||||
CFLAGS_REMOVE_queue_stack_maps.o = $(CC_FLAGS_FTRACE)
|
||||
CFLAGS_REMOVE_lpm_trie.o = $(CC_FLAGS_FTRACE)
|
||||
CFLAGS_REMOVE_ringbuf.o = $(CC_FLAGS_FTRACE)
|
||||
|
||||
149
kernel/bpf/btf.c
149
kernel/bpf/btf.c
@ -6439,6 +6439,101 @@ int btf_ctx_arg_offset(const struct btf *btf, const struct btf_type *func_proto,
|
||||
return off;
|
||||
}
|
||||
|
||||
struct bpf_raw_tp_null_args {
|
||||
const char *func;
|
||||
u64 mask;
|
||||
};
|
||||
|
||||
static const struct bpf_raw_tp_null_args raw_tp_null_args[] = {
|
||||
/* sched */
|
||||
{ "sched_pi_setprio", 0x10 },
|
||||
/* ... from sched_numa_pair_template event class */
|
||||
{ "sched_stick_numa", 0x100 },
|
||||
{ "sched_swap_numa", 0x100 },
|
||||
/* afs */
|
||||
{ "afs_make_fs_call", 0x10 },
|
||||
{ "afs_make_fs_calli", 0x10 },
|
||||
{ "afs_make_fs_call1", 0x10 },
|
||||
{ "afs_make_fs_call2", 0x10 },
|
||||
{ "afs_protocol_error", 0x1 },
|
||||
{ "afs_flock_ev", 0x10 },
|
||||
/* cachefiles */
|
||||
{ "cachefiles_lookup", 0x1 | 0x200 },
|
||||
{ "cachefiles_unlink", 0x1 },
|
||||
{ "cachefiles_rename", 0x1 },
|
||||
{ "cachefiles_prep_read", 0x1 },
|
||||
{ "cachefiles_mark_active", 0x1 },
|
||||
{ "cachefiles_mark_failed", 0x1 },
|
||||
{ "cachefiles_mark_inactive", 0x1 },
|
||||
{ "cachefiles_vfs_error", 0x1 },
|
||||
{ "cachefiles_io_error", 0x1 },
|
||||
{ "cachefiles_ondemand_open", 0x1 },
|
||||
{ "cachefiles_ondemand_copen", 0x1 },
|
||||
{ "cachefiles_ondemand_close", 0x1 },
|
||||
{ "cachefiles_ondemand_read", 0x1 },
|
||||
{ "cachefiles_ondemand_cread", 0x1 },
|
||||
{ "cachefiles_ondemand_fd_write", 0x1 },
|
||||
{ "cachefiles_ondemand_fd_release", 0x1 },
|
||||
/* ext4, from ext4__mballoc event class */
|
||||
{ "ext4_mballoc_discard", 0x10 },
|
||||
{ "ext4_mballoc_free", 0x10 },
|
||||
/* fib */
|
||||
{ "fib_table_lookup", 0x100 },
|
||||
/* filelock */
|
||||
/* ... from filelock_lock event class */
|
||||
{ "posix_lock_inode", 0x10 },
|
||||
{ "fcntl_setlk", 0x10 },
|
||||
{ "locks_remove_posix", 0x10 },
|
||||
{ "flock_lock_inode", 0x10 },
|
||||
/* ... from filelock_lease event class */
|
||||
{ "break_lease_noblock", 0x10 },
|
||||
{ "break_lease_block", 0x10 },
|
||||
{ "break_lease_unblock", 0x10 },
|
||||
{ "generic_delete_lease", 0x10 },
|
||||
{ "time_out_leases", 0x10 },
|
||||
/* host1x */
|
||||
{ "host1x_cdma_push_gather", 0x10000 },
|
||||
/* huge_memory */
|
||||
{ "mm_khugepaged_scan_pmd", 0x10 },
|
||||
{ "mm_collapse_huge_page_isolate", 0x1 },
|
||||
{ "mm_khugepaged_scan_file", 0x10 },
|
||||
{ "mm_khugepaged_collapse_file", 0x10 },
|
||||
/* kmem */
|
||||
{ "mm_page_alloc", 0x1 },
|
||||
{ "mm_page_pcpu_drain", 0x1 },
|
||||
/* .. from mm_page event class */
|
||||
{ "mm_page_alloc_zone_locked", 0x1 },
|
||||
/* netfs */
|
||||
{ "netfs_failure", 0x10 },
|
||||
/* power */
|
||||
{ "device_pm_callback_start", 0x10 },
|
||||
/* qdisc */
|
||||
{ "qdisc_dequeue", 0x1000 },
|
||||
/* rxrpc */
|
||||
{ "rxrpc_recvdata", 0x1 },
|
||||
{ "rxrpc_resend", 0x10 },
|
||||
/* sunrpc */
|
||||
{ "xs_stream_read_data", 0x1 },
|
||||
/* ... from xprt_cong_event event class */
|
||||
{ "xprt_reserve_cong", 0x10 },
|
||||
{ "xprt_release_cong", 0x10 },
|
||||
{ "xprt_get_cong", 0x10 },
|
||||
{ "xprt_put_cong", 0x10 },
|
||||
/* tcp */
|
||||
{ "tcp_send_reset", 0x11 },
|
||||
/* tegra_apb_dma */
|
||||
{ "tegra_dma_tx_status", 0x100 },
|
||||
/* timer_migration */
|
||||
{ "tmigr_update_events", 0x1 },
|
||||
/* writeback, from writeback_folio_template event class */
|
||||
{ "writeback_dirty_folio", 0x10 },
|
||||
{ "folio_wait_writeback", 0x10 },
|
||||
/* rdma */
|
||||
{ "mr_integ_alloc", 0x2000 },
|
||||
/* bpf_testmod */
|
||||
{ "bpf_testmod_test_read", 0x0 },
|
||||
};
|
||||
|
||||
bool btf_ctx_access(int off, int size, enum bpf_access_type type,
|
||||
const struct bpf_prog *prog,
|
||||
struct bpf_insn_access_aux *info)
|
||||
@ -6449,6 +6544,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
|
||||
const char *tname = prog->aux->attach_func_name;
|
||||
struct bpf_verifier_log *log = info->log;
|
||||
const struct btf_param *args;
|
||||
bool ptr_err_raw_tp = false;
|
||||
const char *tag_value;
|
||||
u32 nr_args, arg;
|
||||
int i, ret;
|
||||
@ -6543,6 +6639,12 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
|
||||
return false;
|
||||
}
|
||||
|
||||
if (size != sizeof(u64)) {
|
||||
bpf_log(log, "func '%s' size %d must be 8\n",
|
||||
tname, size);
|
||||
return false;
|
||||
}
|
||||
|
||||
/* check for PTR_TO_RDONLY_BUF_OR_NULL or PTR_TO_RDWR_BUF_OR_NULL */
|
||||
for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
|
||||
const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
|
||||
@ -6588,12 +6690,42 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
|
||||
if (prog_args_trusted(prog))
|
||||
info->reg_type |= PTR_TRUSTED;
|
||||
|
||||
/* Raw tracepoint arguments always get marked as maybe NULL */
|
||||
if (bpf_prog_is_raw_tp(prog))
|
||||
info->reg_type |= PTR_MAYBE_NULL;
|
||||
else if (btf_param_match_suffix(btf, &args[arg], "__nullable"))
|
||||
if (btf_param_match_suffix(btf, &args[arg], "__nullable"))
|
||||
info->reg_type |= PTR_MAYBE_NULL;
|
||||
|
||||
if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
|
||||
struct btf *btf = prog->aux->attach_btf;
|
||||
const struct btf_type *t;
|
||||
const char *tname;
|
||||
|
||||
/* BTF lookups cannot fail, return false on error */
|
||||
t = btf_type_by_id(btf, prog->aux->attach_btf_id);
|
||||
if (!t)
|
||||
return false;
|
||||
tname = btf_name_by_offset(btf, t->name_off);
|
||||
if (!tname)
|
||||
return false;
|
||||
/* Checked by bpf_check_attach_target */
|
||||
tname += sizeof("btf_trace_") - 1;
|
||||
for (i = 0; i < ARRAY_SIZE(raw_tp_null_args); i++) {
|
||||
/* Is this a func with potential NULL args? */
|
||||
if (strcmp(tname, raw_tp_null_args[i].func))
|
||||
continue;
|
||||
if (raw_tp_null_args[i].mask & (0x1 << (arg * 4)))
|
||||
info->reg_type |= PTR_MAYBE_NULL;
|
||||
/* Is the current arg IS_ERR? */
|
||||
if (raw_tp_null_args[i].mask & (0x2 << (arg * 4)))
|
||||
ptr_err_raw_tp = true;
|
||||
break;
|
||||
}
|
||||
/* If we don't know NULL-ness specification and the tracepoint
|
||||
* is coming from a loadable module, be conservative and mark
|
||||
* argument as PTR_MAYBE_NULL.
|
||||
*/
|
||||
if (i == ARRAY_SIZE(raw_tp_null_args) && btf_is_module(btf))
|
||||
info->reg_type |= PTR_MAYBE_NULL;
|
||||
}
|
||||
|
||||
if (tgt_prog) {
|
||||
enum bpf_prog_type tgt_type;
|
||||
|
||||
@ -6638,6 +6770,15 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
|
||||
bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n",
|
||||
tname, arg, info->btf_id, btf_type_str(t),
|
||||
__btf_name_by_offset(btf, t->name_off));
|
||||
|
||||
/* Perform all checks on the validity of type for this argument, but if
|
||||
* we know it can be IS_ERR at runtime, scrub pointer type and mark as
|
||||
* scalar.
|
||||
*/
|
||||
if (ptr_err_raw_tp) {
|
||||
bpf_log(log, "marking pointer arg%d as scalar as it may encode error", arg);
|
||||
info->reg_type = SCALAR_VALUE;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(btf_ctx_access);
|
||||
|
||||
@ -539,6 +539,8 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
|
||||
|
||||
int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt)
|
||||
{
|
||||
int err;
|
||||
|
||||
/* Branch offsets can't overflow when program is shrinking, no need
|
||||
* to call bpf_adj_branches(..., true) here
|
||||
*/
|
||||
@ -546,7 +548,9 @@ int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt)
|
||||
sizeof(struct bpf_insn) * (prog->len - off - cnt));
|
||||
prog->len -= cnt;
|
||||
|
||||
return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false));
|
||||
err = bpf_adj_branches(prog, off, off + cnt, off, false);
|
||||
WARN_ON_ONCE(err);
|
||||
return err;
|
||||
}
|
||||
|
||||
static void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp)
|
||||
@ -2936,7 +2940,7 @@ void __weak bpf_jit_compile(struct bpf_prog *prog)
|
||||
{
|
||||
}
|
||||
|
||||
bool __weak bpf_helper_changes_pkt_data(void *func)
|
||||
bool __weak bpf_helper_changes_pkt_data(enum bpf_func_id func_id)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -420,25 +420,6 @@ static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg)
|
||||
return rec;
|
||||
}
|
||||
|
||||
static bool mask_raw_tp_reg_cond(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) {
|
||||
return reg->type == (PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL) &&
|
||||
bpf_prog_is_raw_tp(env->prog) && !reg->ref_obj_id;
|
||||
}
|
||||
|
||||
static bool mask_raw_tp_reg(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
|
||||
{
|
||||
if (!mask_raw_tp_reg_cond(env, reg))
|
||||
return false;
|
||||
reg->type &= ~PTR_MAYBE_NULL;
|
||||
return true;
|
||||
}
|
||||
|
||||
static void unmask_raw_tp_reg(struct bpf_reg_state *reg, bool result)
|
||||
{
|
||||
if (result)
|
||||
reg->type |= PTR_MAYBE_NULL;
|
||||
}
|
||||
|
||||
static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)
|
||||
{
|
||||
struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;
|
||||
@ -2597,16 +2578,36 @@ static int cmp_subprogs(const void *a, const void *b)
|
||||
((struct bpf_subprog_info *)b)->start;
|
||||
}
|
||||
|
||||
/* Find subprogram that contains instruction at 'off' */
|
||||
static struct bpf_subprog_info *find_containing_subprog(struct bpf_verifier_env *env, int off)
|
||||
{
|
||||
struct bpf_subprog_info *vals = env->subprog_info;
|
||||
int l, r, m;
|
||||
|
||||
if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0)
|
||||
return NULL;
|
||||
|
||||
l = 0;
|
||||
r = env->subprog_cnt - 1;
|
||||
while (l < r) {
|
||||
m = l + (r - l + 1) / 2;
|
||||
if (vals[m].start <= off)
|
||||
l = m;
|
||||
else
|
||||
r = m - 1;
|
||||
}
|
||||
return &vals[l];
|
||||
}
|
||||
|
||||
/* Find subprogram that starts exactly at 'off' */
|
||||
static int find_subprog(struct bpf_verifier_env *env, int off)
|
||||
{
|
||||
struct bpf_subprog_info *p;
|
||||
|
||||
p = bsearch(&off, env->subprog_info, env->subprog_cnt,
|
||||
sizeof(env->subprog_info[0]), cmp_subprogs);
|
||||
if (!p)
|
||||
p = find_containing_subprog(env, off);
|
||||
if (!p || p->start != off)
|
||||
return -ENOENT;
|
||||
return p - env->subprog_info;
|
||||
|
||||
}
|
||||
|
||||
static int add_subprog(struct bpf_verifier_env *env, int off)
|
||||
@ -6781,7 +6782,6 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
|
||||
const char *field_name = NULL;
|
||||
enum bpf_type_flag flag = 0;
|
||||
u32 btf_id = 0;
|
||||
bool mask;
|
||||
int ret;
|
||||
|
||||
if (!env->allow_ptr_leaks) {
|
||||
@ -6853,21 +6853,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
|
||||
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
/* For raw_tp progs, we allow dereference of PTR_MAYBE_NULL
|
||||
* trusted PTR_TO_BTF_ID, these are the ones that are possibly
|
||||
* arguments to the raw_tp. Since internal checks in for trusted
|
||||
* reg in check_ptr_to_btf_access would consider PTR_MAYBE_NULL
|
||||
* modifier as problematic, mask it out temporarily for the
|
||||
* check. Don't apply this to pointers with ref_obj_id > 0, as
|
||||
* those won't be raw_tp args.
|
||||
*
|
||||
* We may end up applying this relaxation to other trusted
|
||||
* PTR_TO_BTF_ID with maybe null flag, since we cannot
|
||||
* distinguish PTR_MAYBE_NULL tagged for arguments vs normal
|
||||
* tagging, but that should expand allowed behavior, and not
|
||||
* cause regression for existing behavior.
|
||||
*/
|
||||
mask = mask_raw_tp_reg(env, reg);
|
||||
|
||||
if (ret != PTR_TO_BTF_ID) {
|
||||
/* just mark; */
|
||||
|
||||
@ -6928,13 +6914,8 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
|
||||
clear_trusted_flags(&flag);
|
||||
}
|
||||
|
||||
if (atype == BPF_READ && value_regno >= 0) {
|
||||
if (atype == BPF_READ && value_regno >= 0)
|
||||
mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);
|
||||
/* We've assigned a new type to regno, so don't undo masking. */
|
||||
if (regno == value_regno)
|
||||
mask = false;
|
||||
}
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -7309,7 +7290,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
|
||||
if (!err && t == BPF_READ && value_regno >= 0)
|
||||
mark_reg_unknown(env, regs, value_regno);
|
||||
} else if (base_type(reg->type) == PTR_TO_BTF_ID &&
|
||||
(mask_raw_tp_reg_cond(env, reg) || !type_may_be_null(reg->type))) {
|
||||
!type_may_be_null(reg->type)) {
|
||||
err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
|
||||
value_regno);
|
||||
} else if (reg->type == CONST_PTR_TO_MAP) {
|
||||
@ -9012,7 +8993,6 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
|
||||
enum bpf_reg_type type = reg->type;
|
||||
u32 *arg_btf_id = NULL;
|
||||
int err = 0;
|
||||
bool mask;
|
||||
|
||||
if (arg_type == ARG_DONTCARE)
|
||||
return 0;
|
||||
@ -9053,11 +9033,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
|
||||
base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)
|
||||
arg_btf_id = fn->arg_btf_id[arg];
|
||||
|
||||
mask = mask_raw_tp_reg(env, reg);
|
||||
err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
err = err ?: check_func_arg_reg_off(env, reg, regno, arg_type);
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
err = check_func_arg_reg_off(env, reg, regno, arg_type);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
@ -9852,17 +9832,14 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
|
||||
return ret;
|
||||
} else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
|
||||
struct bpf_call_arg_meta meta;
|
||||
bool mask;
|
||||
int err;
|
||||
|
||||
if (register_is_null(reg) && type_may_be_null(arg->arg_type))
|
||||
continue;
|
||||
|
||||
memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */
|
||||
mask = mask_raw_tp_reg(env, reg);
|
||||
err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta);
|
||||
err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type);
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
if (err)
|
||||
return err;
|
||||
} else {
|
||||
@ -10022,6 +9999,8 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
|
||||
|
||||
verbose(env, "Func#%d ('%s') is global and assumed valid.\n",
|
||||
subprog, sub_name);
|
||||
if (env->subprog_info[subprog].changes_pkt_data)
|
||||
clear_all_pkt_pointers(env);
|
||||
/* mark global subprog for verifying after main prog */
|
||||
subprog_aux(env, subprog)->called = true;
|
||||
clear_caller_saved_regs(env, caller->regs);
|
||||
@ -10708,7 +10687,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
|
||||
}
|
||||
|
||||
/* With LD_ABS/IND some JITs save/restore skb from r1. */
|
||||
changes_data = bpf_helper_changes_pkt_data(fn->func);
|
||||
changes_data = bpf_helper_changes_pkt_data(func_id);
|
||||
if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
|
||||
verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
|
||||
func_id_name(func_id), func_id);
|
||||
@ -12183,7 +12162,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
|
||||
enum bpf_arg_type arg_type = ARG_DONTCARE;
|
||||
u32 regno = i + 1, ref_id, type_size;
|
||||
bool is_ret_buf_sz = false;
|
||||
bool mask = false;
|
||||
int kf_arg_type;
|
||||
|
||||
t = btf_type_skip_modifiers(btf, args[i].type, NULL);
|
||||
@ -12242,15 +12220,12 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
mask = mask_raw_tp_reg(env, reg);
|
||||
if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) &&
|
||||
(register_is_null(reg) || type_may_be_null(reg->type)) &&
|
||||
!is_kfunc_arg_nullable(meta->btf, &args[i])) {
|
||||
verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
return -EACCES;
|
||||
}
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
|
||||
if (reg->ref_obj_id) {
|
||||
if (is_kfunc_release(meta) && meta->ref_obj_id) {
|
||||
@ -12308,24 +12283,16 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
|
||||
if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta))
|
||||
break;
|
||||
|
||||
/* Allow passing maybe NULL raw_tp arguments to
|
||||
* kfuncs for compatibility. Don't apply this to
|
||||
* arguments with ref_obj_id > 0.
|
||||
*/
|
||||
mask = mask_raw_tp_reg(env, reg);
|
||||
if (!is_trusted_reg(reg)) {
|
||||
if (!is_kfunc_rcu(meta)) {
|
||||
verbose(env, "R%d must be referenced or trusted\n", regno);
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
return -EINVAL;
|
||||
}
|
||||
if (!is_rcu_reg(reg)) {
|
||||
verbose(env, "R%d must be a rcu pointer\n", regno);
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
fallthrough;
|
||||
case KF_ARG_PTR_TO_CTX:
|
||||
case KF_ARG_PTR_TO_DYNPTR:
|
||||
@ -12348,9 +12315,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
|
||||
|
||||
if (is_kfunc_release(meta) && reg->ref_obj_id)
|
||||
arg_type |= OBJ_RELEASE;
|
||||
mask = mask_raw_tp_reg(env, reg);
|
||||
ret = check_func_arg_reg_off(env, reg, regno, arg_type);
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
@ -12527,7 +12492,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
|
||||
ref_tname = btf_name_by_offset(btf, ref_t->name_off);
|
||||
fallthrough;
|
||||
case KF_ARG_PTR_TO_BTF_ID:
|
||||
mask = mask_raw_tp_reg(env, reg);
|
||||
/* Only base_type is checked, further checks are done here */
|
||||
if ((base_type(reg->type) != PTR_TO_BTF_ID ||
|
||||
(bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) &&
|
||||
@ -12536,11 +12500,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
|
||||
verbose(env, "expected %s or socket\n",
|
||||
reg_type_str(env, base_type(reg->type) |
|
||||
(type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS)));
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
return -EINVAL;
|
||||
}
|
||||
ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);
|
||||
unmask_raw_tp_reg(reg, mask);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
break;
|
||||
@ -13513,7 +13475,7 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env,
|
||||
*/
|
||||
static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
|
||||
struct bpf_insn *insn,
|
||||
struct bpf_reg_state *ptr_reg,
|
||||
const struct bpf_reg_state *ptr_reg,
|
||||
const struct bpf_reg_state *off_reg)
|
||||
{
|
||||
struct bpf_verifier_state *vstate = env->cur_state;
|
||||
@ -13527,7 +13489,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
|
||||
struct bpf_sanitize_info info = {};
|
||||
u8 opcode = BPF_OP(insn->code);
|
||||
u32 dst = insn->dst_reg;
|
||||
bool mask;
|
||||
int ret;
|
||||
|
||||
dst_reg = ®s[dst];
|
||||
@ -13554,14 +13515,11 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
|
||||
return -EACCES;
|
||||
}
|
||||
|
||||
mask = mask_raw_tp_reg(env, ptr_reg);
|
||||
if (ptr_reg->type & PTR_MAYBE_NULL) {
|
||||
verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
|
||||
dst, reg_type_str(env, ptr_reg->type));
|
||||
unmask_raw_tp_reg(ptr_reg, mask);
|
||||
return -EACCES;
|
||||
}
|
||||
unmask_raw_tp_reg(ptr_reg, mask);
|
||||
|
||||
switch (base_type(ptr_reg->type)) {
|
||||
case PTR_TO_CTX:
|
||||
@ -16226,6 +16184,29 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off)
|
||||
{
|
||||
struct bpf_subprog_info *subprog;
|
||||
|
||||
subprog = find_containing_subprog(env, off);
|
||||
subprog->changes_pkt_data = true;
|
||||
}
|
||||
|
||||
/* 't' is an index of a call-site.
|
||||
* 'w' is a callee entry point.
|
||||
* Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
|
||||
* Rely on DFS traversal order and absence of recursive calls to guarantee that
|
||||
* callee's change_pkt_data marks would be correct at that moment.
|
||||
*/
|
||||
static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
|
||||
{
|
||||
struct bpf_subprog_info *caller, *callee;
|
||||
|
||||
caller = find_containing_subprog(env, t);
|
||||
callee = find_containing_subprog(env, w);
|
||||
caller->changes_pkt_data |= callee->changes_pkt_data;
|
||||
}
|
||||
|
||||
/* non-recursive DFS pseudo code
|
||||
* 1 procedure DFS-iterative(G,v):
|
||||
* 2 label v as discovered
|
||||
@ -16359,6 +16340,7 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns,
|
||||
bool visit_callee)
|
||||
{
|
||||
int ret, insn_sz;
|
||||
int w;
|
||||
|
||||
insn_sz = bpf_is_ldimm64(&insns[t]) ? 2 : 1;
|
||||
ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);
|
||||
@ -16370,8 +16352,10 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns,
|
||||
mark_jmp_point(env, t + insn_sz);
|
||||
|
||||
if (visit_callee) {
|
||||
w = t + insns[t].imm + 1;
|
||||
mark_prune_point(env, t);
|
||||
ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);
|
||||
merge_callee_effects(env, t, w);
|
||||
ret = push_insn(t, w, BRANCH, env);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@ -16688,6 +16672,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env)
|
||||
mark_prune_point(env, t);
|
||||
mark_jmp_point(env, t);
|
||||
}
|
||||
if (bpf_helper_call(insn) && bpf_helper_changes_pkt_data(insn->imm))
|
||||
mark_subprog_changes_pkt_data(env, t);
|
||||
if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
|
||||
struct bpf_kfunc_call_arg_meta meta;
|
||||
|
||||
@ -16822,6 +16808,7 @@ static int check_cfg(struct bpf_verifier_env *env)
|
||||
}
|
||||
}
|
||||
ret = 0; /* cfg looks good */
|
||||
env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;
|
||||
|
||||
err_free:
|
||||
kvfree(insn_state);
|
||||
@ -20075,7 +20062,6 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
|
||||
* for this case.
|
||||
*/
|
||||
case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED:
|
||||
case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL:
|
||||
if (type == BPF_READ) {
|
||||
if (BPF_MODE(insn->code) == BPF_MEM)
|
||||
insn->code = BPF_LDX | BPF_PROBE_MEM |
|
||||
@ -20311,6 +20297,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
|
||||
func[i]->aux->num_exentries = num_exentries;
|
||||
func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
|
||||
func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
|
||||
func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;
|
||||
if (!i)
|
||||
func[i]->aux->exception_boundary = env->seen_exception;
|
||||
func[i] = bpf_int_jit_compile(func[i]);
|
||||
@ -22141,6 +22128,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
|
||||
}
|
||||
if (tgt_prog) {
|
||||
struct bpf_prog_aux *aux = tgt_prog->aux;
|
||||
bool tgt_changes_pkt_data;
|
||||
|
||||
if (bpf_prog_is_dev_bound(prog->aux) &&
|
||||
!bpf_prog_dev_bound_match(prog, tgt_prog)) {
|
||||
@ -22175,6 +22163,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
|
||||
"Extension programs should be JITed\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
tgt_changes_pkt_data = aux->func
|
||||
? aux->func[subprog]->aux->changes_pkt_data
|
||||
: aux->changes_pkt_data;
|
||||
if (prog->aux->changes_pkt_data && !tgt_changes_pkt_data) {
|
||||
bpf_log(log,
|
||||
"Extension program changes packet data, while original does not\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
if (!tgt_prog->jited) {
|
||||
bpf_log(log, "Can attach to only JITed progs\n");
|
||||
@ -22640,10 +22636,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
|
||||
if (ret < 0)
|
||||
goto skip_full_check;
|
||||
|
||||
ret = check_attach_btf_id(env);
|
||||
if (ret)
|
||||
goto skip_full_check;
|
||||
|
||||
ret = resolve_pseudo_ldimm64(env);
|
||||
if (ret < 0)
|
||||
goto skip_full_check;
|
||||
@ -22658,6 +22650,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
|
||||
if (ret < 0)
|
||||
goto skip_full_check;
|
||||
|
||||
ret = check_attach_btf_id(env);
|
||||
if (ret)
|
||||
goto skip_full_check;
|
||||
|
||||
ret = mark_fastcall_patterns(env);
|
||||
if (ret < 0)
|
||||
goto skip_full_check;
|
||||
|
||||
@ -2250,6 +2250,9 @@ void perf_event_detach_bpf_prog(struct perf_event *event)
|
||||
goto unlock;
|
||||
|
||||
old_array = bpf_event_rcu_dereference(event->tp_event->prog_array);
|
||||
if (!old_array)
|
||||
goto put;
|
||||
|
||||
ret = bpf_prog_array_copy(old_array, event->prog, NULL, 0, &new_array);
|
||||
if (ret < 0) {
|
||||
bpf_prog_array_delete_safe(old_array, event->prog);
|
||||
@ -2258,6 +2261,14 @@ void perf_event_detach_bpf_prog(struct perf_event *event)
|
||||
bpf_prog_array_free_sleepable(old_array);
|
||||
}
|
||||
|
||||
put:
|
||||
/*
|
||||
* It could be that the bpf_prog is not sleepable (and will be freed
|
||||
* via normal RCU), but is called from a point that supports sleepable
|
||||
* programs and uses tasks-trace-RCU.
|
||||
*/
|
||||
synchronize_rcu_tasks_trace();
|
||||
|
||||
bpf_prog_put(event->prog);
|
||||
event->prog = NULL;
|
||||
|
||||
|
||||
@ -1402,9 +1402,13 @@ static void __uprobe_perf_func(struct trace_uprobe *tu,
|
||||
|
||||
#ifdef CONFIG_BPF_EVENTS
|
||||
if (bpf_prog_array_valid(call)) {
|
||||
const struct bpf_prog_array *array;
|
||||
u32 ret;
|
||||
|
||||
ret = bpf_prog_run_array_uprobe(call->prog_array, regs, bpf_prog_run);
|
||||
rcu_read_lock_trace();
|
||||
array = rcu_dereference_check(call->prog_array, rcu_read_lock_trace_held());
|
||||
ret = bpf_prog_run_array_uprobe(array, regs, bpf_prog_run);
|
||||
rcu_read_unlock_trace();
|
||||
if (!ret)
|
||||
return;
|
||||
}
|
||||
|
||||
@ -7899,42 +7899,37 @@ static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv6_proto = {
|
||||
|
||||
#endif /* CONFIG_INET */
|
||||
|
||||
bool bpf_helper_changes_pkt_data(void *func)
|
||||
bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id)
|
||||
{
|
||||
if (func == bpf_skb_vlan_push ||
|
||||
func == bpf_skb_vlan_pop ||
|
||||
func == bpf_skb_store_bytes ||
|
||||
func == bpf_skb_change_proto ||
|
||||
func == bpf_skb_change_head ||
|
||||
func == sk_skb_change_head ||
|
||||
func == bpf_skb_change_tail ||
|
||||
func == sk_skb_change_tail ||
|
||||
func == bpf_skb_adjust_room ||
|
||||
func == sk_skb_adjust_room ||
|
||||
func == bpf_skb_pull_data ||
|
||||
func == sk_skb_pull_data ||
|
||||
func == bpf_clone_redirect ||
|
||||
func == bpf_l3_csum_replace ||
|
||||
func == bpf_l4_csum_replace ||
|
||||
func == bpf_xdp_adjust_head ||
|
||||
func == bpf_xdp_adjust_meta ||
|
||||
func == bpf_msg_pull_data ||
|
||||
func == bpf_msg_push_data ||
|
||||
func == bpf_msg_pop_data ||
|
||||
func == bpf_xdp_adjust_tail ||
|
||||
#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
|
||||
func == bpf_lwt_seg6_store_bytes ||
|
||||
func == bpf_lwt_seg6_adjust_srh ||
|
||||
func == bpf_lwt_seg6_action ||
|
||||
#endif
|
||||
#ifdef CONFIG_INET
|
||||
func == bpf_sock_ops_store_hdr_opt ||
|
||||
#endif
|
||||
func == bpf_lwt_in_push_encap ||
|
||||
func == bpf_lwt_xmit_push_encap)
|
||||
switch (func_id) {
|
||||
case BPF_FUNC_clone_redirect:
|
||||
case BPF_FUNC_l3_csum_replace:
|
||||
case BPF_FUNC_l4_csum_replace:
|
||||
case BPF_FUNC_lwt_push_encap:
|
||||
case BPF_FUNC_lwt_seg6_action:
|
||||
case BPF_FUNC_lwt_seg6_adjust_srh:
|
||||
case BPF_FUNC_lwt_seg6_store_bytes:
|
||||
case BPF_FUNC_msg_pop_data:
|
||||
case BPF_FUNC_msg_pull_data:
|
||||
case BPF_FUNC_msg_push_data:
|
||||
case BPF_FUNC_skb_adjust_room:
|
||||
case BPF_FUNC_skb_change_head:
|
||||
case BPF_FUNC_skb_change_proto:
|
||||
case BPF_FUNC_skb_change_tail:
|
||||
case BPF_FUNC_skb_pull_data:
|
||||
case BPF_FUNC_skb_store_bytes:
|
||||
case BPF_FUNC_skb_vlan_pop:
|
||||
case BPF_FUNC_skb_vlan_push:
|
||||
case BPF_FUNC_store_hdr_opt:
|
||||
case BPF_FUNC_xdp_adjust_head:
|
||||
case BPF_FUNC_xdp_adjust_meta:
|
||||
case BPF_FUNC_xdp_adjust_tail:
|
||||
/* tail-called program could call any of the above */
|
||||
case BPF_FUNC_tail_call:
|
||||
return true;
|
||||
|
||||
return false;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const struct bpf_func_proto bpf_event_output_data_proto __weak;
|
||||
|
||||
@ -159,6 +159,7 @@ static void sock_map_del_link(struct sock *sk,
|
||||
verdict_stop = true;
|
||||
list_del(&link->list);
|
||||
sk_psock_free_link(link);
|
||||
break;
|
||||
}
|
||||
}
|
||||
spin_unlock_bh(&psock->link_lock);
|
||||
@ -411,12 +412,11 @@ static void *sock_map_lookup_sys(struct bpf_map *map, void *key)
|
||||
static int __sock_map_delete(struct bpf_stab *stab, struct sock *sk_test,
|
||||
struct sock **psk)
|
||||
{
|
||||
struct sock *sk;
|
||||
struct sock *sk = NULL;
|
||||
int err = 0;
|
||||
|
||||
spin_lock_bh(&stab->lock);
|
||||
sk = *psk;
|
||||
if (!sk_test || sk_test == sk)
|
||||
if (!sk_test || sk_test == *psk)
|
||||
sk = xchg(psk, NULL);
|
||||
|
||||
if (likely(sk))
|
||||
|
||||
107
tools/testing/selftests/bpf/prog_tests/changes_pkt_data.c
Normal file
107
tools/testing/selftests/bpf/prog_tests/changes_pkt_data.c
Normal file
@ -0,0 +1,107 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
#include "bpf/libbpf.h"
|
||||
#include "changes_pkt_data_freplace.skel.h"
|
||||
#include "changes_pkt_data.skel.h"
|
||||
#include <test_progs.h>
|
||||
|
||||
static void print_verifier_log(const char *log)
|
||||
{
|
||||
if (env.verbosity >= VERBOSE_VERY)
|
||||
fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log);
|
||||
}
|
||||
|
||||
static void test_aux(const char *main_prog_name,
|
||||
const char *to_be_replaced,
|
||||
const char *replacement,
|
||||
bool expect_load)
|
||||
{
|
||||
struct changes_pkt_data_freplace *freplace = NULL;
|
||||
struct bpf_program *freplace_prog = NULL;
|
||||
struct bpf_program *main_prog = NULL;
|
||||
LIBBPF_OPTS(bpf_object_open_opts, opts);
|
||||
struct changes_pkt_data *main = NULL;
|
||||
char log[16*1024];
|
||||
int err;
|
||||
|
||||
opts.kernel_log_buf = log;
|
||||
opts.kernel_log_size = sizeof(log);
|
||||
if (env.verbosity >= VERBOSE_SUPER)
|
||||
opts.kernel_log_level = 1 | 2 | 4;
|
||||
main = changes_pkt_data__open_opts(&opts);
|
||||
if (!ASSERT_OK_PTR(main, "changes_pkt_data__open"))
|
||||
goto out;
|
||||
main_prog = bpf_object__find_program_by_name(main->obj, main_prog_name);
|
||||
if (!ASSERT_OK_PTR(main_prog, "main_prog"))
|
||||
goto out;
|
||||
bpf_program__set_autoload(main_prog, true);
|
||||
err = changes_pkt_data__load(main);
|
||||
print_verifier_log(log);
|
||||
if (!ASSERT_OK(err, "changes_pkt_data__load"))
|
||||
goto out;
|
||||
freplace = changes_pkt_data_freplace__open_opts(&opts);
|
||||
if (!ASSERT_OK_PTR(freplace, "changes_pkt_data_freplace__open"))
|
||||
goto out;
|
||||
freplace_prog = bpf_object__find_program_by_name(freplace->obj, replacement);
|
||||
if (!ASSERT_OK_PTR(freplace_prog, "freplace_prog"))
|
||||
goto out;
|
||||
bpf_program__set_autoload(freplace_prog, true);
|
||||
bpf_program__set_autoattach(freplace_prog, true);
|
||||
bpf_program__set_attach_target(freplace_prog,
|
||||
bpf_program__fd(main_prog),
|
||||
to_be_replaced);
|
||||
err = changes_pkt_data_freplace__load(freplace);
|
||||
print_verifier_log(log);
|
||||
if (expect_load) {
|
||||
ASSERT_OK(err, "changes_pkt_data_freplace__load");
|
||||
} else {
|
||||
ASSERT_ERR(err, "changes_pkt_data_freplace__load");
|
||||
ASSERT_HAS_SUBSTR(log, "Extension program changes packet data", "error log");
|
||||
}
|
||||
|
||||
out:
|
||||
changes_pkt_data_freplace__destroy(freplace);
|
||||
changes_pkt_data__destroy(main);
|
||||
}
|
||||
|
||||
/* There are two global subprograms in both changes_pkt_data.skel.h:
|
||||
* - one changes packet data;
|
||||
* - another does not.
|
||||
* It is ok to freplace subprograms that change packet data with those
|
||||
* that either do or do not. It is only ok to freplace subprograms
|
||||
* that do not change packet data with those that do not as well.
|
||||
* The below tests check outcomes for each combination of such freplace.
|
||||
* Also test a case when main subprogram itself is replaced and is a single
|
||||
* subprogram in a program.
|
||||
*/
|
||||
void test_changes_pkt_data_freplace(void)
|
||||
{
|
||||
struct {
|
||||
const char *main;
|
||||
const char *to_be_replaced;
|
||||
bool changes;
|
||||
} mains[] = {
|
||||
{ "main_with_subprogs", "changes_pkt_data", true },
|
||||
{ "main_with_subprogs", "does_not_change_pkt_data", false },
|
||||
{ "main_changes", "main_changes", true },
|
||||
{ "main_does_not_change", "main_does_not_change", false },
|
||||
};
|
||||
struct {
|
||||
const char *func;
|
||||
bool changes;
|
||||
} replacements[] = {
|
||||
{ "changes_pkt_data", true },
|
||||
{ "does_not_change_pkt_data", false }
|
||||
};
|
||||
char buf[64];
|
||||
|
||||
for (int i = 0; i < ARRAY_SIZE(mains); ++i) {
|
||||
for (int j = 0; j < ARRAY_SIZE(replacements); ++j) {
|
||||
snprintf(buf, sizeof(buf), "%s_with_%s",
|
||||
mains[i].to_be_replaced, replacements[j].func);
|
||||
if (!test__start_subtest(buf))
|
||||
continue;
|
||||
test_aux(mains[i].main, mains[i].to_be_replaced, replacements[j].func,
|
||||
mains[i].changes || !replacements[j].changes);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -3,11 +3,14 @@
|
||||
|
||||
#include <test_progs.h>
|
||||
#include "raw_tp_null.skel.h"
|
||||
#include "raw_tp_null_fail.skel.h"
|
||||
|
||||
void test_raw_tp_null(void)
|
||||
{
|
||||
struct raw_tp_null *skel;
|
||||
|
||||
RUN_TESTS(raw_tp_null_fail);
|
||||
|
||||
skel = raw_tp_null__open_and_load();
|
||||
if (!ASSERT_OK_PTR(skel, "raw_tp_null__open_and_load"))
|
||||
return;
|
||||
|
||||
@ -934,8 +934,10 @@ static void test_sockmap_same_sock(void)
|
||||
|
||||
err = socketpair(AF_UNIX, SOCK_STREAM, 0, stream);
|
||||
ASSERT_OK(err, "socketpair(af_unix, sock_stream)");
|
||||
if (err)
|
||||
if (err) {
|
||||
close(tcp);
|
||||
goto out;
|
||||
}
|
||||
|
||||
for (i = 0; i < 2; i++) {
|
||||
err = bpf_map_update_elem(map, &zero, &stream[0], BPF_ANY);
|
||||
@ -954,14 +956,14 @@ static void test_sockmap_same_sock(void)
|
||||
ASSERT_OK(err, "bpf_map_update_elem(tcp)");
|
||||
}
|
||||
|
||||
close(tcp);
|
||||
err = bpf_map_delete_elem(map, &zero);
|
||||
ASSERT_OK(err, "bpf_map_delete_elem(entry)");
|
||||
ASSERT_ERR(err, "bpf_map_delete_elem(entry)");
|
||||
|
||||
close(stream[0]);
|
||||
close(stream[1]);
|
||||
out:
|
||||
close(dgram);
|
||||
close(tcp);
|
||||
close(udp);
|
||||
test_sockmap_pass_prog__destroy(skel);
|
||||
}
|
||||
|
||||
39
tools/testing/selftests/bpf/progs/changes_pkt_data.c
Normal file
39
tools/testing/selftests/bpf/progs/changes_pkt_data.c
Normal file
@ -0,0 +1,39 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
#include <linux/bpf.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
|
||||
__noinline
|
||||
long changes_pkt_data(struct __sk_buff *sk)
|
||||
{
|
||||
return bpf_skb_pull_data(sk, 0);
|
||||
}
|
||||
|
||||
__noinline __weak
|
||||
long does_not_change_pkt_data(struct __sk_buff *sk)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("?tc")
|
||||
int main_with_subprogs(struct __sk_buff *sk)
|
||||
{
|
||||
changes_pkt_data(sk);
|
||||
does_not_change_pkt_data(sk);
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("?tc")
|
||||
int main_changes(struct __sk_buff *sk)
|
||||
{
|
||||
bpf_skb_pull_data(sk, 0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("?tc")
|
||||
int main_does_not_change(struct __sk_buff *sk)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
@ -0,0 +1,18 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
#include <linux/bpf.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
|
||||
SEC("?freplace")
|
||||
long changes_pkt_data(struct __sk_buff *sk)
|
||||
{
|
||||
return bpf_skb_pull_data(sk, 0);
|
||||
}
|
||||
|
||||
SEC("?freplace")
|
||||
long does_not_change_pkt_data(struct __sk_buff *sk)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
@ -3,6 +3,7 @@
|
||||
|
||||
#include <vmlinux.h>
|
||||
#include <bpf/bpf_tracing.h>
|
||||
#include "bpf_misc.h"
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
||||
@ -17,16 +18,14 @@ int BPF_PROG(test_raw_tp_null, struct sk_buff *skb)
|
||||
if (task->pid != tid)
|
||||
return 0;
|
||||
|
||||
i = i + skb->mark + 1;
|
||||
/* The compiler may move the NULL check before this deref, which causes
|
||||
* the load to fail as deref of scalar. Prevent that by using a barrier.
|
||||
/* If dead code elimination kicks in, the increment +=2 will be
|
||||
* removed. For raw_tp programs attaching to tracepoints in kernel
|
||||
* modules, we mark input arguments as PTR_MAYBE_NULL, so branch
|
||||
* prediction should never kick in.
|
||||
*/
|
||||
barrier();
|
||||
/* If dead code elimination kicks in, the increment below will
|
||||
* be removed. For raw_tp programs, we mark input arguments as
|
||||
* PTR_MAYBE_NULL, so branch prediction should never kick in.
|
||||
*/
|
||||
if (!skb)
|
||||
i += 2;
|
||||
asm volatile ("%[i] += 1; if %[ctx] != 0 goto +1; %[i] += 2;"
|
||||
: [i]"+r"(i)
|
||||
: [ctx]"r"(skb)
|
||||
: "memory");
|
||||
return 0;
|
||||
}
|
||||
|
||||
24
tools/testing/selftests/bpf/progs/raw_tp_null_fail.c
Normal file
24
tools/testing/selftests/bpf/progs/raw_tp_null_fail.c
Normal file
@ -0,0 +1,24 @@
|
||||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
|
||||
|
||||
#include <vmlinux.h>
|
||||
#include <bpf/bpf_tracing.h>
|
||||
#include "bpf_misc.h"
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
||||
/* Ensure module parameter has PTR_MAYBE_NULL */
|
||||
SEC("tp_btf/bpf_testmod_test_raw_tp_null")
|
||||
__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'")
|
||||
int test_raw_tp_null_bpf_testmod_test_raw_tp_null_arg_1(void *ctx) {
|
||||
asm volatile("r1 = *(u64 *)(r1 +0); r1 = *(u64 *)(r1 +0);" ::: __clobber_all);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Check NULL marking */
|
||||
SEC("tp_btf/sched_pi_setprio")
|
||||
__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'")
|
||||
int test_raw_tp_null_sched_pi_setprio_arg_2(void *ctx) {
|
||||
asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u64 *)(r1 +0);" ::: __clobber_all);
|
||||
return 0;
|
||||
}
|
||||
@ -11,6 +11,8 @@ int subprog_tc(struct __sk_buff *skb)
|
||||
|
||||
__sink(skb);
|
||||
__sink(ret);
|
||||
/* let verifier know that 'subprog_tc' can change pointers to skb->data */
|
||||
bpf_skb_change_proto(skb, 0, 0);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
@ -7,11 +7,7 @@
|
||||
#include "bpf_misc.h"
|
||||
|
||||
SEC("tp_btf/bpf_testmod_test_nullable_bare")
|
||||
/* This used to be a failure test, but raw_tp nullable arguments can now
|
||||
* directly be dereferenced, whether they have nullable annotation or not,
|
||||
* and don't need to be explicitly checked.
|
||||
*/
|
||||
__success
|
||||
__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'")
|
||||
int BPF_PROG(handle_tp_btf_nullable_bare1, struct bpf_testmod_test_read_ctx *nullable_ctx)
|
||||
{
|
||||
return nullable_ctx->len;
|
||||
|
||||
@ -11,7 +11,7 @@ __success __retval(0)
|
||||
__naked void btf_ctx_access_accept(void)
|
||||
{
|
||||
asm volatile (" \
|
||||
r2 = *(u32*)(r1 + 8); /* load 2nd argument value (int pointer) */\
|
||||
r2 = *(u64 *)(r1 + 8); /* load 2nd argument value (int pointer) */\
|
||||
r0 = 0; \
|
||||
exit; \
|
||||
" ::: __clobber_all);
|
||||
@ -23,7 +23,43 @@ __success __retval(0)
|
||||
__naked void ctx_access_u32_pointer_accept(void)
|
||||
{
|
||||
asm volatile (" \
|
||||
r2 = *(u32*)(r1 + 0); /* load 1nd argument value (u32 pointer) */\
|
||||
r2 = *(u64 *)(r1 + 0); /* load 1nd argument value (u32 pointer) */\
|
||||
r0 = 0; \
|
||||
exit; \
|
||||
" ::: __clobber_all);
|
||||
}
|
||||
|
||||
SEC("fentry/bpf_fentry_test9")
|
||||
__description("btf_ctx_access u32 pointer reject u32")
|
||||
__failure __msg("size 4 must be 8")
|
||||
__naked void ctx_access_u32_pointer_reject_32(void)
|
||||
{
|
||||
asm volatile (" \
|
||||
r2 = *(u32 *)(r1 + 0); /* load 1st argument with narrow load */\
|
||||
r0 = 0; \
|
||||
exit; \
|
||||
" ::: __clobber_all);
|
||||
}
|
||||
|
||||
SEC("fentry/bpf_fentry_test9")
|
||||
__description("btf_ctx_access u32 pointer reject u16")
|
||||
__failure __msg("size 2 must be 8")
|
||||
__naked void ctx_access_u32_pointer_reject_16(void)
|
||||
{
|
||||
asm volatile (" \
|
||||
r2 = *(u16 *)(r1 + 0); /* load 1st argument with narrow load */\
|
||||
r0 = 0; \
|
||||
exit; \
|
||||
" ::: __clobber_all);
|
||||
}
|
||||
|
||||
SEC("fentry/bpf_fentry_test9")
|
||||
__description("btf_ctx_access u32 pointer reject u8")
|
||||
__failure __msg("size 1 must be 8")
|
||||
__naked void ctx_access_u32_pointer_reject_8(void)
|
||||
{
|
||||
asm volatile (" \
|
||||
r2 = *(u8 *)(r1 + 0); /* load 1st argument with narrow load */\
|
||||
r0 = 0; \
|
||||
exit; \
|
||||
" ::: __clobber_all);
|
||||
|
||||
@ -11,7 +11,7 @@ __success __retval(0)
|
||||
__naked void d_path_accept(void)
|
||||
{
|
||||
asm volatile (" \
|
||||
r1 = *(u32*)(r1 + 0); \
|
||||
r1 = *(u64 *)(r1 + 0); \
|
||||
r2 = r10; \
|
||||
r2 += -8; \
|
||||
r6 = 0; \
|
||||
@ -31,7 +31,7 @@ __failure __msg("helper call is not allowed in probe")
|
||||
__naked void d_path_reject(void)
|
||||
{
|
||||
asm volatile (" \
|
||||
r1 = *(u32*)(r1 + 0); \
|
||||
r1 = *(u64 *)(r1 + 0); \
|
||||
r2 = r10; \
|
||||
r2 += -8; \
|
||||
r6 = 0; \
|
||||
|
||||
@ -50,6 +50,13 @@ struct {
|
||||
__uint(map_flags, BPF_F_NO_PREALLOC);
|
||||
} sk_storage_map SEC(".maps");
|
||||
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
|
||||
__uint(max_entries, 1);
|
||||
__uint(key_size, sizeof(__u32));
|
||||
__uint(value_size, sizeof(__u32));
|
||||
} jmp_table SEC(".maps");
|
||||
|
||||
SEC("cgroup/skb")
|
||||
__description("skb->sk: no NULL check")
|
||||
__failure __msg("invalid mem access 'sock_common_or_null'")
|
||||
@ -1037,4 +1044,53 @@ __naked void sock_create_read_src_port(void)
|
||||
: __clobber_all);
|
||||
}
|
||||
|
||||
__noinline
|
||||
long skb_pull_data2(struct __sk_buff *sk, __u32 len)
|
||||
{
|
||||
return bpf_skb_pull_data(sk, len);
|
||||
}
|
||||
|
||||
__noinline
|
||||
long skb_pull_data1(struct __sk_buff *sk, __u32 len)
|
||||
{
|
||||
return skb_pull_data2(sk, len);
|
||||
}
|
||||
|
||||
/* global function calls bpf_skb_pull_data(), which invalidates packet
|
||||
* pointers established before global function call.
|
||||
*/
|
||||
SEC("tc")
|
||||
__failure __msg("invalid mem access")
|
||||
int invalidate_pkt_pointers_from_global_func(struct __sk_buff *sk)
|
||||
{
|
||||
int *p = (void *)(long)sk->data;
|
||||
|
||||
if ((void *)(p + 1) > (void *)(long)sk->data_end)
|
||||
return TCX_DROP;
|
||||
skb_pull_data1(sk, 0);
|
||||
*p = 42; /* this is unsafe */
|
||||
return TCX_PASS;
|
||||
}
|
||||
|
||||
__noinline
|
||||
int tail_call(struct __sk_buff *sk)
|
||||
{
|
||||
bpf_tail_call_static(sk, &jmp_table, 0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Tail calls invalidate packet pointers. */
|
||||
SEC("tc")
|
||||
__failure __msg("invalid mem access")
|
||||
int invalidate_pkt_pointers_by_tail_call(struct __sk_buff *sk)
|
||||
{
|
||||
int *p = (void *)(long)sk->data;
|
||||
|
||||
if ((void *)(p + 1) > (void *)(long)sk->data_end)
|
||||
return TCX_DROP;
|
||||
tail_call(sk);
|
||||
*p = 42; /* this is unsafe */
|
||||
return TCX_PASS;
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
||||
Loading…
Reference in New Issue
Block a user