Skip to content

Commit

Permalink
Merge branch 'bpf-verifier-log-rotation'
Browse files Browse the repository at this point in the history
Andrii Nakryiko says:

====================
This patch set changes BPF verifier log behavior to behave as a rotating log,
by default. If user-supplied log buffer is big enough to contain entire
verifier log output, there is no effective difference. But where previously
user supplied too small log buffer and would get -ENOSPC error result and the
beginning part of the verifier log, now there will be no error and user will
get ending part of verifier log filling up user-supplied log buffer.  Which
is, in absolute majority of cases, is exactly what's useful, relevant, and
what users want and need, as the ending of the verifier log is containing
details of verifier failure and relevant state that got us to that failure.
So this rotating mode is made default, but for some niche advanced debugging
scenarios it's possible to request old behavior by specifying additional
BPF_LOG_FIXED (8) flag.

This patch set adjusts libbpf to allow specifying flags beyond 1 | 2 | 4. We
also add --log-size and --log-fixed options to veristat to be able to both
test this functionality manually, but also to be used in various debugging
scenarios. We also add selftests that tries many variants of log buffer size
to stress-test correctness of internal verifier log bookkeeping code.

Further, this patch set is merged with log_size_actual v1 patchset ([0]),
which adds ability to get required log buffer size to fit entire verifier
log output.

This addresses a long-standing limitation, which causes users and BPF loader
library writers to guess and pre-size log buffer, often allocating unnecessary
extra memory for this or doing extra program verifications just to size logs
better, ultimately wasting resources. This was requested most recently by Go
BPF library maintainers ([1]).

See respective patches for details. A bunch of them some drive-by fixes
detecting during working with the code. Some other further refactor and
compratmentalize verifier log handling code into kernel/bpf/log.c, which
should also make it simpler to integrate such verbose log for other
complicated bpf() syscall commands, if necessary. The rest are actual logic
to calculate maximum log buffer size needed and return it to user-space.
Few patches wire this on libbpf side, and the rest add selftests to test
proper log truncation and log_buf==NULL handling.

This turned into a pretty sizable patch set with lots of arithmetics, but
hopefully the set of features added to verifier log in this patch set are
both useful for BPF users and are self-contained and isolated enough to not
cause troubles going forward.

v3->v4:
  - s/log_size_actual/log_true_size/ (Alexei);
  - log_buf==NULL && log_size==0 don't trigger -ENOSPC (Lorenz);
  - added WARN_ON_ONCE if we try bpf_vlog_reset() forward (Lorenz);
  - added selftests for truncation in BPF_LOG_FIXED mode;
  - fixed edge case in BPF_LOG_FIXED when log_size==1, leaving buf not zero
    terminated;
v2->v3:
  - typos and comment improvement (Lorenz);
  - merged with log_size_actual v1 ([0]) patch set (Alexei);
  - added log_buf==NULL condition allowed (Lorenz);
  - added BPF_BTF_LOAD logs tests (Lorenz);
  - more clean up and refactoring of internal verifier log API;
v1->v2:
  - return -ENOSPC even in rotating log mode for preserving backwards
    compatibility (Lorenz);

  [0] https://patchwork.kernel.org/project/netdevbpf/list/?series=735213&state=*
  [1] https://lore.kernel.org/bpf/CAN+4W8iNoEbQzQVbB_o1W0MWBDV4xCJAq7K3f6psVE-kkCfMqg@mail.gmail.com/
====================

Signed-off-by: Daniel Borkmann <[email protected]>
  • Loading branch information
borkmann committed Apr 11, 2023
2 parents eafa921 + 054b6c7 commit 255f0e1
Show file tree
Hide file tree
Showing 15 changed files with 965 additions and 184 deletions.
2 changes: 1 addition & 1 deletion include/linux/bpf.h
Original file line number Diff line number Diff line change
Expand Up @@ -2175,7 +2175,7 @@ int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
size_t actual_size);

/* verify correctness of eBPF program */
int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr);
int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);

#ifndef CONFIG_BPF_JIT_ALWAYS_ON
void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
Expand Down
41 changes: 21 additions & 20 deletions include/linux/bpf_verifier.h
Original file line number Diff line number Diff line change
Expand Up @@ -491,39 +491,36 @@ struct bpf_insn_aux_data {
#define BPF_VERIFIER_TMP_LOG_SIZE 1024

struct bpf_verifier_log {
u32 level;
char kbuf[BPF_VERIFIER_TMP_LOG_SIZE];
/* Logical start and end positions of a "log window" of the verifier log.
* start_pos == 0 means we haven't truncated anything.
* Once truncation starts to happen, start_pos + len_total == end_pos,
* except during log reset situations, in which (end_pos - start_pos)
* might get smaller than len_total (see bpf_vlog_reset()).
* Generally, (end_pos - start_pos) gives number of useful data in
* user log buffer.
*/
u64 start_pos;
u64 end_pos;
char __user *ubuf;
u32 len_used;
u32 level;
u32 len_total;
u32 len_max;
char kbuf[BPF_VERIFIER_TMP_LOG_SIZE];
};

static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log)
{
return log->len_used >= log->len_total - 1;
}

#define BPF_LOG_LEVEL1 1
#define BPF_LOG_LEVEL2 2
#define BPF_LOG_STATS 4
#define BPF_LOG_FIXED 8
#define BPF_LOG_LEVEL (BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)
#define BPF_LOG_MASK (BPF_LOG_LEVEL | BPF_LOG_STATS)
#define BPF_LOG_MASK (BPF_LOG_LEVEL | BPF_LOG_STATS | BPF_LOG_FIXED)
#define BPF_LOG_KERNEL (BPF_LOG_MASK + 1) /* kernel internal flag */
#define BPF_LOG_MIN_ALIGNMENT 8U
#define BPF_LOG_ALIGNMENT 40U

static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
{
return log &&
((log->level && log->ubuf && !bpf_verifier_log_full(log)) ||
log->level == BPF_LOG_KERNEL);
}

static inline bool
bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)
{
return log->len_total >= 128 && log->len_total <= UINT_MAX >> 2 &&
log->level && log->ubuf && !(log->level & ~BPF_LOG_MASK);
return log && log->level;
}

#define BPF_MAX_SUBPROGS 256
Expand Down Expand Up @@ -603,7 +600,7 @@ struct bpf_verifier_env {
u32 scratched_regs;
/* Same as scratched_regs but for stack slots */
u64 scratched_stack_slots;
u32 prev_log_len, prev_insn_print_len;
u64 prev_log_pos, prev_insn_print_pos;
/* buffer used in reg_type_str() to generate reg_type string */
char type_str_buf[TYPE_STR_BUF_LEN];
};
Expand All @@ -614,6 +611,10 @@ __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
const char *fmt, ...);
__printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
const char *fmt, ...);
int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,
char __user *log_buf, u32 log_size);
void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos);
int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual);

static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)
{
Expand Down
2 changes: 1 addition & 1 deletion include/linux/btf.h
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ extern const struct file_operations btf_fops;

void btf_get(struct btf *btf);
void btf_put(struct btf *btf);
int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr);
int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_sz);
struct btf *btf_get_by_fd(int fd);
int btf_get_info_by_fd(const struct btf *btf,
const union bpf_attr *attr,
Expand Down
10 changes: 10 additions & 0 deletions include/uapi/linux/bpf.h
Original file line number Diff line number Diff line change
Expand Up @@ -1407,6 +1407,11 @@ union bpf_attr {
__aligned_u64 fd_array; /* array of FDs */
__aligned_u64 core_relos;
__u32 core_relo_rec_size; /* sizeof(struct bpf_core_relo) */
/* output: actual total log contents size (including termintaing zero).
* It could be both larger than original log_size (if log was
* truncated), or smaller (if log buffer wasn't filled completely).
*/
__u32 log_true_size;
};

struct { /* anonymous struct used by BPF_OBJ_* commands */
Expand Down Expand Up @@ -1492,6 +1497,11 @@ union bpf_attr {
__u32 btf_size;
__u32 btf_log_size;
__u32 btf_log_level;
/* output: actual total log contents size (including termintaing zero).
* It could be both larger than original log_size (if log was
* truncated), or smaller (if log buffer wasn't filled completely).
*/
__u32 btf_log_true_size;
};

struct {
Expand Down
3 changes: 2 additions & 1 deletion kernel/bpf/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse
endif
CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy)

obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o
obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o
obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o
obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o
obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
Expand Down
74 changes: 41 additions & 33 deletions kernel/bpf/btf.c
Original file line number Diff line number Diff line change
Expand Up @@ -5504,38 +5504,45 @@ static int btf_check_type_tags(struct btf_verifier_env *env,
return 0;
}

static struct btf *btf_parse(bpfptr_t btf_data, u32 btf_data_size,
u32 log_level, char __user *log_ubuf, u32 log_size)
static int finalize_log(struct bpf_verifier_log *log, bpfptr_t uattr, u32 uattr_size)
{
u32 log_true_size;
int err;

err = bpf_vlog_finalize(log, &log_true_size);

if (uattr_size >= offsetofend(union bpf_attr, btf_log_true_size) &&
copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, btf_log_true_size),
&log_true_size, sizeof(log_true_size)))
err = -EFAULT;

return err;
}

static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
{
bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel);
char __user *log_ubuf = u64_to_user_ptr(attr->btf_log_buf);
struct btf_struct_metas *struct_meta_tab;
struct btf_verifier_env *env = NULL;
struct bpf_verifier_log *log;
struct btf *btf = NULL;
u8 *data;
int err;
int err, ret;

if (btf_data_size > BTF_MAX_SIZE)
if (attr->btf_size > BTF_MAX_SIZE)
return ERR_PTR(-E2BIG);

env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
if (!env)
return ERR_PTR(-ENOMEM);

log = &env->log;
if (log_level || log_ubuf || log_size) {
/* user requested verbose verifier output
* and supplied buffer to store the verification trace
*/
log->level = log_level;
log->ubuf = log_ubuf;
log->len_total = log_size;

/* log attributes have to be sane */
if (!bpf_verifier_log_attr_valid(log)) {
err = -EINVAL;
goto errout;
}
}
/* user could have requested verbose verifier output
* and supplied buffer to store the verification trace
*/
err = bpf_vlog_init(&env->log, attr->btf_log_level,
log_ubuf, attr->btf_log_size);
if (err)
goto errout_free;

btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
if (!btf) {
Expand All @@ -5544,16 +5551,16 @@ static struct btf *btf_parse(bpfptr_t btf_data, u32 btf_data_size,
}
env->btf = btf;

data = kvmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN);
data = kvmalloc(attr->btf_size, GFP_KERNEL | __GFP_NOWARN);
if (!data) {
err = -ENOMEM;
goto errout;
}

btf->data = data;
btf->data_size = btf_data_size;
btf->data_size = attr->btf_size;

if (copy_from_bpfptr(data, btf_data, btf_data_size)) {
if (copy_from_bpfptr(data, btf_data, attr->btf_size)) {
err = -EFAULT;
goto errout;
}
Expand All @@ -5576,7 +5583,7 @@ static struct btf *btf_parse(bpfptr_t btf_data, u32 btf_data_size,
if (err)
goto errout;

struct_meta_tab = btf_parse_struct_metas(log, btf);
struct_meta_tab = btf_parse_struct_metas(&env->log, btf);
if (IS_ERR(struct_meta_tab)) {
err = PTR_ERR(struct_meta_tab);
goto errout;
Expand All @@ -5593,10 +5600,9 @@ static struct btf *btf_parse(bpfptr_t btf_data, u32 btf_data_size,
}
}

if (log->level && bpf_verifier_log_full(log)) {
err = -ENOSPC;
goto errout_meta;
}
err = finalize_log(&env->log, uattr, uattr_size);
if (err)
goto errout_free;

btf_verifier_env_free(env);
refcount_set(&btf->refcnt, 1);
Expand All @@ -5605,6 +5611,11 @@ static struct btf *btf_parse(bpfptr_t btf_data, u32 btf_data_size,
errout_meta:
btf_free_struct_meta_tab(btf);
errout:
/* overwrite err with -ENOSPC or -EFAULT */
ret = finalize_log(&env->log, uattr, uattr_size);
if (ret)
err = ret;
errout_free:
btf_verifier_env_free(env);
if (btf)
btf_free(btf);
Expand Down Expand Up @@ -7213,15 +7224,12 @@ static int __btf_new_fd(struct btf *btf)
return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC);
}

int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr)
int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
{
struct btf *btf;
int ret;

btf = btf_parse(make_bpfptr(attr->btf, uattr.is_kernel),
attr->btf_size, attr->btf_log_level,
u64_to_user_ptr(attr->btf_log_buf),
attr->btf_log_size);
btf = btf_parse(attr, uattr, uattr_size);
if (IS_ERR(btf))
return PTR_ERR(btf);

Expand Down
Loading

0 comments on commit 255f0e1

Please sign in to comment.