Skip to content

Commit c454a46

Browse files
iamkafaiAlexei Starovoitov
iamkafai
authored and
Alexei Starovoitov
committed
bpf: Add bpf_line_info support
This patch adds bpf_line_info support. It accepts an array of bpf_line_info objects during BPF_PROG_LOAD. The "line_info", "line_info_cnt" and "line_info_rec_size" are added to the "union bpf_attr". The "line_info_rec_size" makes bpf_line_info extensible in the future. The new "check_btf_line()" ensures the userspace line_info is valid for the kernel to use. When the verifier is translating/patching the bpf_prog (through "bpf_patch_insn_single()"), the line_infos' insn_off is also adjusted by the newly added "bpf_adj_linfo()". If the bpf_prog is jited, this patch also provides the jited addrs (in aux->jited_linfo) for the corresponding line_info.insn_off. "bpf_prog_fill_jited_linfo()" is added to fill the aux->jited_linfo. It is currently called by the x86 jit. Other jits can also use "bpf_prog_fill_jited_linfo()" and it will be done in the followup patches. In the future, if it deemed necessary, a particular jit could also provide its own "bpf_prog_fill_jited_linfo()" implementation. A few "*line_info*" fields are added to the bpf_prog_info such that the user can get the xlated line_info back (i.e. the line_info with its insn_off reflecting the translated prog). The jited_line_info is available if the prog is jited. It is an array of __u64. If the prog is not jited, jited_line_info_cnt is 0. The verifier's verbose log with line_info will be done in a follow up patch. Signed-off-by: Martin KaFai Lau <kafai@fb.com> Acked-by: Yonghong Song <yhs@fb.com> Signed-off-by: Alexei Starovoitov <ast@kernel.org>
1 parent 6baefa1 commit c454a46

File tree

10 files changed

+419
-33
lines changed

10 files changed

+419
-33
lines changed

‎arch/x86/net/bpf_jit_comp.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1181,6 +1181,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
11811181
}
11821182

11831183
if (!image || !prog->is_func || extra_pass) {
1184+
if (image)
1185+
bpf_prog_fill_jited_linfo(prog, addrs);
11841186
out_addrs:
11851187
kfree(addrs);
11861188
kfree(jit_data);

‎include/linux/bpf.h

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,28 @@ struct bpf_prog_aux {
319319
struct bpf_prog_offload *offload;
320320
struct btf *btf;
321321
struct bpf_func_info *func_info;
322+
/* bpf_line_info loaded from userspace. linfo->insn_off
323+
* has the xlated insn offset.
324+
* Both the main and sub prog share the same linfo.
325+
* The subprog can access its first linfo by
326+
* using the linfo_idx.
327+
*/
328+
struct bpf_line_info *linfo;
329+
/* jited_linfo is the jited addr of the linfo. It has a
330+
* one to one mapping to linfo:
331+
* jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
332+
* Both the main and sub prog share the same jited_linfo.
333+
* The subprog can access its first jited_linfo by
334+
* using the linfo_idx.
335+
*/
336+
void **jited_linfo;
322337
u32 func_info_cnt;
338+
u32 nr_linfo;
339+
/* subprog can use linfo_idx to access its first linfo and
340+
* jited_linfo.
341+
* main prog always has linfo_idx == 0
342+
*/
343+
u32 linfo_idx;
323344
union {
324345
struct work_struct work;
325346
struct rcu_head rcu;

‎include/linux/bpf_verifier.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,7 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
203203

204204
struct bpf_subprog_info {
205205
u32 start; /* insn idx of function entry point */
206+
u32 linfo_idx; /* The idx to the main_prog->aux->linfo */
206207
u16 stack_depth; /* max. stack depth used by this function */
207208
};
208209

‎include/linux/btf.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
4646
struct seq_file *m);
4747
int btf_get_fd_by_id(u32 id);
4848
u32 btf_id(const struct btf *btf);
49+
bool btf_name_offset_valid(const struct btf *btf, u32 offset);
4950

5051
#ifdef CONFIG_BPF_SYSCALL
5152
const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id);

‎include/linux/filter.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -718,6 +718,13 @@ void bpf_prog_free(struct bpf_prog *fp);
718718

719719
bool bpf_opcode_in_insntable(u8 code);
720720

721+
void bpf_prog_free_linfo(struct bpf_prog *prog);
722+
void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
723+
const u32 *insn_to_jit_off);
724+
int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog);
725+
void bpf_prog_free_jited_linfo(struct bpf_prog *prog);
726+
void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);
727+
721728
struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
722729
struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
723730
gfp_t gfp_extra_flags);

‎include/uapi/linux/bpf.h

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,9 @@ union bpf_attr {
356356
__u32 func_info_rec_size; /* userspace bpf_func_info size */
357357
__aligned_u64 func_info; /* func info */
358358
__u32 func_info_cnt; /* number of bpf_func_info records */
359+
__u32 line_info_rec_size; /* userspace bpf_line_info size */
360+
__aligned_u64 line_info; /* line info */
361+
__u32 line_info_cnt; /* number of bpf_line_info records */
359362
};
360363

361364
struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -2679,6 +2682,12 @@ struct bpf_prog_info {
26792682
__u32 func_info_rec_size;
26802683
__aligned_u64 func_info;
26812684
__u32 func_info_cnt;
2685+
__u32 line_info_cnt;
2686+
__aligned_u64 line_info;
2687+
__aligned_u64 jited_line_info;
2688+
__u32 jited_line_info_cnt;
2689+
__u32 line_info_rec_size;
2690+
__u32 jited_line_info_rec_size;
26822691
} __attribute__((aligned(8)));
26832692

26842693
struct bpf_map_info {
@@ -2995,4 +3004,14 @@ struct bpf_func_info {
29953004
__u32 type_id;
29963005
};
29973006

3007+
#define BPF_LINE_INFO_LINE_NUM(line_col) ((line_col) >> 10)
3008+
#define BPF_LINE_INFO_LINE_COL(line_col) ((line_col) & 0x3ff)
3009+
3010+
struct bpf_line_info {
3011+
__u32 insn_off;
3012+
__u32 file_name_off;
3013+
__u32 line_off;
3014+
__u32 line_col;
3015+
};
3016+
29983017
#endif /* _UAPI__LINUX_BPF_H__ */

‎kernel/bpf/btf.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -444,7 +444,7 @@ static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
444444
return kind_ops[BTF_INFO_KIND(t->info)];
445445
}
446446

447-
static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
447+
bool btf_name_offset_valid(const struct btf *btf, u32 offset)
448448
{
449449
return BTF_STR_OFFSET_VALID(offset) &&
450450
offset < btf->hdr.str_len;

‎kernel/bpf/core.c

Lines changed: 116 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,91 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
105105
}
106106
EXPORT_SYMBOL_GPL(bpf_prog_alloc);
107107

108+
int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)
109+
{
110+
if (!prog->aux->nr_linfo || !prog->jit_requested)
111+
return 0;
112+
113+
prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo,
114+
sizeof(*prog->aux->jited_linfo),
115+
GFP_KERNEL | __GFP_NOWARN);
116+
if (!prog->aux->jited_linfo)
117+
return -ENOMEM;
118+
119+
return 0;
120+
}
121+
122+
void bpf_prog_free_jited_linfo(struct bpf_prog *prog)
123+
{
124+
kfree(prog->aux->jited_linfo);
125+
prog->aux->jited_linfo = NULL;
126+
}
127+
128+
void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog)
129+
{
130+
if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0])
131+
bpf_prog_free_jited_linfo(prog);
132+
}
133+
134+
/* The jit engine is responsible to provide an array
135+
* for insn_off to the jited_off mapping (insn_to_jit_off).
136+
*
137+
* The idx to this array is the insn_off. Hence, the insn_off
138+
* here is relative to the prog itself instead of the main prog.
139+
* This array has one entry for each xlated bpf insn.
140+
*
141+
* jited_off is the byte off to the last byte of the jited insn.
142+
*
143+
* Hence, with
144+
* insn_start:
145+
* The first bpf insn off of the prog. The insn off
146+
* here is relative to the main prog.
147+
* e.g. if prog is a subprog, insn_start > 0
148+
* linfo_idx:
149+
* The prog's idx to prog->aux->linfo and jited_linfo
150+
*
151+
* jited_linfo[linfo_idx] = prog->bpf_func
152+
*
153+
* For i > linfo_idx,
154+
*
155+
* jited_linfo[i] = prog->bpf_func +
156+
* insn_to_jit_off[linfo[i].insn_off - insn_start - 1]
157+
*/
158+
void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,
159+
const u32 *insn_to_jit_off)
160+
{
161+
u32 linfo_idx, insn_start, insn_end, nr_linfo, i;
162+
const struct bpf_line_info *linfo;
163+
void **jited_linfo;
164+
165+
if (!prog->aux->jited_linfo)
166+
/* Userspace did not provide linfo */
167+
return;
168+
169+
linfo_idx = prog->aux->linfo_idx;
170+
linfo = &prog->aux->linfo[linfo_idx];
171+
insn_start = linfo[0].insn_off;
172+
insn_end = insn_start + prog->len;
173+
174+
jited_linfo = &prog->aux->jited_linfo[linfo_idx];
175+
jited_linfo[0] = prog->bpf_func;
176+
177+
nr_linfo = prog->aux->nr_linfo - linfo_idx;
178+
179+
for (i = 1; i < nr_linfo && linfo[i].insn_off < insn_end; i++)
180+
/* The verifier ensures that linfo[i].insn_off is
181+
* strictly increasing
182+
*/
183+
jited_linfo[i] = prog->bpf_func +
184+
insn_to_jit_off[linfo[i].insn_off - insn_start - 1];
185+
}
186+
187+
void bpf_prog_free_linfo(struct bpf_prog *prog)
188+
{
189+
bpf_prog_free_jited_linfo(prog);
190+
kvfree(prog->aux->linfo);
191+
}
192+
108193
struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
109194
gfp_t gfp_extra_flags)
110195
{
@@ -294,6 +379,26 @@ static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta,
294379
return ret;
295380
}
296381

382+
static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta)
383+
{
384+
struct bpf_line_info *linfo;
385+
u32 i, nr_linfo;
386+
387+
nr_linfo = prog->aux->nr_linfo;
388+
if (!nr_linfo || !delta)
389+
return;
390+
391+
linfo = prog->aux->linfo;
392+
393+
for (i = 0; i < nr_linfo; i++)
394+
if (off < linfo[i].insn_off)
395+
break;
396+
397+
/* Push all off < linfo[i].insn_off by delta */
398+
for (; i < nr_linfo; i++)
399+
linfo[i].insn_off += delta;
400+
}
401+
297402
struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
298403
const struct bpf_insn *patch, u32 len)
299404
{
@@ -349,6 +454,8 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
349454
*/
350455
BUG_ON(bpf_adj_branches(prog_adj, off, insn_delta, false));
351456

457+
bpf_adj_linfo(prog_adj, off, insn_delta);
458+
352459
return prog_adj;
353460
}
354461

@@ -1591,13 +1698,20 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
15911698
* be JITed, but falls back to the interpreter.
15921699
*/
15931700
if (!bpf_prog_is_dev_bound(fp->aux)) {
1701+
*err = bpf_prog_alloc_jited_linfo(fp);
1702+
if (*err)
1703+
return fp;
1704+
15941705
fp = bpf_int_jit_compile(fp);
1595-
#ifdef CONFIG_BPF_JIT_ALWAYS_ON
15961706
if (!fp->jited) {
1707+
bpf_prog_free_jited_linfo(fp);
1708+
#ifdef CONFIG_BPF_JIT_ALWAYS_ON
15971709
*err = -ENOTSUPP;
15981710
return fp;
1599-
}
16001711
#endif
1712+
} else {
1713+
bpf_prog_free_unused_jited_linfo(fp);
1714+
}
16011715
} else {
16021716
*err = bpf_prog_offload_compile(fp);
16031717
if (*err)

‎kernel/bpf/syscall.c

Lines changed: 77 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1215,6 +1215,7 @@ static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
12151215
bpf_prog_kallsyms_del_all(prog);
12161216
btf_put(prog->aux->btf);
12171217
kvfree(prog->aux->func_info);
1218+
bpf_prog_free_linfo(prog);
12181219

12191220
call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
12201221
}
@@ -1439,7 +1440,7 @@ bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type,
14391440
}
14401441

14411442
/* last field in 'union bpf_attr' used by this command */
1442-
#define BPF_PROG_LOAD_LAST_FIELD func_info_cnt
1443+
#define BPF_PROG_LOAD_LAST_FIELD line_info_cnt
14431444

14441445
static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
14451446
{
@@ -1560,6 +1561,7 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
15601561
return err;
15611562

15621563
free_used_maps:
1564+
bpf_prog_free_linfo(prog);
15631565
kvfree(prog->aux->func_info);
15641566
btf_put(prog->aux->btf);
15651567
bpf_prog_kallsyms_del_subprogs(prog);
@@ -2041,6 +2043,37 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog)
20412043
return insns;
20422044
}
20432045

2046+
static int set_info_rec_size(struct bpf_prog_info *info)
2047+
{
2048+
/*
2049+
* Ensure info.*_rec_size is the same as kernel expected size
2050+
*
2051+
* or
2052+
*
2053+
* Only allow zero *_rec_size if both _rec_size and _cnt are
2054+
* zero. In this case, the kernel will set the expected
2055+
* _rec_size back to the info.
2056+
*/
2057+
2058+
if ((info->func_info_cnt || info->func_info_rec_size) &&
2059+
info->func_info_rec_size != sizeof(struct bpf_func_info))
2060+
return -EINVAL;
2061+
2062+
if ((info->line_info_cnt || info->line_info_rec_size) &&
2063+
info->line_info_rec_size != sizeof(struct bpf_line_info))
2064+
return -EINVAL;
2065+
2066+
if ((info->jited_line_info_cnt || info->jited_line_info_rec_size) &&
2067+
info->jited_line_info_rec_size != sizeof(__u64))
2068+
return -EINVAL;
2069+
2070+
info->func_info_rec_size = sizeof(struct bpf_func_info);
2071+
info->line_info_rec_size = sizeof(struct bpf_line_info);
2072+
info->jited_line_info_rec_size = sizeof(__u64);
2073+
2074+
return 0;
2075+
}
2076+
20442077
static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
20452078
const union bpf_attr *attr,
20462079
union bpf_attr __user *uattr)
@@ -2083,18 +2116,18 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
20832116
return -EFAULT;
20842117
}
20852118

2086-
if ((info.func_info_cnt || info.func_info_rec_size) &&
2087-
info.func_info_rec_size != sizeof(struct bpf_func_info))
2088-
return -EINVAL;
2089-
2090-
info.func_info_rec_size = sizeof(struct bpf_func_info);
2119+
err = set_info_rec_size(&info);
2120+
if (err)
2121+
return err;
20912122

20922123
if (!capable(CAP_SYS_ADMIN)) {
20932124
info.jited_prog_len = 0;
20942125
info.xlated_prog_len = 0;
20952126
info.nr_jited_ksyms = 0;
20962127
info.nr_jited_func_lens = 0;
20972128
info.func_info_cnt = 0;
2129+
info.line_info_cnt = 0;
2130+
info.jited_line_info_cnt = 0;
20982131
goto done;
20992132
}
21002133

@@ -2251,6 +2284,44 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
22512284
}
22522285
}
22532286

2287+
ulen = info.line_info_cnt;
2288+
info.line_info_cnt = prog->aux->nr_linfo;
2289+
if (info.line_info_cnt && ulen) {
2290+
if (bpf_dump_raw_ok()) {
2291+
__u8 __user *user_linfo;
2292+
2293+
user_linfo = u64_to_user_ptr(info.line_info);
2294+
ulen = min_t(u32, info.line_info_cnt, ulen);
2295+
if (copy_to_user(user_linfo, prog->aux->linfo,
2296+
info.line_info_rec_size * ulen))
2297+
return -EFAULT;
2298+
} else {
2299+
info.line_info = 0;
2300+
}
2301+
}
2302+
2303+
ulen = info.jited_line_info_cnt;
2304+
if (prog->aux->jited_linfo)
2305+
info.jited_line_info_cnt = prog->aux->nr_linfo;
2306+
else
2307+
info.jited_line_info_cnt = 0;
2308+
if (info.jited_line_info_cnt && ulen) {
2309+
if (bpf_dump_raw_ok()) {
2310+
__u64 __user *user_linfo;
2311+
u32 i;
2312+
2313+
user_linfo = u64_to_user_ptr(info.jited_line_info);
2314+
ulen = min_t(u32, info.jited_line_info_cnt, ulen);
2315+
for (i = 0; i < ulen; i++) {
2316+
if (put_user((__u64)(long)prog->aux->jited_linfo[i],
2317+
&user_linfo[i]))
2318+
return -EFAULT;
2319+
}
2320+
} else {
2321+
info.jited_line_info = 0;
2322+
}
2323+
}
2324+
22542325
done:
22552326
if (copy_to_user(uinfo, &info, info_len) ||
22562327
put_user(info_len, &uattr->info.info_len))

0 commit comments

Comments
 (0)