Skip to content

Commit 394e40a

Browse files
rgushchinborkmann
rgushchin
authored and
borkmann
committed
bpf: extend bpf_prog_array to store pointers to the cgroup storage
This patch converts bpf_prog_array from an array of prog pointers to the array of struct bpf_prog_array_item elements. This allows to save a cgroup storage pointer for each bpf program efficiently attached to a cgroup. Signed-off-by: Roman Gushchin <guro@fb.com> Cc: Alexei Starovoitov <ast@kernel.org> Cc: Daniel Borkmann <daniel@iogearbox.net> Acked-by: Martin KaFai Lau <kafai@fb.com> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
1 parent d7bf2c1 commit 394e40a

File tree

4 files changed

+70
-56
lines changed

4 files changed

+70
-56
lines changed

‎drivers/media/rc/bpf-lirc.c

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -195,14 +195,16 @@ void lirc_bpf_run(struct rc_dev *rcdev, u32 sample)
195195
*/
196196
void lirc_bpf_free(struct rc_dev *rcdev)
197197
{
198-
struct bpf_prog **progs;
198+
struct bpf_prog_array_item *item;
199199

200200
if (!rcdev->raw->progs)
201201
return;
202202

203-
progs = rcu_dereference(rcdev->raw->progs)->progs;
204-
while (*progs)
205-
bpf_prog_put(*progs++);
203+
item = rcu_dereference(rcdev->raw->progs)->items;
204+
while (item->prog) {
205+
bpf_prog_put(item->prog);
206+
item++;
207+
}
206208

207209
bpf_prog_array_free(rcdev->raw->progs);
208210
}

‎include/linux/bpf.h

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -349,9 +349,14 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
349349
* The 'struct bpf_prog_array *' should only be replaced with xchg()
350350
* since other cpus are walking the array of pointers in parallel.
351351
*/
352+
struct bpf_prog_array_item {
353+
struct bpf_prog *prog;
354+
struct bpf_cgroup_storage *cgroup_storage;
355+
};
356+
352357
struct bpf_prog_array {
353358
struct rcu_head rcu;
354-
struct bpf_prog *progs[0];
359+
struct bpf_prog_array_item items[0];
355360
};
356361

357362
struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
@@ -372,18 +377,20 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
372377

373378
#define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \
374379
({ \
375-
struct bpf_prog **_prog, *__prog; \
380+
struct bpf_prog_array_item *_item; \
381+
struct bpf_prog *_prog; \
376382
struct bpf_prog_array *_array; \
377383
u32 _ret = 1; \
378384
preempt_disable(); \
379385
rcu_read_lock(); \
380386
_array = rcu_dereference(array); \
381387
if (unlikely(check_non_null && !_array))\
382388
goto _out; \
383-
_prog = _array->progs; \
384-
while ((__prog = READ_ONCE(*_prog))) { \
385-
_ret &= func(__prog, ctx); \
386-
_prog++; \
389+
_item = &_array->items[0]; \
390+
while ((_prog = READ_ONCE(_item->prog))) { \
391+
bpf_cgroup_storage_set(_item->cgroup_storage); \
392+
_ret &= func(_prog, ctx); \
393+
_item++; \
387394
} \
388395
_out: \
389396
rcu_read_unlock(); \

‎kernel/bpf/cgroup.c

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -117,15 +117,18 @@ static int compute_effective_progs(struct cgroup *cgrp,
117117
cnt = 0;
118118
p = cgrp;
119119
do {
120-
if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
121-
list_for_each_entry(pl,
122-
&p->bpf.progs[type], node) {
123-
if (!pl->prog)
124-
continue;
125-
progs->progs[cnt++] = pl->prog;
126-
}
127-
p = cgroup_parent(p);
128-
} while (p);
120+
if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
121+
continue;
122+
123+
list_for_each_entry(pl, &p->bpf.progs[type], node) {
124+
if (!pl->prog)
125+
continue;
126+
127+
progs->items[cnt].prog = pl->prog;
128+
progs->items[cnt].cgroup_storage = pl->storage;
129+
cnt++;
130+
}
131+
} while ((p = cgroup_parent(p)));
129132

130133
rcu_assign_pointer(*array, progs);
131134
return 0;

‎kernel/bpf/core.c

Lines changed: 39 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1542,7 +1542,8 @@ struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
15421542
{
15431543
if (prog_cnt)
15441544
return kzalloc(sizeof(struct bpf_prog_array) +
1545-
sizeof(struct bpf_prog *) * (prog_cnt + 1),
1545+
sizeof(struct bpf_prog_array_item) *
1546+
(prog_cnt + 1),
15461547
flags);
15471548

15481549
return &empty_prog_array.hdr;
@@ -1556,43 +1557,45 @@ void bpf_prog_array_free(struct bpf_prog_array __rcu *progs)
15561557
kfree_rcu(progs, rcu);
15571558
}
15581559

1559-
int bpf_prog_array_length(struct bpf_prog_array __rcu *progs)
1560+
int bpf_prog_array_length(struct bpf_prog_array __rcu *array)
15601561
{
1561-
struct bpf_prog **prog;
1562+
struct bpf_prog_array_item *item;
15621563
u32 cnt = 0;
15631564

15641565
rcu_read_lock();
1565-
prog = rcu_dereference(progs)->progs;
1566-
for (; *prog; prog++)
1567-
if (*prog != &dummy_bpf_prog.prog)
1566+
item = rcu_dereference(array)->items;
1567+
for (; item->prog; item++)
1568+
if (item->prog != &dummy_bpf_prog.prog)
15681569
cnt++;
15691570
rcu_read_unlock();
15701571
return cnt;
15711572
}
15721573

1573-
static bool bpf_prog_array_copy_core(struct bpf_prog **prog,
1574+
1575+
static bool bpf_prog_array_copy_core(struct bpf_prog_array __rcu *array,
15741576
u32 *prog_ids,
15751577
u32 request_cnt)
15761578
{
1579+
struct bpf_prog_array_item *item;
15771580
int i = 0;
15781581

1579-
for (; *prog; prog++) {
1580-
if (*prog == &dummy_bpf_prog.prog)
1582+
item = rcu_dereference(array)->items;
1583+
for (; item->prog; item++) {
1584+
if (item->prog == &dummy_bpf_prog.prog)
15811585
continue;
1582-
prog_ids[i] = (*prog)->aux->id;
1586+
prog_ids[i] = item->prog->aux->id;
15831587
if (++i == request_cnt) {
1584-
prog++;
1588+
item++;
15851589
break;
15861590
}
15871591
}
15881592

1589-
return !!(*prog);
1593+
return !!(item->prog);
15901594
}
15911595

1592-
int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
1596+
int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *array,
15931597
__u32 __user *prog_ids, u32 cnt)
15941598
{
1595-
struct bpf_prog **prog;
15961599
unsigned long err = 0;
15971600
bool nospc;
15981601
u32 *ids;
@@ -1611,8 +1614,7 @@ int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
16111614
if (!ids)
16121615
return -ENOMEM;
16131616
rcu_read_lock();
1614-
prog = rcu_dereference(progs)->progs;
1615-
nospc = bpf_prog_array_copy_core(prog, ids, cnt);
1617+
nospc = bpf_prog_array_copy_core(array, ids, cnt);
16161618
rcu_read_unlock();
16171619
err = copy_to_user(prog_ids, ids, cnt * sizeof(u32));
16181620
kfree(ids);
@@ -1623,14 +1625,14 @@ int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
16231625
return 0;
16241626
}
16251627

1626-
void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *progs,
1628+
void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *array,
16271629
struct bpf_prog *old_prog)
16281630
{
1629-
struct bpf_prog **prog = progs->progs;
1631+
struct bpf_prog_array_item *item = array->items;
16301632

1631-
for (; *prog; prog++)
1632-
if (*prog == old_prog) {
1633-
WRITE_ONCE(*prog, &dummy_bpf_prog.prog);
1633+
for (; item->prog; item++)
1634+
if (item->prog == old_prog) {
1635+
WRITE_ONCE(item->prog, &dummy_bpf_prog.prog);
16341636
break;
16351637
}
16361638
}
@@ -1641,7 +1643,7 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
16411643
struct bpf_prog_array **new_array)
16421644
{
16431645
int new_prog_cnt, carry_prog_cnt = 0;
1644-
struct bpf_prog **existing_prog;
1646+
struct bpf_prog_array_item *existing;
16451647
struct bpf_prog_array *array;
16461648
bool found_exclude = false;
16471649
int new_prog_idx = 0;
@@ -1650,15 +1652,15 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
16501652
* the new array.
16511653
*/
16521654
if (old_array) {
1653-
existing_prog = old_array->progs;
1654-
for (; *existing_prog; existing_prog++) {
1655-
if (*existing_prog == exclude_prog) {
1655+
existing = old_array->items;
1656+
for (; existing->prog; existing++) {
1657+
if (existing->prog == exclude_prog) {
16561658
found_exclude = true;
16571659
continue;
16581660
}
1659-
if (*existing_prog != &dummy_bpf_prog.prog)
1661+
if (existing->prog != &dummy_bpf_prog.prog)
16601662
carry_prog_cnt++;
1661-
if (*existing_prog == include_prog)
1663+
if (existing->prog == include_prog)
16621664
return -EEXIST;
16631665
}
16641666
}
@@ -1684,15 +1686,17 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
16841686

16851687
/* Fill in the new prog array */
16861688
if (carry_prog_cnt) {
1687-
existing_prog = old_array->progs;
1688-
for (; *existing_prog; existing_prog++)
1689-
if (*existing_prog != exclude_prog &&
1690-
*existing_prog != &dummy_bpf_prog.prog)
1691-
array->progs[new_prog_idx++] = *existing_prog;
1689+
existing = old_array->items;
1690+
for (; existing->prog; existing++)
1691+
if (existing->prog != exclude_prog &&
1692+
existing->prog != &dummy_bpf_prog.prog) {
1693+
array->items[new_prog_idx++].prog =
1694+
existing->prog;
1695+
}
16921696
}
16931697
if (include_prog)
1694-
array->progs[new_prog_idx++] = include_prog;
1695-
array->progs[new_prog_idx] = NULL;
1698+
array->items[new_prog_idx++].prog = include_prog;
1699+
array->items[new_prog_idx].prog = NULL;
16961700
*new_array = array;
16971701
return 0;
16981702
}
@@ -1701,7 +1705,6 @@ int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
17011705
u32 *prog_ids, u32 request_cnt,
17021706
u32 *prog_cnt)
17031707
{
1704-
struct bpf_prog **prog;
17051708
u32 cnt = 0;
17061709

17071710
if (array)
@@ -1714,8 +1717,7 @@ int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
17141717
return 0;
17151718

17161719
/* this function is called under trace/bpf_trace.c: bpf_event_mutex */
1717-
prog = rcu_dereference_check(array, 1)->progs;
1718-
return bpf_prog_array_copy_core(prog, prog_ids, request_cnt) ? -ENOSPC
1720+
return bpf_prog_array_copy_core(array, prog_ids, request_cnt) ? -ENOSPC
17191721
: 0;
17201722
}
17211723

0 commit comments

Comments
 (0)