Skip to content

Commit 6e3bf9b

Browse files
davem330davem330
committed
Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf
Daniel Borkmann says: ==================== pull-request: bpf 2018-08-18 The following pull-request contains BPF updates for your *net* tree. The main changes are: 1) Fix a BPF selftest failure in test_cgroup_storage due to rlimit restrictions, from Yonghong. 2) Fix a suspicious RCU rcu_dereference_check() warning triggered from removing a device's XDP memory allocator by using the correct rhashtable lookup function, from Tariq. 3) A batch of BPF sockmap and ULP fixes mainly fixing leaks and races as well as enforcing module aliases for ULPs. Another fix for BPF map redirect to make them work again with tail calls, from Daniel. 4) Fix XDP BPF samples to unload their programs upon SIGTERM, from Jesper. ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
2 parents 3fe49d6 + f6069b9 commit 6e3bf9b

File tree

16 files changed

+123
-130
lines changed

16 files changed

+123
-130
lines changed

‎include/linux/filter.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -543,7 +543,6 @@ struct bpf_redirect_info {
543543
u32 flags;
544544
struct bpf_map *map;
545545
struct bpf_map *map_to_flush;
546-
unsigned long map_owner;
547546
u32 kern_flags;
548547
};
549548

@@ -781,6 +780,8 @@ static inline bool bpf_dump_raw_ok(void)
781780
struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
782781
const struct bpf_insn *patch, u32 len);
783782

783+
void bpf_clear_redirect_map(struct bpf_map *map);
784+
784785
static inline bool xdp_return_frame_no_direct(void)
785786
{
786787
struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);

‎include/net/tcp.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2065,6 +2065,10 @@ int tcp_set_ulp_id(struct sock *sk, const int ulp);
20652065
void tcp_get_available_ulp(char *buf, size_t len);
20662066
void tcp_cleanup_ulp(struct sock *sk);
20672067

2068+
#define MODULE_ALIAS_TCP_ULP(name) \
2069+
__MODULE_INFO(alias, alias_userspace, name); \
2070+
__MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name)
2071+
20682072
/* Call BPF_SOCK_OPS program that returns an int. If the return value
20692073
* is < 0, then the BPF op failed (for example if the loaded BPF
20702074
* program does not support the chosen operation or there is no BPF

‎include/trace/events/xdp.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,9 +147,8 @@ struct _bpf_dtab_netdev {
147147

148148
#define devmap_ifindex(fwd, map) \
149149
(!fwd ? 0 : \
150-
(!map ? 0 : \
151-
((map->map_type == BPF_MAP_TYPE_DEVMAP) ? \
152-
((struct _bpf_dtab_netdev *)fwd)->dev->ifindex : 0)))
150+
((map->map_type == BPF_MAP_TYPE_DEVMAP) ? \
151+
((struct _bpf_dtab_netdev *)fwd)->dev->ifindex : 0))
153152

154153
#define _trace_xdp_redirect_map(dev, xdp, fwd, map, idx) \
155154
trace_xdp_redirect_map(dev, xdp, devmap_ifindex(fwd, map), \

‎kernel/bpf/core.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1579,7 +1579,7 @@ static bool bpf_prog_array_copy_core(struct bpf_prog_array __rcu *array,
15791579
struct bpf_prog_array_item *item;
15801580
int i = 0;
15811581

1582-
item = rcu_dereference(array)->items;
1582+
item = rcu_dereference_check(array, 1)->items;
15831583
for (; item->prog; item++) {
15841584
if (item->prog == &dummy_bpf_prog.prog)
15851585
continue;

‎kernel/bpf/cpumap.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -479,6 +479,8 @@ static void cpu_map_free(struct bpf_map *map)
479479
* It does __not__ ensure pending flush operations (if any) are
480480
* complete.
481481
*/
482+
483+
bpf_clear_redirect_map(map);
482484
synchronize_rcu();
483485

484486
/* To ensure all pending flush operations have completed wait for flush

‎kernel/bpf/devmap.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ static void dev_map_free(struct bpf_map *map)
161161
list_del_rcu(&dtab->list);
162162
spin_unlock(&dev_map_lock);
163163

164+
bpf_clear_redirect_map(map);
164165
synchronize_rcu();
165166

166167
/* To ensure all pending flush operations have completed wait for flush

‎kernel/bpf/sockmap.c

Lines changed: 68 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ struct bpf_stab {
5858
struct bpf_map map;
5959
struct sock **sock_map;
6060
struct bpf_sock_progs progs;
61+
raw_spinlock_t lock;
6162
};
6263

6364
struct bucket {
@@ -89,9 +90,9 @@ enum smap_psock_state {
8990

9091
struct smap_psock_map_entry {
9192
struct list_head list;
93+
struct bpf_map *map;
9294
struct sock **entry;
9395
struct htab_elem __rcu *hash_link;
94-
struct bpf_htab __rcu *htab;
9596
};
9697

9798
struct smap_psock {
@@ -343,13 +344,18 @@ static void bpf_tcp_close(struct sock *sk, long timeout)
343344
e = psock_map_pop(sk, psock);
344345
while (e) {
345346
if (e->entry) {
346-
osk = cmpxchg(e->entry, sk, NULL);
347+
struct bpf_stab *stab = container_of(e->map, struct bpf_stab, map);
348+
349+
raw_spin_lock_bh(&stab->lock);
350+
osk = *e->entry;
347351
if (osk == sk) {
352+
*e->entry = NULL;
348353
smap_release_sock(psock, sk);
349354
}
355+
raw_spin_unlock_bh(&stab->lock);
350356
} else {
351357
struct htab_elem *link = rcu_dereference(e->hash_link);
352-
struct bpf_htab *htab = rcu_dereference(e->htab);
358+
struct bpf_htab *htab = container_of(e->map, struct bpf_htab, map);
353359
struct hlist_head *head;
354360
struct htab_elem *l;
355361
struct bucket *b;
@@ -370,6 +376,7 @@ static void bpf_tcp_close(struct sock *sk, long timeout)
370376
}
371377
raw_spin_unlock_bh(&b->lock);
372378
}
379+
kfree(e);
373380
e = psock_map_pop(sk, psock);
374381
}
375382
rcu_read_unlock();
@@ -1641,6 +1648,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
16411648
return ERR_PTR(-ENOMEM);
16421649

16431650
bpf_map_init_from_attr(&stab->map, attr);
1651+
raw_spin_lock_init(&stab->lock);
16441652

16451653
/* make sure page count doesn't overflow */
16461654
cost = (u64) stab->map.max_entries * sizeof(struct sock *);
@@ -1675,8 +1683,10 @@ static void smap_list_map_remove(struct smap_psock *psock,
16751683

16761684
spin_lock_bh(&psock->maps_lock);
16771685
list_for_each_entry_safe(e, tmp, &psock->maps, list) {
1678-
if (e->entry == entry)
1686+
if (e->entry == entry) {
16791687
list_del(&e->list);
1688+
kfree(e);
1689+
}
16801690
}
16811691
spin_unlock_bh(&psock->maps_lock);
16821692
}
@@ -1690,8 +1700,10 @@ static void smap_list_hash_remove(struct smap_psock *psock,
16901700
list_for_each_entry_safe(e, tmp, &psock->maps, list) {
16911701
struct htab_elem *c = rcu_dereference(e->hash_link);
16921702

1693-
if (c == hash_link)
1703+
if (c == hash_link) {
16941704
list_del(&e->list);
1705+
kfree(e);
1706+
}
16951707
}
16961708
spin_unlock_bh(&psock->maps_lock);
16971709
}
@@ -1711,14 +1723,15 @@ static void sock_map_free(struct bpf_map *map)
17111723
* and a grace period expire to ensure psock is really safe to remove.
17121724
*/
17131725
rcu_read_lock();
1726+
raw_spin_lock_bh(&stab->lock);
17141727
for (i = 0; i < stab->map.max_entries; i++) {
17151728
struct smap_psock *psock;
17161729
struct sock *sock;
17171730

1718-
sock = xchg(&stab->sock_map[i], NULL);
1731+
sock = stab->sock_map[i];
17191732
if (!sock)
17201733
continue;
1721-
1734+
stab->sock_map[i] = NULL;
17221735
psock = smap_psock_sk(sock);
17231736
/* This check handles a racing sock event that can get the
17241737
* sk_callback_lock before this case but after xchg happens
@@ -1730,6 +1743,7 @@ static void sock_map_free(struct bpf_map *map)
17301743
smap_release_sock(psock, sock);
17311744
}
17321745
}
1746+
raw_spin_unlock_bh(&stab->lock);
17331747
rcu_read_unlock();
17341748

17351749
sock_map_remove_complete(stab);
@@ -1773,19 +1787,23 @@ static int sock_map_delete_elem(struct bpf_map *map, void *key)
17731787
if (k >= map->max_entries)
17741788
return -EINVAL;
17751789

1776-
sock = xchg(&stab->sock_map[k], NULL);
1790+
raw_spin_lock_bh(&stab->lock);
1791+
sock = stab->sock_map[k];
1792+
stab->sock_map[k] = NULL;
1793+
raw_spin_unlock_bh(&stab->lock);
17771794
if (!sock)
17781795
return -EINVAL;
17791796

17801797
psock = smap_psock_sk(sock);
17811798
if (!psock)
1782-
goto out;
1783-
1784-
if (psock->bpf_parse)
1799+
return 0;
1800+
if (psock->bpf_parse) {
1801+
write_lock_bh(&sock->sk_callback_lock);
17851802
smap_stop_sock(psock, sock);
1803+
write_unlock_bh(&sock->sk_callback_lock);
1804+
}
17861805
smap_list_map_remove(psock, &stab->sock_map[k]);
17871806
smap_release_sock(psock, sock);
1788-
out:
17891807
return 0;
17901808
}
17911809

@@ -1821,11 +1839,9 @@ static int sock_map_delete_elem(struct bpf_map *map, void *key)
18211839
static int __sock_map_ctx_update_elem(struct bpf_map *map,
18221840
struct bpf_sock_progs *progs,
18231841
struct sock *sock,
1824-
struct sock **map_link,
18251842
void *key)
18261843
{
18271844
struct bpf_prog *verdict, *parse, *tx_msg;
1828-
struct smap_psock_map_entry *e = NULL;
18291845
struct smap_psock *psock;
18301846
bool new = false;
18311847
int err = 0;
@@ -1898,14 +1914,6 @@ static int __sock_map_ctx_update_elem(struct bpf_map *map,
18981914
new = true;
18991915
}
19001916

1901-
if (map_link) {
1902-
e = kzalloc(sizeof(*e), GFP_ATOMIC | __GFP_NOWARN);
1903-
if (!e) {
1904-
err = -ENOMEM;
1905-
goto out_free;
1906-
}
1907-
}
1908-
19091917
/* 3. At this point we have a reference to a valid psock that is
19101918
* running. Attach any BPF programs needed.
19111919
*/
@@ -1927,17 +1935,6 @@ static int __sock_map_ctx_update_elem(struct bpf_map *map,
19271935
write_unlock_bh(&sock->sk_callback_lock);
19281936
}
19291937

1930-
/* 4. Place psock in sockmap for use and stop any programs on
1931-
* the old sock assuming its not the same sock we are replacing
1932-
* it with. Because we can only have a single set of programs if
1933-
* old_sock has a strp we can stop it.
1934-
*/
1935-
if (map_link) {
1936-
e->entry = map_link;
1937-
spin_lock_bh(&psock->maps_lock);
1938-
list_add_tail(&e->list, &psock->maps);
1939-
spin_unlock_bh(&psock->maps_lock);
1940-
}
19411938
return err;
19421939
out_free:
19431940
smap_release_sock(psock, sock);
@@ -1948,7 +1945,6 @@ static int __sock_map_ctx_update_elem(struct bpf_map *map,
19481945
}
19491946
if (tx_msg)
19501947
bpf_prog_put(tx_msg);
1951-
kfree(e);
19521948
return err;
19531949
}
19541950

@@ -1958,36 +1954,57 @@ static int sock_map_ctx_update_elem(struct bpf_sock_ops_kern *skops,
19581954
{
19591955
struct bpf_stab *stab = container_of(map, struct bpf_stab, map);
19601956
struct bpf_sock_progs *progs = &stab->progs;
1961-
struct sock *osock, *sock;
1957+
struct sock *osock, *sock = skops->sk;
1958+
struct smap_psock_map_entry *e;
1959+
struct smap_psock *psock;
19621960
u32 i = *(u32 *)key;
19631961
int err;
19641962

19651963
if (unlikely(flags > BPF_EXIST))
19661964
return -EINVAL;
1967-
19681965
if (unlikely(i >= stab->map.max_entries))
19691966
return -E2BIG;
19701967

1971-
sock = READ_ONCE(stab->sock_map[i]);
1972-
if (flags == BPF_EXIST && !sock)
1973-
return -ENOENT;
1974-
else if (flags == BPF_NOEXIST && sock)
1975-
return -EEXIST;
1968+
e = kzalloc(sizeof(*e), GFP_ATOMIC | __GFP_NOWARN);
1969+
if (!e)
1970+
return -ENOMEM;
19761971

1977-
sock = skops->sk;
1978-
err = __sock_map_ctx_update_elem(map, progs, sock, &stab->sock_map[i],
1979-
key);
1972+
err = __sock_map_ctx_update_elem(map, progs, sock, key);
19801973
if (err)
19811974
goto out;
19821975

1983-
osock = xchg(&stab->sock_map[i], sock);
1984-
if (osock) {
1985-
struct smap_psock *opsock = smap_psock_sk(osock);
1976+
/* psock guaranteed to be present. */
1977+
psock = smap_psock_sk(sock);
1978+
raw_spin_lock_bh(&stab->lock);
1979+
osock = stab->sock_map[i];
1980+
if (osock && flags == BPF_NOEXIST) {
1981+
err = -EEXIST;
1982+
goto out_unlock;
1983+
}
1984+
if (!osock && flags == BPF_EXIST) {
1985+
err = -ENOENT;
1986+
goto out_unlock;
1987+
}
19861988

1987-
smap_list_map_remove(opsock, &stab->sock_map[i]);
1988-
smap_release_sock(opsock, osock);
1989+
e->entry = &stab->sock_map[i];
1990+
e->map = map;
1991+
spin_lock_bh(&psock->maps_lock);
1992+
list_add_tail(&e->list, &psock->maps);
1993+
spin_unlock_bh(&psock->maps_lock);
1994+
1995+
stab->sock_map[i] = sock;
1996+
if (osock) {
1997+
psock = smap_psock_sk(osock);
1998+
smap_list_map_remove(psock, &stab->sock_map[i]);
1999+
smap_release_sock(psock, osock);
19892000
}
2001+
raw_spin_unlock_bh(&stab->lock);
2002+
return 0;
2003+
out_unlock:
2004+
smap_release_sock(psock, sock);
2005+
raw_spin_unlock_bh(&stab->lock);
19902006
out:
2007+
kfree(e);
19912008
return err;
19922009
}
19932010

@@ -2350,7 +2367,7 @@ static int sock_hash_ctx_update_elem(struct bpf_sock_ops_kern *skops,
23502367
b = __select_bucket(htab, hash);
23512368
head = &b->head;
23522369

2353-
err = __sock_map_ctx_update_elem(map, progs, sock, NULL, key);
2370+
err = __sock_map_ctx_update_elem(map, progs, sock, key);
23542371
if (err)
23552372
goto err;
23562373

@@ -2376,8 +2393,7 @@ static int sock_hash_ctx_update_elem(struct bpf_sock_ops_kern *skops,
23762393
}
23772394

23782395
rcu_assign_pointer(e->hash_link, l_new);
2379-
rcu_assign_pointer(e->htab,
2380-
container_of(map, struct bpf_htab, map));
2396+
e->map = map;
23812397
spin_lock_bh(&psock->maps_lock);
23822398
list_add_tail(&e->list, &psock->maps);
23832399
spin_unlock_bh(&psock->maps_lock);

‎kernel/bpf/verifier.c

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5844,27 +5844,6 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
58445844
goto patch_call_imm;
58455845
}
58465846

5847-
if (insn->imm == BPF_FUNC_redirect_map) {
5848-
/* Note, we cannot use prog directly as imm as subsequent
5849-
* rewrites would still change the prog pointer. The only
5850-
* stable address we can use is aux, which also works with
5851-
* prog clones during blinding.
5852-
*/
5853-
u64 addr = (unsigned long)prog->aux;
5854-
struct bpf_insn r4_ld[] = {
5855-
BPF_LD_IMM64(BPF_REG_4, addr),
5856-
*insn,
5857-
};
5858-
cnt = ARRAY_SIZE(r4_ld);
5859-
5860-
new_prog = bpf_patch_insn_data(env, i + delta, r4_ld, cnt);
5861-
if (!new_prog)
5862-
return -ENOMEM;
5863-
5864-
delta += cnt - 1;
5865-
env->prog = prog = new_prog;
5866-
insn = new_prog->insnsi + i + delta;
5867-
}
58685847
patch_call_imm:
58695848
fn = env->ops->get_func_proto(insn->imm, env->prog);
58705849
/* all functions that have prototype and verifier allowed

‎kernel/bpf/xskmap.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ static void xsk_map_free(struct bpf_map *map)
7575
struct xsk_map *m = container_of(map, struct xsk_map, map);
7676
int i;
7777

78+
bpf_clear_redirect_map(map);
7879
synchronize_net();
7980

8081
for (i = 0; i < map->max_entries; i++) {

0 commit comments

Comments
 (0)