@@ -132,6 +132,7 @@ struct smap_psock {
132
132
struct work_struct gc_work ;
133
133
134
134
struct proto * sk_proto ;
135
+ void (* save_unhash )(struct sock * sk );
135
136
void (* save_close )(struct sock * sk , long timeout );
136
137
void (* save_data_ready )(struct sock * sk );
137
138
void (* save_write_space )(struct sock * sk );
@@ -143,6 +144,7 @@ static int bpf_tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
143
144
static int bpf_tcp_sendmsg (struct sock * sk , struct msghdr * msg , size_t size );
144
145
static int bpf_tcp_sendpage (struct sock * sk , struct page * page ,
145
146
int offset , size_t size , int flags );
147
+ static void bpf_tcp_unhash (struct sock * sk );
146
148
static void bpf_tcp_close (struct sock * sk , long timeout );
147
149
148
150
static inline struct smap_psock * smap_psock_sk (const struct sock * sk )
@@ -184,6 +186,7 @@ static void build_protos(struct proto prot[SOCKMAP_NUM_CONFIGS],
184
186
struct proto * base )
185
187
{
186
188
prot [SOCKMAP_BASE ] = * base ;
189
+ prot [SOCKMAP_BASE ].unhash = bpf_tcp_unhash ;
187
190
prot [SOCKMAP_BASE ].close = bpf_tcp_close ;
188
191
prot [SOCKMAP_BASE ].recvmsg = bpf_tcp_recvmsg ;
189
192
prot [SOCKMAP_BASE ].stream_memory_read = bpf_tcp_stream_read ;
@@ -217,6 +220,7 @@ static int bpf_tcp_init(struct sock *sk)
217
220
return - EBUSY ;
218
221
}
219
222
223
+ psock -> save_unhash = sk -> sk_prot -> unhash ;
220
224
psock -> save_close = sk -> sk_prot -> close ;
221
225
psock -> sk_proto = sk -> sk_prot ;
222
226
@@ -305,30 +309,12 @@ static struct smap_psock_map_entry *psock_map_pop(struct sock *sk,
305
309
return e ;
306
310
}
307
311
308
- static void bpf_tcp_close (struct sock * sk , long timeout )
312
+ static void bpf_tcp_remove (struct sock * sk , struct smap_psock * psock )
309
313
{
310
- void (* close_fun )(struct sock * sk , long timeout );
311
314
struct smap_psock_map_entry * e ;
312
315
struct sk_msg_buff * md , * mtmp ;
313
- struct smap_psock * psock ;
314
316
struct sock * osk ;
315
317
316
- lock_sock (sk );
317
- rcu_read_lock ();
318
- psock = smap_psock_sk (sk );
319
- if (unlikely (!psock )) {
320
- rcu_read_unlock ();
321
- release_sock (sk );
322
- return sk -> sk_prot -> close (sk , timeout );
323
- }
324
-
325
- /* The psock may be destroyed anytime after exiting the RCU critial
326
- * section so by the time we use close_fun the psock may no longer
327
- * be valid. However, bpf_tcp_close is called with the sock lock
328
- * held so the close hook and sk are still valid.
329
- */
330
- close_fun = psock -> save_close ;
331
-
332
318
if (psock -> cork ) {
333
319
free_start_sg (psock -> sock , psock -> cork , true);
334
320
kfree (psock -> cork );
@@ -379,6 +365,42 @@ static void bpf_tcp_close(struct sock *sk, long timeout)
379
365
kfree (e );
380
366
e = psock_map_pop (sk , psock );
381
367
}
368
+ }
369
+
370
+ static void bpf_tcp_unhash (struct sock * sk )
371
+ {
372
+ void (* unhash_fun )(struct sock * sk );
373
+ struct smap_psock * psock ;
374
+
375
+ rcu_read_lock ();
376
+ psock = smap_psock_sk (sk );
377
+ if (unlikely (!psock )) {
378
+ rcu_read_unlock ();
379
+ if (sk -> sk_prot -> unhash )
380
+ sk -> sk_prot -> unhash (sk );
381
+ return ;
382
+ }
383
+ unhash_fun = psock -> save_unhash ;
384
+ bpf_tcp_remove (sk , psock );
385
+ rcu_read_unlock ();
386
+ unhash_fun (sk );
387
+ }
388
+
389
+ static void bpf_tcp_close (struct sock * sk , long timeout )
390
+ {
391
+ void (* close_fun )(struct sock * sk , long timeout );
392
+ struct smap_psock * psock ;
393
+
394
+ lock_sock (sk );
395
+ rcu_read_lock ();
396
+ psock = smap_psock_sk (sk );
397
+ if (unlikely (!psock )) {
398
+ rcu_read_unlock ();
399
+ release_sock (sk );
400
+ return sk -> sk_prot -> close (sk , timeout );
401
+ }
402
+ close_fun = psock -> save_close ;
403
+ bpf_tcp_remove (sk , psock );
382
404
rcu_read_unlock ();
383
405
release_sock (sk );
384
406
close_fun (sk , timeout );
@@ -2097,8 +2119,12 @@ static int sock_map_update_elem(struct bpf_map *map,
2097
2119
return - EINVAL ;
2098
2120
}
2099
2121
2122
+ /* ULPs are currently supported only for TCP sockets in ESTABLISHED
2123
+ * state.
2124
+ */
2100
2125
if (skops .sk -> sk_type != SOCK_STREAM ||
2101
- skops .sk -> sk_protocol != IPPROTO_TCP ) {
2126
+ skops .sk -> sk_protocol != IPPROTO_TCP ||
2127
+ skops .sk -> sk_state != TCP_ESTABLISHED ) {
2102
2128
fput (socket -> file );
2103
2129
return - EOPNOTSUPP ;
2104
2130
}
@@ -2453,6 +2479,16 @@ static int sock_hash_update_elem(struct bpf_map *map,
2453
2479
return - EINVAL ;
2454
2480
}
2455
2481
2482
+ /* ULPs are currently supported only for TCP sockets in ESTABLISHED
2483
+ * state.
2484
+ */
2485
+ if (skops .sk -> sk_type != SOCK_STREAM ||
2486
+ skops .sk -> sk_protocol != IPPROTO_TCP ||
2487
+ skops .sk -> sk_state != TCP_ESTABLISHED ) {
2488
+ fput (socket -> file );
2489
+ return - EOPNOTSUPP ;
2490
+ }
2491
+
2456
2492
lock_sock (skops .sk );
2457
2493
preempt_disable ();
2458
2494
rcu_read_lock ();
@@ -2543,10 +2579,22 @@ const struct bpf_map_ops sock_hash_ops = {
2543
2579
.map_check_btf = map_check_no_btf ,
2544
2580
};
2545
2581
2582
+ static bool bpf_is_valid_sock_op (struct bpf_sock_ops_kern * ops )
2583
+ {
2584
+ return ops -> op == BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB ||
2585
+ ops -> op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB ;
2586
+ }
2546
2587
BPF_CALL_4 (bpf_sock_map_update , struct bpf_sock_ops_kern * , bpf_sock ,
2547
2588
struct bpf_map * , map , void * , key , u64 , flags )
2548
2589
{
2549
2590
WARN_ON_ONCE (!rcu_read_lock_held ());
2591
+
2592
+ /* ULPs are currently supported only for TCP sockets in ESTABLISHED
2593
+ * state. This checks that the sock ops triggering the update is
2594
+ * one indicating we are (or will be soon) in an ESTABLISHED state.
2595
+ */
2596
+ if (!bpf_is_valid_sock_op (bpf_sock ))
2597
+ return - EOPNOTSUPP ;
2550
2598
return sock_map_ctx_update_elem (bpf_sock , map , key , flags );
2551
2599
}
2552
2600
@@ -2565,6 +2613,9 @@ BPF_CALL_4(bpf_sock_hash_update, struct bpf_sock_ops_kern *, bpf_sock,
2565
2613
struct bpf_map * , map , void * , key , u64 , flags )
2566
2614
{
2567
2615
WARN_ON_ONCE (!rcu_read_lock_held ());
2616
+
2617
+ if (!bpf_is_valid_sock_op (bpf_sock ))
2618
+ return - EOPNOTSUPP ;
2568
2619
return sock_hash_ctx_update_elem (bpf_sock , map , key , flags );
2569
2620
}
2570
2621
0 commit comments