Skip to content

Commit efe4208

Browse files
edumazetdavem330
edumazet
authored and
davem330
committed
ipv6: make lookups simpler and faster
TCP listener refactoring, part 4 : To speed up inet lookups, we moved IPv4 addresses from inet to struct sock_common Now is time to do the same for IPv6, because it permits us to have fast lookups for all kind of sockets, including upcoming SYN_RECV. Getting IPv6 addresses in TCP lookups currently requires two extra cache lines, plus a dereference (and memory stall). inet6_sk(sk) does the dereference of inet_sk(__sk)->pinet6 This patch is way bigger than its IPv4 counter part, because for IPv4, we could add aliases (inet_daddr, inet_rcv_saddr), while on IPv6, it's not doable easily. inet6_sk(sk)->daddr becomes sk->sk_v6_daddr inet6_sk(sk)->rcv_saddr becomes sk->sk_v6_rcv_saddr And timewait socket also have tw->tw_v6_daddr & tw->tw_v6_rcv_saddr at the same offset. We get rid of INET6_TW_MATCH() as INET6_MATCH() is now the generic macro. Signed-off-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 05dbc7b commit efe4208

35 files changed

+213
-288
lines changed

‎include/linux/ipv6.h

Lines changed: 6 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,6 @@ struct ipv6_fl_socklist;
141141
*/
142142
struct ipv6_pinfo {
143143
struct in6_addr saddr;
144-
struct in6_addr rcv_saddr;
145-
struct in6_addr daddr;
146144
struct in6_pktinfo sticky_pktinfo;
147145
const struct in6_addr *daddr_cache;
148146
#ifdef CONFIG_IPV6_SUBTREES
@@ -256,22 +254,10 @@ struct tcp6_sock {
256254

257255
extern int inet6_sk_rebuild_header(struct sock *sk);
258256

259-
struct inet6_timewait_sock {
260-
struct in6_addr tw_v6_daddr;
261-
struct in6_addr tw_v6_rcv_saddr;
262-
};
263-
264257
struct tcp6_timewait_sock {
265258
struct tcp_timewait_sock tcp6tw_tcp;
266-
struct inet6_timewait_sock tcp6tw_inet6;
267259
};
268260

269-
static inline struct inet6_timewait_sock *inet6_twsk(const struct sock *sk)
270-
{
271-
return (struct inet6_timewait_sock *)(((u8 *)sk) +
272-
inet_twsk(sk)->tw_ipv6_offset);
273-
}
274-
275261
#if IS_ENABLED(CONFIG_IPV6)
276262
static inline struct ipv6_pinfo * inet6_sk(const struct sock *__sk)
277263
{
@@ -321,21 +307,11 @@ static inline void inet_sk_copy_descendant(struct sock *sk_to,
321307
#define __ipv6_only_sock(sk) (inet6_sk(sk)->ipv6only)
322308
#define ipv6_only_sock(sk) ((sk)->sk_family == PF_INET6 && __ipv6_only_sock(sk))
323309

324-
static inline u16 inet6_tw_offset(const struct proto *prot)
325-
{
326-
return prot->twsk_prot->twsk_obj_size -
327-
sizeof(struct inet6_timewait_sock);
328-
}
329-
330-
static inline struct in6_addr *__inet6_rcv_saddr(const struct sock *sk)
310+
static inline const struct in6_addr *inet6_rcv_saddr(const struct sock *sk)
331311
{
332-
return likely(sk->sk_state != TCP_TIME_WAIT) ?
333-
&inet6_sk(sk)->rcv_saddr : &inet6_twsk(sk)->tw_v6_rcv_saddr;
334-
}
335-
336-
static inline struct in6_addr *inet6_rcv_saddr(const struct sock *sk)
337-
{
338-
return sk->sk_family == AF_INET6 ? __inet6_rcv_saddr(sk) : NULL;
312+
if (sk->sk_family == AF_INET6)
313+
return &sk->sk_v6_rcv_saddr;
314+
return NULL;
339315
}
340316

341317
static inline int inet_v6_ipv6only(const struct sock *sk)
@@ -363,7 +339,6 @@ static inline struct raw6_sock *raw6_sk(const struct sock *sk)
363339
return NULL;
364340
}
365341

366-
#define __inet6_rcv_saddr(__sk) NULL
367342
#define inet6_rcv_saddr(__sk) NULL
368343
#define tcp_twsk_ipv6only(__sk) 0
369344
#define inet_v6_ipv6only(__sk) 0
@@ -372,19 +347,10 @@ static inline struct raw6_sock *raw6_sk(const struct sock *sk)
372347
#define INET6_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif) \
373348
(((__sk)->sk_portpair == (__ports)) && \
374349
((__sk)->sk_family == AF_INET6) && \
375-
ipv6_addr_equal(&inet6_sk(__sk)->daddr, (__saddr)) && \
376-
ipv6_addr_equal(&inet6_sk(__sk)->rcv_saddr, (__daddr)) && \
350+
ipv6_addr_equal(&(__sk)->sk_v6_daddr, (__saddr)) && \
351+
ipv6_addr_equal(&(__sk)->sk_v6_rcv_saddr, (__daddr)) && \
377352
(!(__sk)->sk_bound_dev_if || \
378353
((__sk)->sk_bound_dev_if == (__dif))) && \
379354
net_eq(sock_net(__sk), (__net)))
380355

381-
#define INET6_TW_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif) \
382-
(((__sk)->sk_portpair == (__ports)) && \
383-
((__sk)->sk_family == AF_INET6) && \
384-
ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_daddr, (__saddr)) && \
385-
ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_rcv_saddr, (__daddr)) && \
386-
(!(__sk)->sk_bound_dev_if || \
387-
((__sk)->sk_bound_dev_if == (__dif))) && \
388-
net_eq(sock_net(__sk), (__net)))
389-
390356
#endif /* _IPV6_H */

‎include/net/inet6_hashtables.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,8 @@ static inline unsigned int inet6_ehashfn(struct net *net,
4343
static inline int inet6_sk_ehashfn(const struct sock *sk)
4444
{
4545
const struct inet_sock *inet = inet_sk(sk);
46-
const struct ipv6_pinfo *np = inet6_sk(sk);
47-
const struct in6_addr *laddr = &np->rcv_saddr;
48-
const struct in6_addr *faddr = &np->daddr;
46+
const struct in6_addr *laddr = &sk->sk_v6_rcv_saddr;
47+
const struct in6_addr *faddr = &sk->sk_v6_daddr;
4948
const __u16 lport = inet->inet_num;
5049
const __be16 fport = inet->inet_dport;
5150
struct net *net = sock_net(sk);

‎include/net/inet_timewait_sock.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,9 @@ struct inet_timewait_sock {
116116
#define tw_prot __tw_common.skc_prot
117117
#define tw_net __tw_common.skc_net
118118
#define tw_daddr __tw_common.skc_daddr
119+
#define tw_v6_daddr __tw_common.skc_v6_daddr
119120
#define tw_rcv_saddr __tw_common.skc_rcv_saddr
121+
#define tw_v6_rcv_saddr __tw_common.skc_v6_rcv_saddr
120122
#define tw_dport __tw_common.skc_dport
121123
#define tw_num __tw_common.skc_num
122124

@@ -133,7 +135,7 @@ struct inet_timewait_sock {
133135
tw_transparent : 1,
134136
tw_pad : 6, /* 6 bits hole */
135137
tw_tos : 8,
136-
tw_ipv6_offset : 16;
138+
tw_pad2 : 16 /* 16 bits hole */
137139
kmemcheck_bitfield_end(flags);
138140
u32 tw_ttd;
139141
struct inet_bind_bucket *tw_tb;

‎include/net/ip.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ static __inline__ void inet_reset_saddr(struct sock *sk)
374374
struct ipv6_pinfo *np = inet6_sk(sk);
375375

376376
memset(&np->saddr, 0, sizeof(np->saddr));
377-
memset(&np->rcv_saddr, 0, sizeof(np->rcv_saddr));
377+
memset(&sk->sk_v6_rcv_saddr, 0, sizeof(sk->sk_v6_rcv_saddr));
378378
}
379379
#endif
380380
}

‎include/net/ip6_checksum.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ static inline void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)
7070
{
7171
struct ipv6_pinfo *np = inet6_sk(sk);
7272

73-
__tcp_v6_send_check(skb, &np->saddr, &np->daddr);
73+
__tcp_v6_send_check(skb, &np->saddr, &sk->sk_v6_daddr);
7474
}
7575

7676
int udp6_csum_init(struct sk_buff *skb, struct udphdr *uh, int proto);

‎include/net/sock.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,12 @@ struct sock_common {
191191
#ifdef CONFIG_NET_NS
192192
struct net *skc_net;
193193
#endif
194+
195+
#if IS_ENABLED(CONFIG_IPV6)
196+
struct in6_addr skc_v6_daddr;
197+
struct in6_addr skc_v6_rcv_saddr;
198+
#endif
199+
194200
/*
195201
* fields between dontcopy_begin/dontcopy_end
196202
* are not copied in sock_copy()
@@ -314,6 +320,9 @@ struct sock {
314320
#define sk_bind_node __sk_common.skc_bind_node
315321
#define sk_prot __sk_common.skc_prot
316322
#define sk_net __sk_common.skc_net
323+
#define sk_v6_daddr __sk_common.skc_v6_daddr
324+
#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
325+
317326
socket_lock_t sk_lock;
318327
struct sk_buff_head sk_receive_queue;
319328
/*

‎net/dccp/ipv6.c

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ static inline void dccp_v6_send_check(struct sock *sk, struct sk_buff *skb)
6767
struct dccp_hdr *dh = dccp_hdr(skb);
6868

6969
dccp_csum_outgoing(skb);
70-
dh->dccph_checksum = dccp_v6_csum_finish(skb, &np->saddr, &np->daddr);
70+
dh->dccph_checksum = dccp_v6_csum_finish(skb, &np->saddr, &sk->sk_v6_daddr);
7171
}
7272

7373
static inline __u64 dccp_v6_init_sequence(struct sk_buff *skb)
@@ -467,11 +467,11 @@ static struct sock *dccp_v6_request_recv_sock(struct sock *sk,
467467

468468
memcpy(newnp, np, sizeof(struct ipv6_pinfo));
469469

470-
ipv6_addr_set_v4mapped(newinet->inet_daddr, &newnp->daddr);
470+
ipv6_addr_set_v4mapped(newinet->inet_daddr, &newsk->sk_v6_daddr);
471471

472472
ipv6_addr_set_v4mapped(newinet->inet_saddr, &newnp->saddr);
473473

474-
newnp->rcv_saddr = newnp->saddr;
474+
newsk->sk_v6_rcv_saddr = newnp->saddr;
475475

476476
inet_csk(newsk)->icsk_af_ops = &dccp_ipv6_mapped;
477477
newsk->sk_backlog_rcv = dccp_v4_do_rcv;
@@ -538,9 +538,9 @@ static struct sock *dccp_v6_request_recv_sock(struct sock *sk,
538538

539539
memcpy(newnp, np, sizeof(struct ipv6_pinfo));
540540

541-
newnp->daddr = ireq6->rmt_addr;
541+
newsk->sk_v6_daddr = ireq6->rmt_addr;
542542
newnp->saddr = ireq6->loc_addr;
543-
newnp->rcv_saddr = ireq6->loc_addr;
543+
newsk->sk_v6_rcv_saddr = ireq6->loc_addr;
544544
newsk->sk_bound_dev_if = ireq6->iif;
545545

546546
/* Now IPv6 options...
@@ -885,7 +885,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
885885
return -EINVAL;
886886
}
887887

888-
np->daddr = usin->sin6_addr;
888+
sk->sk_v6_daddr = usin->sin6_addr;
889889
np->flow_label = fl6.flowlabel;
890890

891891
/*
@@ -915,16 +915,16 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
915915
goto failure;
916916
}
917917
ipv6_addr_set_v4mapped(inet->inet_saddr, &np->saddr);
918-
ipv6_addr_set_v4mapped(inet->inet_rcv_saddr, &np->rcv_saddr);
918+
ipv6_addr_set_v4mapped(inet->inet_rcv_saddr, &sk->sk_v6_rcv_saddr);
919919

920920
return err;
921921
}
922922

923-
if (!ipv6_addr_any(&np->rcv_saddr))
924-
saddr = &np->rcv_saddr;
923+
if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr))
924+
saddr = &sk->sk_v6_rcv_saddr;
925925

926926
fl6.flowi6_proto = IPPROTO_DCCP;
927-
fl6.daddr = np->daddr;
927+
fl6.daddr = sk->sk_v6_daddr;
928928
fl6.saddr = saddr ? *saddr : np->saddr;
929929
fl6.flowi6_oif = sk->sk_bound_dev_if;
930930
fl6.fl6_dport = usin->sin6_port;
@@ -941,7 +941,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
941941

942942
if (saddr == NULL) {
943943
saddr = &fl6.saddr;
944-
np->rcv_saddr = *saddr;
944+
sk->sk_v6_rcv_saddr = *saddr;
945945
}
946946

947947
/* set the source address */
@@ -963,7 +963,7 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
963963
goto late_failure;
964964

965965
dp->dccps_iss = secure_dccpv6_sequence_number(np->saddr.s6_addr32,
966-
np->daddr.s6_addr32,
966+
sk->sk_v6_daddr.s6_addr32,
967967
inet->inet_sport,
968968
inet->inet_dport);
969969
err = dccp_connect(sk);

‎net/dccp/ipv6.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ struct dccp6_request_sock {
3030

3131
struct dccp6_timewait_sock {
3232
struct inet_timewait_sock inet;
33-
struct inet6_timewait_sock tw6;
3433
};
3534

3635
#endif /* _DCCP_IPV6_H */

‎net/dccp/minisocks.c

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,9 @@ void dccp_time_wait(struct sock *sk, int state, int timeo)
5656
#if IS_ENABLED(CONFIG_IPV6)
5757
if (tw->tw_family == PF_INET6) {
5858
const struct ipv6_pinfo *np = inet6_sk(sk);
59-
struct inet6_timewait_sock *tw6;
6059

61-
tw->tw_ipv6_offset = inet6_tw_offset(sk->sk_prot);
62-
tw6 = inet6_twsk((struct sock *)tw);
63-
tw6->tw_v6_daddr = np->daddr;
64-
tw6->tw_v6_rcv_saddr = np->rcv_saddr;
60+
tw->tw_v6_daddr = sk->sk_v6_daddr;
61+
tw->tw_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
6562
tw->tw_ipv6only = np->ipv6only;
6663
}
6764
#endif

‎net/ipv4/inet_diag.c

Lines changed: 15 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -121,13 +121,13 @@ int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk,
121121

122122
#if IS_ENABLED(CONFIG_IPV6)
123123
if (r->idiag_family == AF_INET6) {
124-
const struct ipv6_pinfo *np = inet6_sk(sk);
125124

126-
*(struct in6_addr *)r->id.idiag_src = np->rcv_saddr;
127-
*(struct in6_addr *)r->id.idiag_dst = np->daddr;
125+
*(struct in6_addr *)r->id.idiag_src = sk->sk_v6_rcv_saddr;
126+
*(struct in6_addr *)r->id.idiag_dst = sk->sk_v6_daddr;
128127

129128
if (ext & (1 << (INET_DIAG_TCLASS - 1)))
130-
if (nla_put_u8(skb, INET_DIAG_TCLASS, np->tclass) < 0)
129+
if (nla_put_u8(skb, INET_DIAG_TCLASS,
130+
inet6_sk(sk)->tclass) < 0)
131131
goto errout;
132132
}
133133
#endif
@@ -255,11 +255,8 @@ static int inet_twsk_diag_fill(struct inet_timewait_sock *tw,
255255
r->idiag_inode = 0;
256256
#if IS_ENABLED(CONFIG_IPV6)
257257
if (tw->tw_family == AF_INET6) {
258-
const struct inet6_timewait_sock *tw6 =
259-
inet6_twsk((struct sock *)tw);
260-
261-
*(struct in6_addr *)r->id.idiag_src = tw6->tw_v6_rcv_saddr;
262-
*(struct in6_addr *)r->id.idiag_dst = tw6->tw_v6_daddr;
258+
*(struct in6_addr *)r->id.idiag_src = tw->tw_v6_rcv_saddr;
259+
*(struct in6_addr *)r->id.idiag_dst = tw->tw_v6_daddr;
263260
}
264261
#endif
265262

@@ -273,10 +270,11 @@ static int sk_diag_fill(struct sock *sk, struct sk_buff *skb,
273270
const struct nlmsghdr *unlh)
274271
{
275272
if (sk->sk_state == TCP_TIME_WAIT)
276-
return inet_twsk_diag_fill((struct inet_timewait_sock *)sk,
277-
skb, r, portid, seq, nlmsg_flags,
278-
unlh);
279-
return inet_csk_diag_fill(sk, skb, r, user_ns, portid, seq, nlmsg_flags, unlh);
273+
return inet_twsk_diag_fill(inet_twsk(sk), skb, r, portid, seq,
274+
nlmsg_flags, unlh);
275+
276+
return inet_csk_diag_fill(sk, skb, r, user_ns, portid, seq,
277+
nlmsg_flags, unlh);
280278
}
281279

282280
int inet_diag_dump_one_icsk(struct inet_hashinfo *hashinfo, struct sk_buff *in_skb,
@@ -489,10 +487,9 @@ int inet_diag_bc_sk(const struct nlattr *bc, struct sock *sk)
489487
entry.family = sk->sk_family;
490488
#if IS_ENABLED(CONFIG_IPV6)
491489
if (entry.family == AF_INET6) {
492-
struct ipv6_pinfo *np = inet6_sk(sk);
493490

494-
entry.saddr = np->rcv_saddr.s6_addr32;
495-
entry.daddr = np->daddr.s6_addr32;
491+
entry.saddr = sk->sk_v6_rcv_saddr.s6_addr32;
492+
entry.daddr = sk->sk_v6_daddr.s6_addr32;
496493
} else
497494
#endif
498495
{
@@ -649,10 +646,8 @@ static int inet_twsk_diag_dump(struct sock *sk,
649646
entry.family = tw->tw_family;
650647
#if IS_ENABLED(CONFIG_IPV6)
651648
if (tw->tw_family == AF_INET6) {
652-
struct inet6_timewait_sock *tw6 =
653-
inet6_twsk((struct sock *)tw);
654-
entry.saddr = tw6->tw_v6_rcv_saddr.s6_addr32;
655-
entry.daddr = tw6->tw_v6_daddr.s6_addr32;
649+
entry.saddr = tw->tw_v6_rcv_saddr.s6_addr32;
650+
entry.daddr = tw->tw_v6_daddr.s6_addr32;
656651
} else
657652
#endif
658653
{

‎net/ipv4/ping.c

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -202,15 +202,14 @@ static struct sock *ping_lookup(struct net *net, struct sk_buff *skb, u16 ident)
202202
#if IS_ENABLED(CONFIG_IPV6)
203203
} else if (skb->protocol == htons(ETH_P_IPV6) &&
204204
sk->sk_family == AF_INET6) {
205-
struct ipv6_pinfo *np = inet6_sk(sk);
206205

207206
pr_debug("found: %p: num=%d, daddr=%pI6c, dif=%d\n", sk,
208207
(int) isk->inet_num,
209-
&inet6_sk(sk)->rcv_saddr,
208+
&sk->sk_v6_rcv_saddr,
210209
sk->sk_bound_dev_if);
211210

212-
if (!ipv6_addr_any(&np->rcv_saddr) &&
213-
!ipv6_addr_equal(&np->rcv_saddr,
211+
if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr) &&
212+
!ipv6_addr_equal(&sk->sk_v6_rcv_saddr,
214213
&ipv6_hdr(skb)->daddr))
215214
continue;
216215
#endif
@@ -362,7 +361,7 @@ static void ping_set_saddr(struct sock *sk, struct sockaddr *saddr)
362361
} else if (saddr->sa_family == AF_INET6) {
363362
struct sockaddr_in6 *addr = (struct sockaddr_in6 *) saddr;
364363
struct ipv6_pinfo *np = inet6_sk(sk);
365-
np->rcv_saddr = np->saddr = addr->sin6_addr;
364+
sk->sk_v6_rcv_saddr = np->saddr = addr->sin6_addr;
366365
#endif
367366
}
368367
}
@@ -376,7 +375,7 @@ static void ping_clear_saddr(struct sock *sk, int dif)
376375
#if IS_ENABLED(CONFIG_IPV6)
377376
} else if (sk->sk_family == AF_INET6) {
378377
struct ipv6_pinfo *np = inet6_sk(sk);
379-
memset(&np->rcv_saddr, 0, sizeof(np->rcv_saddr));
378+
memset(&sk->sk_v6_rcv_saddr, 0, sizeof(sk->sk_v6_rcv_saddr));
380379
memset(&np->saddr, 0, sizeof(np->saddr));
381380
#endif
382381
}
@@ -418,7 +417,7 @@ int ping_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len)
418417
err = 0;
419418
if ((sk->sk_family == AF_INET && isk->inet_rcv_saddr) ||
420419
(sk->sk_family == AF_INET6 &&
421-
!ipv6_addr_any(&inet6_sk(sk)->rcv_saddr)))
420+
!ipv6_addr_any(&sk->sk_v6_rcv_saddr)))
422421
sk->sk_userlocks |= SOCK_BINDADDR_LOCK;
423422

424423
if (snum)
@@ -429,7 +428,7 @@ int ping_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len)
429428

430429
#if IS_ENABLED(CONFIG_IPV6)
431430
if (sk->sk_family == AF_INET6)
432-
memset(&inet6_sk(sk)->daddr, 0, sizeof(inet6_sk(sk)->daddr));
431+
memset(&sk->sk_v6_daddr, 0, sizeof(sk->sk_v6_daddr));
433432
#endif
434433

435434
sk_dst_reset(sk);

0 commit comments

Comments
 (0)