Skip to content

Commit d4546c2

Browse files
davem330davem330
committed
net: Convert GRO SKB handling to list_head.
Manage pending per-NAPI GRO packets via list_head. Return an SKB pointer from the GRO receive handlers. When GRO receive handlers return non-NULL, it means that this SKB needs to be completed at this time and removed from the NAPI queue. Several operations are greatly simplified by this transformation, especially timing out the oldest SKB in the list when gro_count exceeds MAX_GRO_SKBS, and napi_gro_flush() which walks the queue in reverse order. Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 9ff3b40 commit d4546c2

24 files changed

+133
-141
lines changed

‎drivers/net/geneve.c

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -418,11 +418,12 @@ static int geneve_hlen(struct genevehdr *gh)
418418
return sizeof(*gh) + gh->opt_len * 4;
419419
}
420420

421-
static struct sk_buff **geneve_gro_receive(struct sock *sk,
422-
struct sk_buff **head,
423-
struct sk_buff *skb)
421+
static struct sk_buff *geneve_gro_receive(struct sock *sk,
422+
struct list_head *head,
423+
struct sk_buff *skb)
424424
{
425-
struct sk_buff *p, **pp = NULL;
425+
struct sk_buff *pp = NULL;
426+
struct sk_buff *p;
426427
struct genevehdr *gh, *gh2;
427428
unsigned int hlen, gh_len, off_gnv;
428429
const struct packet_offload *ptype;
@@ -449,7 +450,7 @@ static struct sk_buff **geneve_gro_receive(struct sock *sk,
449450
goto out;
450451
}
451452

452-
for (p = *head; p; p = p->next) {
453+
list_for_each_entry(p, head, list) {
453454
if (!NAPI_GRO_CB(p)->same_flow)
454455
continue;
455456

‎drivers/net/vxlan.c

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -568,11 +568,12 @@ static struct vxlanhdr *vxlan_gro_remcsum(struct sk_buff *skb,
568568
return vh;
569569
}
570570

571-
static struct sk_buff **vxlan_gro_receive(struct sock *sk,
572-
struct sk_buff **head,
573-
struct sk_buff *skb)
571+
static struct sk_buff *vxlan_gro_receive(struct sock *sk,
572+
struct list_head *head,
573+
struct sk_buff *skb)
574574
{
575-
struct sk_buff *p, **pp = NULL;
575+
struct sk_buff *pp = NULL;
576+
struct sk_buff *p;
576577
struct vxlanhdr *vh, *vh2;
577578
unsigned int hlen, off_vx;
578579
int flush = 1;
@@ -607,7 +608,7 @@ static struct sk_buff **vxlan_gro_receive(struct sock *sk,
607608

608609
skb_gro_pull(skb, sizeof(struct vxlanhdr)); /* pull vxlan header */
609610

610-
for (p = *head; p; p = p->next) {
611+
list_for_each_entry(p, head, list) {
611612
if (!NAPI_GRO_CB(p)->same_flow)
612613
continue;
613614

‎include/linux/etherdevice.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,7 @@ struct net_device *devm_alloc_etherdev_mqs(struct device *dev, int sizeof_priv,
5959
unsigned int rxqs);
6060
#define devm_alloc_etherdev(dev, sizeof_priv) devm_alloc_etherdev_mqs(dev, sizeof_priv, 1, 1)
6161

62-
struct sk_buff **eth_gro_receive(struct sk_buff **head,
63-
struct sk_buff *skb);
62+
struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb);
6463
int eth_gro_complete(struct sk_buff *skb, int nhoff);
6564

6665
/* Reserved Ethernet Addresses per IEEE 802.1Q */

‎include/linux/netdevice.h

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,7 @@ struct napi_struct {
322322
int poll_owner;
323323
#endif
324324
struct net_device *dev;
325-
struct sk_buff *gro_list;
325+
struct list_head gro_list;
326326
struct sk_buff *skb;
327327
struct hrtimer timer;
328328
struct list_head dev_list;
@@ -2255,10 +2255,10 @@ static inline int gro_recursion_inc_test(struct sk_buff *skb)
22552255
return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
22562256
}
22572257

2258-
typedef struct sk_buff **(*gro_receive_t)(struct sk_buff **, struct sk_buff *);
2259-
static inline struct sk_buff **call_gro_receive(gro_receive_t cb,
2260-
struct sk_buff **head,
2261-
struct sk_buff *skb)
2258+
typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);
2259+
static inline struct sk_buff *call_gro_receive(gro_receive_t cb,
2260+
struct list_head *head,
2261+
struct sk_buff *skb)
22622262
{
22632263
if (unlikely(gro_recursion_inc_test(skb))) {
22642264
NAPI_GRO_CB(skb)->flush |= 1;
@@ -2268,12 +2268,12 @@ static inline struct sk_buff **call_gro_receive(gro_receive_t cb,
22682268
return cb(head, skb);
22692269
}
22702270

2271-
typedef struct sk_buff **(*gro_receive_sk_t)(struct sock *, struct sk_buff **,
2272-
struct sk_buff *);
2273-
static inline struct sk_buff **call_gro_receive_sk(gro_receive_sk_t cb,
2274-
struct sock *sk,
2275-
struct sk_buff **head,
2276-
struct sk_buff *skb)
2271+
typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *,
2272+
struct sk_buff *);
2273+
static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb,
2274+
struct sock *sk,
2275+
struct list_head *head,
2276+
struct sk_buff *skb)
22772277
{
22782278
if (unlikely(gro_recursion_inc_test(skb))) {
22792279
NAPI_GRO_CB(skb)->flush |= 1;
@@ -2299,8 +2299,8 @@ struct packet_type {
22992299
struct offload_callbacks {
23002300
struct sk_buff *(*gso_segment)(struct sk_buff *skb,
23012301
netdev_features_t features);
2302-
struct sk_buff **(*gro_receive)(struct sk_buff **head,
2303-
struct sk_buff *skb);
2302+
struct sk_buff *(*gro_receive)(struct list_head *head,
2303+
struct sk_buff *skb);
23042304
int (*gro_complete)(struct sk_buff *skb, int nhoff);
23052305
};
23062306

@@ -2568,7 +2568,7 @@ struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
25682568
struct net_device *dev_get_by_napi_id(unsigned int napi_id);
25692569
int netdev_get_name(struct net *net, char *name, int ifindex);
25702570
int dev_restart(struct net_device *dev);
2571-
int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb);
2571+
int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
25722572

25732573
static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
25742574
{
@@ -2784,13 +2784,13 @@ static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
27842784
}
27852785

27862786
#ifdef CONFIG_XFRM_OFFLOAD
2787-
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
2787+
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
27882788
{
27892789
if (PTR_ERR(pp) != -EINPROGRESS)
27902790
NAPI_GRO_CB(skb)->flush |= flush;
27912791
}
27922792
#else
2793-
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
2793+
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
27942794
{
27952795
NAPI_GRO_CB(skb)->flush |= flush;
27962796
}

‎include/linux/skbuff.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,8 @@ struct sk_buff {
677677
int ip_defrag_offset;
678678
};
679679
};
680-
struct rb_node rbnode; /* used in netem & tcp stack */
680+
struct rb_node rbnode; /* used in netem & tcp stack */
681+
struct list_head list;
681682
};
682683
struct sock *sk;
683684

‎include/linux/udp.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ struct udp_sock {
7474
void (*encap_destroy)(struct sock *sk);
7575

7676
/* GRO functions for UDP socket */
77-
struct sk_buff ** (*gro_receive)(struct sock *sk,
78-
struct sk_buff **head,
77+
struct sk_buff * (*gro_receive)(struct sock *sk,
78+
struct list_head *head,
7979
struct sk_buff *skb);
8080
int (*gro_complete)(struct sock *sk,
8181
struct sk_buff *skb,

‎include/net/inet_common.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ int inet_ctl_sock_create(struct sock **sk, unsigned short family,
4343
int inet_recv_error(struct sock *sk, struct msghdr *msg, int len,
4444
int *addr_len);
4545

46-
struct sk_buff **inet_gro_receive(struct sk_buff **head, struct sk_buff *skb);
46+
struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb);
4747
int inet_gro_complete(struct sk_buff *skb, int nhoff);
4848
struct sk_buff *inet_gso_segment(struct sk_buff *skb,
4949
netdev_features_t features);

‎include/net/tcp.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1788,7 +1788,7 @@ void tcp_v4_destroy_sock(struct sock *sk);
17881788

17891789
struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
17901790
netdev_features_t features);
1791-
struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb);
1791+
struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb);
17921792
int tcp_gro_complete(struct sk_buff *skb);
17931793

17941794
void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);

‎include/net/udp.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,8 @@ static inline void udp_csum_pull_header(struct sk_buff *skb)
170170
typedef struct sock *(*udp_lookup_t)(struct sk_buff *skb, __be16 sport,
171171
__be16 dport);
172172

173-
struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
174-
struct udphdr *uh, udp_lookup_t lookup);
173+
struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
174+
struct udphdr *uh, udp_lookup_t lookup);
175175
int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
176176

177177
struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,

‎include/net/udp_tunnel.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,9 @@ static inline int udp_sock_create(struct net *net,
6565

6666
typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb);
6767
typedef void (*udp_tunnel_encap_destroy_t)(struct sock *sk);
68-
typedef struct sk_buff **(*udp_tunnel_gro_receive_t)(struct sock *sk,
69-
struct sk_buff **head,
70-
struct sk_buff *skb);
68+
typedef struct sk_buff *(*udp_tunnel_gro_receive_t)(struct sock *sk,
69+
struct list_head *head,
70+
struct sk_buff *skb);
7171
typedef int (*udp_tunnel_gro_complete_t)(struct sock *sk, struct sk_buff *skb,
7272
int nhoff);
7373

‎net/8021q/vlan.c

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -647,13 +647,14 @@ static int vlan_ioctl_handler(struct net *net, void __user *arg)
647647
return err;
648648
}
649649

650-
static struct sk_buff **vlan_gro_receive(struct sk_buff **head,
651-
struct sk_buff *skb)
650+
static struct sk_buff *vlan_gro_receive(struct list_head *head,
651+
struct sk_buff *skb)
652652
{
653-
struct sk_buff *p, **pp = NULL;
654-
struct vlan_hdr *vhdr;
655-
unsigned int hlen, off_vlan;
656653
const struct packet_offload *ptype;
654+
unsigned int hlen, off_vlan;
655+
struct sk_buff *pp = NULL;
656+
struct vlan_hdr *vhdr;
657+
struct sk_buff *p;
657658
__be16 type;
658659
int flush = 1;
659660

@@ -675,7 +676,7 @@ static struct sk_buff **vlan_gro_receive(struct sk_buff **head,
675676

676677
flush = 0;
677678

678-
for (p = *head; p; p = p->next) {
679+
list_for_each_entry(p, head, list) {
679680
struct vlan_hdr *vhdr2;
680681

681682
if (!NAPI_GRO_CB(p)->same_flow)

‎net/core/dev.c

Lines changed: 28 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -4881,36 +4881,25 @@ static int napi_gro_complete(struct sk_buff *skb)
48814881
*/
48824882
void napi_gro_flush(struct napi_struct *napi, bool flush_old)
48834883
{
4884-
struct sk_buff *skb, *prev = NULL;
4885-
4886-
/* scan list and build reverse chain */
4887-
for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4888-
skb->prev = prev;
4889-
prev = skb;
4890-
}
4891-
4892-
for (skb = prev; skb; skb = prev) {
4893-
skb->next = NULL;
4884+
struct sk_buff *skb, *p;
48944885

4886+
list_for_each_entry_safe_reverse(skb, p, &napi->gro_list, list) {
48954887
if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
48964888
return;
4897-
4898-
prev = skb->prev;
4889+
list_del_init(&skb->list);
48994890
napi_gro_complete(skb);
49004891
napi->gro_count--;
49014892
}
4902-
4903-
napi->gro_list = NULL;
49044893
}
49054894
EXPORT_SYMBOL(napi_gro_flush);
49064895

49074896
static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
49084897
{
4909-
struct sk_buff *p;
49104898
unsigned int maclen = skb->dev->hard_header_len;
49114899
u32 hash = skb_get_hash_raw(skb);
4900+
struct sk_buff *p;
49124901

4913-
for (p = napi->gro_list; p; p = p->next) {
4902+
list_for_each_entry(p, &napi->gro_list, list) {
49144903
unsigned long diffs;
49154904

49164905
NAPI_GRO_CB(p)->flush = 0;
@@ -4977,12 +4966,12 @@ static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
49774966

49784967
static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
49794968
{
4980-
struct sk_buff **pp = NULL;
4969+
struct list_head *head = &offload_base;
49814970
struct packet_offload *ptype;
49824971
__be16 type = skb->protocol;
4983-
struct list_head *head = &offload_base;
4984-
int same_flow;
4972+
struct sk_buff *pp = NULL;
49854973
enum gro_result ret;
4974+
int same_flow;
49864975
int grow;
49874976

49884977
if (netif_elide_gro(skb->dev))
@@ -5039,11 +5028,8 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
50395028
ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
50405029

50415030
if (pp) {
5042-
struct sk_buff *nskb = *pp;
5043-
5044-
*pp = nskb->next;
5045-
nskb->next = NULL;
5046-
napi_gro_complete(nskb);
5031+
list_del_init(&pp->list);
5032+
napi_gro_complete(pp);
50475033
napi->gro_count--;
50485034
}
50495035

@@ -5054,15 +5040,10 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
50545040
goto normal;
50555041

50565042
if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
5057-
struct sk_buff *nskb = napi->gro_list;
5043+
struct sk_buff *nskb;
50585044

5059-
/* locate the end of the list to select the 'oldest' flow */
5060-
while (nskb->next) {
5061-
pp = &nskb->next;
5062-
nskb = *pp;
5063-
}
5064-
*pp = NULL;
5065-
nskb->next = NULL;
5045+
nskb = list_last_entry(&napi->gro_list, struct sk_buff, list);
5046+
list_del(&nskb->list);
50665047
napi_gro_complete(nskb);
50675048
} else {
50685049
napi->gro_count++;
@@ -5071,8 +5052,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
50715052
NAPI_GRO_CB(skb)->age = jiffies;
50725053
NAPI_GRO_CB(skb)->last = skb;
50735054
skb_shinfo(skb)->gso_size = skb_gro_len(skb);
5074-
skb->next = napi->gro_list;
5075-
napi->gro_list = skb;
5055+
list_add(&skb->list, &napi->gro_list);
50765056
ret = GRO_HELD;
50775057

50785058
pull:
@@ -5478,7 +5458,7 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
54785458
NAPIF_STATE_IN_BUSY_POLL)))
54795459
return false;
54805460

5481-
if (n->gro_list) {
5461+
if (!list_empty(&n->gro_list)) {
54825462
unsigned long timeout = 0;
54835463

54845464
if (work_done)
@@ -5687,7 +5667,7 @@ static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
56875667
/* Note : we use a relaxed variant of napi_schedule_prep() not setting
56885668
* NAPI_STATE_MISSED, since we do not react to a device IRQ.
56895669
*/
5690-
if (napi->gro_list && !napi_disable_pending(napi) &&
5670+
if (!list_empty(&napi->gro_list) && !napi_disable_pending(napi) &&
56915671
!test_and_set_bit(NAPI_STATE_SCHED, &napi->state))
56925672
__napi_schedule_irqoff(napi);
56935673

@@ -5701,7 +5681,7 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
57015681
hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
57025682
napi->timer.function = napi_watchdog;
57035683
napi->gro_count = 0;
5704-
napi->gro_list = NULL;
5684+
INIT_LIST_HEAD(&napi->gro_list);
57055685
napi->skb = NULL;
57065686
napi->poll = poll;
57075687
if (weight > NAPI_POLL_WEIGHT)
@@ -5734,6 +5714,14 @@ void napi_disable(struct napi_struct *n)
57345714
}
57355715
EXPORT_SYMBOL(napi_disable);
57365716

5717+
static void gro_list_free(struct list_head *head)
5718+
{
5719+
struct sk_buff *skb, *p;
5720+
5721+
list_for_each_entry_safe(skb, p, head, list)
5722+
kfree_skb(skb);
5723+
}
5724+
57375725
/* Must be called in process context */
57385726
void netif_napi_del(struct napi_struct *napi)
57395727
{
@@ -5743,8 +5731,8 @@ void netif_napi_del(struct napi_struct *napi)
57435731
list_del_init(&napi->dev_list);
57445732
napi_free_frags(napi);
57455733

5746-
kfree_skb_list(napi->gro_list);
5747-
napi->gro_list = NULL;
5734+
gro_list_free(&napi->gro_list);
5735+
INIT_LIST_HEAD(&napi->gro_list);
57485736
napi->gro_count = 0;
57495737
}
57505738
EXPORT_SYMBOL(netif_napi_del);
@@ -5787,7 +5775,7 @@ static int napi_poll(struct napi_struct *n, struct list_head *repoll)
57875775
goto out_unlock;
57885776
}
57895777

5790-
if (n->gro_list) {
5778+
if (!list_empty(&n->gro_list)) {
57915779
/* flush too old packets
57925780
* If HZ < 1000, flush all packets.
57935781
*/

0 commit comments

Comments
 (0)