Skip to content

Commit 029ac21

Browse files
davem330davem330
committed
Merge branch 'net-sched-singly-linked-list'
Florian Westphal says: ==================== sched: convert queues to single-linked list During Netfilter Workshop 2016 Eric Dumazet pointed out that qdisc schedulers use doubly-linked lists, even though single-linked list would be enough. The double-linked skb lists incur one extra write on enqueue/dequeue operations (to change ->prev pointer of next list elem). This series converts qdiscs to single-linked version, listhead maintains pointers to first (for dequeue) and last skb (for enqueue). Most qdiscs don't queue at all and instead use a leaf qdisc (typically pfifo_fast) so only a few schedulers needed changes. I briefly tested netem and htb and they seemed fine. UDP_STREAM netperf with 64 byte packets via veth+pfifo_fast shows a small (~2%) improvement. ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
2 parents 106323b + 48da34b commit 029ac21

File tree

7 files changed

+114
-42
lines changed

7 files changed

+114
-42
lines changed

‎include/net/sch_generic.h

Lines changed: 56 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,14 @@ struct qdisc_size_table {
3636
u16 data[];
3737
};
3838

39+
/* similar to sk_buff_head, but skb->prev pointer is undefined. */
40+
struct qdisc_skb_head {
41+
struct sk_buff *head;
42+
struct sk_buff *tail;
43+
__u32 qlen;
44+
spinlock_t lock;
45+
};
46+
3947
struct Qdisc {
4048
int (*enqueue)(struct sk_buff *skb,
4149
struct Qdisc *sch,
@@ -76,7 +84,7 @@ struct Qdisc {
7684
* For performance sake on SMP, we put highly modified fields at the end
7785
*/
7886
struct sk_buff *gso_skb ____cacheline_aligned_in_smp;
79-
struct sk_buff_head q;
87+
struct qdisc_skb_head q;
8088
struct gnet_stats_basic_packed bstats;
8189
seqcount_t running;
8290
struct gnet_stats_queue qstats;
@@ -600,10 +608,27 @@ static inline void qdisc_qstats_overlimit(struct Qdisc *sch)
600608
sch->qstats.overlimits++;
601609
}
602610

611+
static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh)
612+
{
613+
qh->head = NULL;
614+
qh->tail = NULL;
615+
qh->qlen = 0;
616+
}
617+
603618
static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch,
604-
struct sk_buff_head *list)
619+
struct qdisc_skb_head *qh)
605620
{
606-
__skb_queue_tail(list, skb);
621+
struct sk_buff *last = qh->tail;
622+
623+
if (last) {
624+
skb->next = NULL;
625+
last->next = skb;
626+
qh->tail = skb;
627+
} else {
628+
qh->tail = skb;
629+
qh->head = skb;
630+
}
631+
qh->qlen++;
607632
qdisc_qstats_backlog_inc(sch, skb);
608633

609634
return NET_XMIT_SUCCESS;
@@ -614,22 +639,31 @@ static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch)
614639
return __qdisc_enqueue_tail(skb, sch, &sch->q);
615640
}
616641

617-
static inline struct sk_buff *__qdisc_dequeue_head(struct Qdisc *sch,
618-
struct sk_buff_head *list)
642+
static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh)
619643
{
620-
struct sk_buff *skb = __skb_dequeue(list);
644+
struct sk_buff *skb = qh->head;
621645

622646
if (likely(skb != NULL)) {
623-
qdisc_qstats_backlog_dec(sch, skb);
624-
qdisc_bstats_update(sch, skb);
647+
qh->head = skb->next;
648+
qh->qlen--;
649+
if (qh->head == NULL)
650+
qh->tail = NULL;
651+
skb->next = NULL;
625652
}
626653

627654
return skb;
628655
}
629656

630657
static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch)
631658
{
632-
return __qdisc_dequeue_head(sch, &sch->q);
659+
struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);
660+
661+
if (likely(skb != NULL)) {
662+
qdisc_qstats_backlog_dec(sch, skb);
663+
qdisc_bstats_update(sch, skb);
664+
}
665+
666+
return skb;
633667
}
634668

635669
/* Instead of calling kfree_skb() while root qdisc lock is held,
@@ -642,10 +676,10 @@ static inline void __qdisc_drop(struct sk_buff *skb, struct sk_buff **to_free)
642676
}
643677

644678
static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch,
645-
struct sk_buff_head *list,
679+
struct qdisc_skb_head *qh,
646680
struct sk_buff **to_free)
647681
{
648-
struct sk_buff *skb = __skb_dequeue(list);
682+
struct sk_buff *skb = __qdisc_dequeue_head(qh);
649683

650684
if (likely(skb != NULL)) {
651685
unsigned int len = qdisc_pkt_len(skb);
@@ -666,7 +700,9 @@ static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch,
666700

667701
static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch)
668702
{
669-
return skb_peek(&sch->q);
703+
const struct qdisc_skb_head *qh = &sch->q;
704+
705+
return qh->head;
670706
}
671707

672708
/* generic pseudo peek method for non-work-conserving qdisc */
@@ -701,15 +737,19 @@ static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch)
701737
return skb;
702738
}
703739

704-
static inline void __qdisc_reset_queue(struct sk_buff_head *list)
740+
static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh)
705741
{
706742
/*
707743
* We do not know the backlog in bytes of this list, it
708744
* is up to the caller to correct it
709745
*/
710-
if (!skb_queue_empty(list)) {
711-
rtnl_kfree_skbs(list->next, list->prev);
712-
__skb_queue_head_init(list);
746+
ASSERT_RTNL();
747+
if (qh->qlen) {
748+
rtnl_kfree_skbs(qh->head, qh->tail);
749+
750+
qh->head = NULL;
751+
qh->tail = NULL;
752+
qh->qlen = 0;
713753
}
714754
}
715755

‎net/sched/sch_codel.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ struct codel_sched_data {
6969
static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)
7070
{
7171
struct Qdisc *sch = ctx;
72-
struct sk_buff *skb = __skb_dequeue(&sch->q);
72+
struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);
7373

7474
if (skb)
7575
sch->qstats.backlog -= qdisc_pkt_len(skb);
@@ -172,7 +172,7 @@ static int codel_change(struct Qdisc *sch, struct nlattr *opt)
172172

173173
qlen = sch->q.qlen;
174174
while (sch->q.qlen > sch->limit) {
175-
struct sk_buff *skb = __skb_dequeue(&sch->q);
175+
struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);
176176

177177
dropped += qdisc_pkt_len(skb);
178178
qdisc_qstats_backlog_dec(sch, skb);

‎net/sched/sch_fifo.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ static int bfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch,
3131
static int pfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch,
3232
struct sk_buff **to_free)
3333
{
34-
if (likely(skb_queue_len(&sch->q) < sch->limit))
34+
if (likely(sch->q.qlen < sch->limit))
3535
return qdisc_enqueue_tail(skb, sch);
3636

3737
return qdisc_drop(skb, sch, to_free);
@@ -42,7 +42,7 @@ static int pfifo_tail_enqueue(struct sk_buff *skb, struct Qdisc *sch,
4242
{
4343
unsigned int prev_backlog;
4444

45-
if (likely(skb_queue_len(&sch->q) < sch->limit))
45+
if (likely(sch->q.qlen < sch->limit))
4646
return qdisc_enqueue_tail(skb, sch);
4747

4848
prev_backlog = sch->qstats.backlog;

‎net/sched/sch_generic.c

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -466,7 +466,7 @@ static const u8 prio2band[TC_PRIO_MAX + 1] = {
466466
*/
467467
struct pfifo_fast_priv {
468468
u32 bitmap;
469-
struct sk_buff_head q[PFIFO_FAST_BANDS];
469+
struct qdisc_skb_head q[PFIFO_FAST_BANDS];
470470
};
471471

472472
/*
@@ -477,7 +477,7 @@ struct pfifo_fast_priv {
477477
*/
478478
static const int bitmap2band[] = {-1, 0, 1, 0, 2, 0, 1, 0};
479479

480-
static inline struct sk_buff_head *band2list(struct pfifo_fast_priv *priv,
480+
static inline struct qdisc_skb_head *band2list(struct pfifo_fast_priv *priv,
481481
int band)
482482
{
483483
return priv->q + band;
@@ -486,10 +486,10 @@ static inline struct sk_buff_head *band2list(struct pfifo_fast_priv *priv,
486486
static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
487487
struct sk_buff **to_free)
488488
{
489-
if (skb_queue_len(&qdisc->q) < qdisc_dev(qdisc)->tx_queue_len) {
489+
if (qdisc->q.qlen < qdisc_dev(qdisc)->tx_queue_len) {
490490
int band = prio2band[skb->priority & TC_PRIO_MAX];
491491
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
492-
struct sk_buff_head *list = band2list(priv, band);
492+
struct qdisc_skb_head *list = band2list(priv, band);
493493

494494
priv->bitmap |= (1 << band);
495495
qdisc->q.qlen++;
@@ -505,11 +505,16 @@ static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
505505
int band = bitmap2band[priv->bitmap];
506506

507507
if (likely(band >= 0)) {
508-
struct sk_buff_head *list = band2list(priv, band);
509-
struct sk_buff *skb = __qdisc_dequeue_head(qdisc, list);
508+
struct qdisc_skb_head *qh = band2list(priv, band);
509+
struct sk_buff *skb = __qdisc_dequeue_head(qh);
510+
511+
if (likely(skb != NULL)) {
512+
qdisc_qstats_backlog_dec(qdisc, skb);
513+
qdisc_bstats_update(qdisc, skb);
514+
}
510515

511516
qdisc->q.qlen--;
512-
if (skb_queue_empty(list))
517+
if (qh->qlen == 0)
513518
priv->bitmap &= ~(1 << band);
514519

515520
return skb;
@@ -524,9 +529,9 @@ static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
524529
int band = bitmap2band[priv->bitmap];
525530

526531
if (band >= 0) {
527-
struct sk_buff_head *list = band2list(priv, band);
532+
struct qdisc_skb_head *qh = band2list(priv, band);
528533

529-
return skb_peek(list);
534+
return qh->head;
530535
}
531536

532537
return NULL;
@@ -564,7 +569,7 @@ static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt)
564569
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
565570

566571
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
567-
__skb_queue_head_init(band2list(priv, prio));
572+
qdisc_skb_head_init(band2list(priv, prio));
568573

569574
/* Can by-pass the queue discipline */
570575
qdisc->flags |= TCQ_F_CAN_BYPASS;
@@ -612,7 +617,8 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
612617
sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
613618
sch->padded = (char *) sch - (char *) p;
614619
}
615-
skb_queue_head_init(&sch->q);
620+
qdisc_skb_head_init(&sch->q);
621+
spin_lock_init(&sch->q.lock);
616622

617623
spin_lock_init(&sch->busylock);
618624
lockdep_set_class(&sch->busylock,

‎net/sched/sch_htb.c

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ struct htb_sched {
162162
struct work_struct work;
163163

164164
/* non shaped skbs; let them go directly thru */
165-
struct sk_buff_head direct_queue;
165+
struct qdisc_skb_head direct_queue;
166166
long direct_pkts;
167167

168168
struct qdisc_watchdog watchdog;
@@ -570,6 +570,22 @@ static inline void htb_deactivate(struct htb_sched *q, struct htb_class *cl)
570570
list_del_init(&cl->un.leaf.drop_list);
571571
}
572572

573+
static void htb_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch,
574+
struct qdisc_skb_head *qh)
575+
{
576+
struct sk_buff *last = qh->tail;
577+
578+
if (last) {
579+
skb->next = NULL;
580+
last->next = skb;
581+
qh->tail = skb;
582+
} else {
583+
qh->tail = skb;
584+
qh->head = skb;
585+
}
586+
qh->qlen++;
587+
}
588+
573589
static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
574590
struct sk_buff **to_free)
575591
{
@@ -580,7 +596,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
580596
if (cl == HTB_DIRECT) {
581597
/* enqueue to helper queue */
582598
if (q->direct_queue.qlen < q->direct_qlen) {
583-
__skb_queue_tail(&q->direct_queue, skb);
599+
htb_enqueue_tail(skb, sch, &q->direct_queue);
584600
q->direct_pkts++;
585601
} else {
586602
return qdisc_drop(skb, sch, to_free);
@@ -888,7 +904,7 @@ static struct sk_buff *htb_dequeue(struct Qdisc *sch)
888904
unsigned long start_at;
889905

890906
/* try to dequeue direct packets as high prio (!) to minimize cpu work */
891-
skb = __skb_dequeue(&q->direct_queue);
907+
skb = __qdisc_dequeue_head(&q->direct_queue);
892908
if (skb != NULL) {
893909
ok:
894910
qdisc_bstats_update(sch, skb);
@@ -1019,7 +1035,7 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt)
10191035

10201036
qdisc_watchdog_init(&q->watchdog, sch);
10211037
INIT_WORK(&q->work, htb_work_func);
1022-
__skb_queue_head_init(&q->direct_queue);
1038+
qdisc_skb_head_init(&q->direct_queue);
10231039

10241040
if (tb[TCA_HTB_DIRECT_QLEN])
10251041
q->direct_qlen = nla_get_u32(tb[TCA_HTB_DIRECT_QLEN]);

‎net/sched/sch_netem.c

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,16 @@ static struct sk_buff *netem_segment(struct sk_buff *skb, struct Qdisc *sch,
413413
return segs;
414414
}
415415

416+
static void netem_enqueue_skb_head(struct qdisc_skb_head *qh, struct sk_buff *skb)
417+
{
418+
skb->next = qh->head;
419+
420+
if (!qh->head)
421+
qh->tail = skb;
422+
qh->head = skb;
423+
qh->qlen++;
424+
}
425+
416426
/*
417427
* Insert one skb into qdisc.
418428
* Note: parent depends on return value to account for queue length.
@@ -502,7 +512,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
502512
1<<(prandom_u32() % 8);
503513
}
504514

505-
if (unlikely(skb_queue_len(&sch->q) >= sch->limit))
515+
if (unlikely(sch->q.qlen >= sch->limit))
506516
return qdisc_drop(skb, sch, to_free);
507517

508518
qdisc_qstats_backlog_inc(sch, skb);
@@ -522,8 +532,8 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
522532
if (q->rate) {
523533
struct sk_buff *last;
524534

525-
if (!skb_queue_empty(&sch->q))
526-
last = skb_peek_tail(&sch->q);
535+
if (sch->q.qlen)
536+
last = sch->q.tail;
527537
else
528538
last = netem_rb_to_skb(rb_last(&q->t_root));
529539
if (last) {
@@ -552,7 +562,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
552562
cb->time_to_send = psched_get_time();
553563
q->counter = 0;
554564

555-
__skb_queue_head(&sch->q, skb);
565+
netem_enqueue_skb_head(&sch->q, skb);
556566
sch->qstats.requeues++;
557567
}
558568

@@ -587,7 +597,7 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch)
587597
struct rb_node *p;
588598

589599
tfifo_dequeue:
590-
skb = __skb_dequeue(&sch->q);
600+
skb = __qdisc_dequeue_head(&sch->q);
591601
if (skb) {
592602
qdisc_qstats_backlog_dec(sch, skb);
593603
deliver:

‎net/sched/sch_pie.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ static int pie_change(struct Qdisc *sch, struct nlattr *opt)
231231
/* Drop excess packets if new limit is lower */
232232
qlen = sch->q.qlen;
233233
while (sch->q.qlen > sch->limit) {
234-
struct sk_buff *skb = __skb_dequeue(&sch->q);
234+
struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);
235235

236236
dropped += qdisc_pkt_len(skb);
237237
qdisc_qstats_backlog_dec(sch, skb);
@@ -511,7 +511,7 @@ static int pie_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
511511
static struct sk_buff *pie_qdisc_dequeue(struct Qdisc *sch)
512512
{
513513
struct sk_buff *skb;
514-
skb = __qdisc_dequeue_head(sch, &sch->q);
514+
skb = qdisc_dequeue_head(sch);
515515

516516
if (!skb)
517517
return NULL;

0 commit comments

Comments
 (0)