Skip to content

Commit b0ab6f9

Browse files
jrfastabdavem330
jrfastab
authored and
davem330
committed
net: sched: enable per cpu qstats
After previous patches to simplify qstats the qstats can be made per cpu with a packed union in Qdisc struct. Signed-off-by: John Fastabend <john.r.fastabend@intel.com> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 6401585 commit b0ab6f9

17 files changed

+87
-25
lines changed

‎include/net/gen_stats.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ int gnet_stats_copy_rate_est(struct gnet_dump *d,
4141
const struct gnet_stats_basic_packed *b,
4242
struct gnet_stats_rate_est64 *r);
4343
int gnet_stats_copy_queue(struct gnet_dump *d,
44-
struct gnet_stats_queue *q, __u32 len);
44+
struct gnet_stats_queue __percpu *cpu_q,
45+
struct gnet_stats_queue *q, __u32 qlen);
4546
int gnet_stats_copy_app(struct gnet_dump *d, void *st, int len);
4647

4748
int gnet_stats_finish_copy(struct gnet_dump *d);

‎include/net/sch_generic.h

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,10 @@ struct Qdisc {
9090
struct gnet_stats_basic_cpu __percpu *cpu_bstats;
9191
} __packed;
9292
unsigned int __state;
93-
struct gnet_stats_queue qstats;
93+
union {
94+
struct gnet_stats_queue qstats;
95+
struct gnet_stats_queue __percpu *cpu_qstats;
96+
} __packed;
9497
struct rcu_head rcu_head;
9598
int padded;
9699
atomic_t refcnt;
@@ -543,6 +546,13 @@ static inline void qdisc_qstats_drop(struct Qdisc *sch)
543546
sch->qstats.drops++;
544547
}
545548

549+
static inline void qdisc_qstats_drop_cpu(struct Qdisc *sch)
550+
{
551+
struct gnet_stats_queue *qstats = this_cpu_ptr(sch->cpu_qstats);
552+
553+
qstats->drops++;
554+
}
555+
546556
static inline void qdisc_qstats_overlimit(struct Qdisc *sch)
547557
{
548558
sch->qstats.overlimits++;

‎net/core/gen_stats.c

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -215,33 +215,74 @@ gnet_stats_copy_rate_est(struct gnet_dump *d,
215215
}
216216
EXPORT_SYMBOL(gnet_stats_copy_rate_est);
217217

218+
static void
219+
__gnet_stats_copy_queue_cpu(struct gnet_stats_queue *qstats,
220+
const struct gnet_stats_queue __percpu *q)
221+
{
222+
int i;
223+
224+
for_each_possible_cpu(i) {
225+
const struct gnet_stats_queue *qcpu = per_cpu_ptr(q, i);
226+
227+
qstats->qlen = 0;
228+
qstats->backlog += qcpu->backlog;
229+
qstats->drops += qcpu->drops;
230+
qstats->requeues += qcpu->requeues;
231+
qstats->overlimits += qcpu->overlimits;
232+
}
233+
}
234+
235+
static void __gnet_stats_copy_queue(struct gnet_stats_queue *qstats,
236+
const struct gnet_stats_queue __percpu *cpu,
237+
const struct gnet_stats_queue *q,
238+
__u32 qlen)
239+
{
240+
if (cpu) {
241+
__gnet_stats_copy_queue_cpu(qstats, cpu);
242+
} else {
243+
qstats->qlen = q->qlen;
244+
qstats->backlog = q->backlog;
245+
qstats->drops = q->drops;
246+
qstats->requeues = q->requeues;
247+
qstats->overlimits = q->overlimits;
248+
}
249+
250+
qstats->qlen = qlen;
251+
}
252+
218253
/**
219254
* gnet_stats_copy_queue - copy queue statistics into statistics TLV
220255
* @d: dumping handle
256+
* @cpu_q: per cpu queue statistics
221257
* @q: queue statistics
222258
* @qlen: queue length statistics
223259
*
224260
* Appends the queue statistics to the top level TLV created by
225-
* gnet_stats_start_copy().
261+
* gnet_stats_start_copy(). Using per cpu queue statistics if
262+
* they are available.
226263
*
227264
* Returns 0 on success or -1 with the statistic lock released
228265
* if the room in the socket buffer was not sufficient.
229266
*/
230267
int
231268
gnet_stats_copy_queue(struct gnet_dump *d,
269+
struct gnet_stats_queue __percpu *cpu_q,
232270
struct gnet_stats_queue *q, __u32 qlen)
233271
{
234-
q->qlen = qlen;
272+
struct gnet_stats_queue qstats = {0};
273+
274+
__gnet_stats_copy_queue(&qstats, cpu_q, q, qlen);
235275

236276
if (d->compat_tc_stats) {
237-
d->tc_stats.drops = q->drops;
238-
d->tc_stats.qlen = q->qlen;
239-
d->tc_stats.backlog = q->backlog;
240-
d->tc_stats.overlimits = q->overlimits;
277+
d->tc_stats.drops = qstats.drops;
278+
d->tc_stats.qlen = qstats.qlen;
279+
d->tc_stats.backlog = qstats.backlog;
280+
d->tc_stats.overlimits = qstats.overlimits;
241281
}
242282

243283
if (d->tail)
244-
return gnet_stats_copy(d, TCA_STATS_QUEUE, q, sizeof(*q));
284+
return gnet_stats_copy(d, TCA_STATS_QUEUE,
285+
&qstats, sizeof(qstats));
245286

246287
return 0;
247288
}

‎net/sched/act_api.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -623,7 +623,7 @@ int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *a,
623623
if (gnet_stats_copy_basic(&d, NULL, &p->tcfc_bstats) < 0 ||
624624
gnet_stats_copy_rate_est(&d, &p->tcfc_bstats,
625625
&p->tcfc_rate_est) < 0 ||
626-
gnet_stats_copy_queue(&d,
626+
gnet_stats_copy_queue(&d, NULL,
627627
&p->tcfc_qstats,
628628
p->tcfc_qstats.qlen) < 0)
629629
goto errout;

‎net/sched/sch_api.c

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -947,6 +947,10 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
947947
alloc_percpu(struct gnet_stats_basic_cpu);
948948
if (!sch->cpu_bstats)
949949
goto err_out4;
950+
951+
sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
952+
if (!sch->cpu_qstats)
953+
goto err_out4;
950954
}
951955

952956
if (tca[TCA_STAB]) {
@@ -995,6 +999,7 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
995999

9961000
err_out4:
9971001
free_percpu(sch->cpu_bstats);
1002+
free_percpu(sch->cpu_qstats);
9981003
/*
9991004
* Any broken qdiscs that would require a ops->reset() here?
10001005
* The qdisc was never in action so it shouldn't be necessary.
@@ -1313,6 +1318,7 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
13131318
u32 portid, u32 seq, u16 flags, int event)
13141319
{
13151320
struct gnet_stats_basic_cpu __percpu *cpu_bstats = NULL;
1321+
struct gnet_stats_queue __percpu *cpu_qstats = NULL;
13161322
struct tcmsg *tcm;
13171323
struct nlmsghdr *nlh;
13181324
unsigned char *b = skb_tail_pointer(skb);
@@ -1349,12 +1355,14 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
13491355
if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
13501356
goto nla_put_failure;
13511357

1352-
if (qdisc_is_percpu_stats(q))
1358+
if (qdisc_is_percpu_stats(q)) {
13531359
cpu_bstats = q->cpu_bstats;
1360+
cpu_qstats = q->cpu_qstats;
1361+
}
13541362

13551363
if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats) < 0 ||
13561364
gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 ||
1357-
gnet_stats_copy_queue(&d, &q->qstats, qlen) < 0)
1365+
gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)
13581366
goto nla_put_failure;
13591367

13601368
if (gnet_stats_finish_copy(&d) < 0)

‎net/sched/sch_atm.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -638,7 +638,7 @@ atm_tc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
638638
struct atm_flow_data *flow = (struct atm_flow_data *)arg;
639639

640640
if (gnet_stats_copy_basic(d, NULL, &flow->bstats) < 0 ||
641-
gnet_stats_copy_queue(d, &flow->qstats, flow->q->q.qlen) < 0)
641+
gnet_stats_copy_queue(d, NULL, &flow->qstats, flow->q->q.qlen) < 0)
642642
return -1;
643643

644644
return 0;

‎net/sched/sch_cbq.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1602,7 +1602,7 @@ cbq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
16021602

16031603
if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
16041604
gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
1605-
gnet_stats_copy_queue(d, &cl->qstats, cl->q->q.qlen) < 0)
1605+
gnet_stats_copy_queue(d, NULL, &cl->qstats, cl->q->q.qlen) < 0)
16061606
return -1;
16071607

16081608
return gnet_stats_copy_app(d, &cl->xstats, sizeof(cl->xstats));

‎net/sched/sch_drr.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg,
284284

285285
if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
286286
gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
287-
gnet_stats_copy_queue(d, &cl->qdisc->qstats, qlen) < 0)
287+
gnet_stats_copy_queue(d, NULL, &cl->qdisc->qstats, qlen) < 0)
288288
return -1;
289289

290290
return gnet_stats_copy_app(d, &xstats, sizeof(xstats));

‎net/sched/sch_fq_codel.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,7 @@ static int fq_codel_dump_class_stats(struct Qdisc *sch, unsigned long cl,
550550
qs.backlog = q->backlogs[idx];
551551
qs.drops = flow->dropped;
552552
}
553-
if (gnet_stats_copy_queue(d, &qs, 0) < 0)
553+
if (gnet_stats_copy_queue(d, NULL, &qs, 0) < 0)
554554
return -1;
555555
if (idx < q->flows_cnt)
556556
return gnet_stats_copy_app(d, &xstats, sizeof(xstats));

‎net/sched/sch_hfsc.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1378,7 +1378,7 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
13781378

13791379
if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
13801380
gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
1381-
gnet_stats_copy_queue(d, &cl->qstats, cl->qdisc->q.qlen) < 0)
1381+
gnet_stats_copy_queue(d, NULL, &cl->qstats, cl->qdisc->q.qlen) < 0)
13821382
return -1;
13831383

13841384
return gnet_stats_copy_app(d, &xstats, sizeof(xstats));

‎net/sched/sch_htb.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1147,7 +1147,7 @@ htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d)
11471147

11481148
if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
11491149
gnet_stats_copy_rate_est(d, NULL, &cl->rate_est) < 0 ||
1150-
gnet_stats_copy_queue(d, &cl->qstats, qlen) < 0)
1150+
gnet_stats_copy_queue(d, NULL, &cl->qstats, qlen) < 0)
11511151
return -1;
11521152

11531153
return gnet_stats_copy_app(d, &cl->xstats, sizeof(cl->xstats));

‎net/sched/sch_mq.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ static int mq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
200200

201201
sch = dev_queue->qdisc_sleeping;
202202
if (gnet_stats_copy_basic(d, NULL, &sch->bstats) < 0 ||
203-
gnet_stats_copy_queue(d, &sch->qstats, sch->q.qlen) < 0)
203+
gnet_stats_copy_queue(d, NULL, &sch->qstats, sch->q.qlen) < 0)
204204
return -1;
205205
return 0;
206206
}

‎net/sched/sch_mqprio.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -356,14 +356,15 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
356356
/* Reclaim root sleeping lock before completing stats */
357357
spin_lock_bh(d->lock);
358358
if (gnet_stats_copy_basic(d, NULL, &bstats) < 0 ||
359-
gnet_stats_copy_queue(d, &qstats, qlen) < 0)
359+
gnet_stats_copy_queue(d, NULL, &qstats, qlen) < 0)
360360
return -1;
361361
} else {
362362
struct netdev_queue *dev_queue = mqprio_queue_get(sch, cl);
363363

364364
sch = dev_queue->qdisc_sleeping;
365365
if (gnet_stats_copy_basic(d, NULL, &sch->bstats) < 0 ||
366-
gnet_stats_copy_queue(d, &sch->qstats, sch->q.qlen) < 0)
366+
gnet_stats_copy_queue(d, NULL,
367+
&sch->qstats, sch->q.qlen) < 0)
367368
return -1;
368369
}
369370
return 0;

‎net/sched/sch_multiq.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,7 +361,7 @@ static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
361361

362362
cl_q = q->queues[cl - 1];
363363
if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats) < 0 ||
364-
gnet_stats_copy_queue(d, &cl_q->qstats, cl_q->q.qlen) < 0)
364+
gnet_stats_copy_queue(d, NULL, &cl_q->qstats, cl_q->q.qlen) < 0)
365365
return -1;
366366

367367
return 0;

‎net/sched/sch_prio.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
325325

326326
cl_q = q->queues[cl - 1];
327327
if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats) < 0 ||
328-
gnet_stats_copy_queue(d, &cl_q->qstats, cl_q->q.qlen) < 0)
328+
gnet_stats_copy_queue(d, NULL, &cl_q->qstats, cl_q->q.qlen) < 0)
329329
return -1;
330330

331331
return 0;

‎net/sched/sch_qfq.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -670,7 +670,8 @@ static int qfq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
670670

671671
if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
672672
gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
673-
gnet_stats_copy_queue(d, &cl->qdisc->qstats, cl->qdisc->q.qlen) < 0)
673+
gnet_stats_copy_queue(d, NULL,
674+
&cl->qdisc->qstats, cl->qdisc->q.qlen) < 0)
674675
return -1;
675676

676677
return gnet_stats_copy_app(d, &xstats, sizeof(xstats));

‎net/sched/sch_sfq.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -871,7 +871,7 @@ static int sfq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
871871
qs.qlen = slot->qlen;
872872
qs.backlog = slot->backlog;
873873
}
874-
if (gnet_stats_copy_queue(d, &qs, qs.qlen) < 0)
874+
if (gnet_stats_copy_queue(d, NULL, &qs, qs.qlen) < 0)
875875
return -1;
876876
return gnet_stats_copy_app(d, &xstats, sizeof(xstats));
877877
}

0 commit comments

Comments
 (0)