Skip to content

Commit 1c0d32f

Browse files
Eric Dumazetdavem330
Eric Dumazet
authored and
davem330
committed
net_sched: gen_estimator: complete rewrite of rate estimators
1) Old code was hard to maintain, due to complex lock chains. (We probably will be able to remove some kfree_rcu() in callers) 2) Using a single timer to update all estimators does not scale. 3) Code was buggy on 32bit kernel (WRITE_ONCE() on 64bit quantity is not supposed to work well) In this rewrite : - I removed the RB tree that had to be scanned in gen_estimator_active(). qdisc dumps should be much faster. - Each estimator has its own timer. - Estimations are maintained in net_rate_estimator structure, instead of dirtying the qdisc. Minor, but part of the simplification. - Reading the estimator uses RCU and a seqcount to provide proper support for 32bit kernels. - We reduce memory need when estimators are not used, since we store a pointer, instead of the bytes/packets counters. - xt_rateest_mt() no longer has to grab a spinlock. (In the future, xt_rateest_tg() could be switched to per cpu counters) Signed-off-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent a6e1693 commit 1c0d32f

File tree

17 files changed

+182
-266
lines changed

17 files changed

+182
-266
lines changed

‎include/net/act_api.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ struct tc_action {
3636
struct tcf_t tcfa_tm;
3737
struct gnet_stats_basic_packed tcfa_bstats;
3838
struct gnet_stats_queue tcfa_qstats;
39-
struct gnet_stats_rate_est64 tcfa_rate_est;
39+
struct net_rate_estimator __rcu *tcfa_rate_est;
4040
spinlock_t tcfa_lock;
4141
struct rcu_head tcfa_rcu;
4242
struct gnet_stats_basic_cpu __percpu *cpu_bstats;

‎include/net/gen_stats.h

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ struct gnet_stats_basic_cpu {
1111
struct u64_stats_sync syncp;
1212
};
1313

14+
struct net_rate_estimator;
15+
1416
struct gnet_dump {
1517
spinlock_t * lock;
1618
struct sk_buff * skb;
@@ -42,8 +44,7 @@ void __gnet_stats_copy_basic(const seqcount_t *running,
4244
struct gnet_stats_basic_cpu __percpu *cpu,
4345
struct gnet_stats_basic_packed *b);
4446
int gnet_stats_copy_rate_est(struct gnet_dump *d,
45-
const struct gnet_stats_basic_packed *b,
46-
struct gnet_stats_rate_est64 *r);
47+
struct net_rate_estimator __rcu **ptr);
4748
int gnet_stats_copy_queue(struct gnet_dump *d,
4849
struct gnet_stats_queue __percpu *cpu_q,
4950
struct gnet_stats_queue *q, __u32 qlen);
@@ -53,16 +54,16 @@ int gnet_stats_finish_copy(struct gnet_dump *d);
5354

5455
int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
5556
struct gnet_stats_basic_cpu __percpu *cpu_bstats,
56-
struct gnet_stats_rate_est64 *rate_est,
57+
struct net_rate_estimator __rcu **rate_est,
5758
spinlock_t *stats_lock,
5859
seqcount_t *running, struct nlattr *opt);
59-
void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
60-
struct gnet_stats_rate_est64 *rate_est);
60+
void gen_kill_estimator(struct net_rate_estimator __rcu **ptr);
6161
int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
6262
struct gnet_stats_basic_cpu __percpu *cpu_bstats,
63-
struct gnet_stats_rate_est64 *rate_est,
63+
struct net_rate_estimator __rcu **ptr,
6464
spinlock_t *stats_lock,
6565
seqcount_t *running, struct nlattr *opt);
66-
bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
67-
const struct gnet_stats_rate_est64 *rate_est);
66+
bool gen_estimator_active(struct net_rate_estimator __rcu **ptr);
67+
bool gen_estimator_read(struct net_rate_estimator __rcu **ptr,
68+
struct gnet_stats_rate_est64 *sample);
6869
#endif

‎include/net/netfilter/xt_rateest.h

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,23 @@
11
#ifndef _XT_RATEEST_H
22
#define _XT_RATEEST_H
33

4+
#include <net/gen_stats.h>
5+
46
struct xt_rateest {
57
/* keep lock and bstats on same cache line to speedup xt_rateest_tg() */
68
struct gnet_stats_basic_packed bstats;
79
spinlock_t lock;
8-
/* keep rstats and lock on same cache line to speedup xt_rateest_mt() */
9-
struct gnet_stats_rate_est64 rstats;
10+
1011

1112
/* following fields not accessed in hot path */
13+
unsigned int refcnt;
1214
struct hlist_node list;
1315
char name[IFNAMSIZ];
14-
unsigned int refcnt;
1516
struct gnet_estimator params;
1617
struct rcu_head rcu;
18+
19+
/* keep this field far away to speedup xt_rateest_mt() */
20+
struct net_rate_estimator __rcu *rate_est;
1721
};
1822

1923
struct xt_rateest *xt_rateest_lookup(const char *name);

‎include/net/sch_generic.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ struct Qdisc {
7676

7777
struct netdev_queue *dev_queue;
7878

79-
struct gnet_stats_rate_est64 rate_est;
79+
struct net_rate_estimator __rcu *rate_est;
8080
struct gnet_stats_basic_cpu __percpu *cpu_bstats;
8181
struct gnet_stats_queue __percpu *cpu_qstats;
8282

0 commit comments

Comments
 (0)