Skip to content

Commit e625305

Browse files
htejunhtejun
committed
percpu-refcount: make percpu_ref based on longs instead of ints
percpu_ref is currently based on ints and the number of refs it can cover is (1 << 31). This makes it impossible to use a percpu_ref to count memory objects or pages on 64bit machines as it may overflow. This forces those users to somehow aggregate the references before contributing to the percpu_ref which is often cumbersome and sometimes challenging to get the same level of performance as using the percpu_ref directly. While using ints for the percpu counters makes them pack tighter on 64bit machines, the possible gain from using ints instead of longs is extremely small compared to the overall gain from per-cpu operation. This patch makes percpu_ref based on longs so that it can be used to directly count memory objects or pages. Signed-off-by: Tejun Heo <tj@kernel.org> Cc: Kent Overstreet <kmo@daterainc.com> Cc: Johannes Weiner <hannes@cmpxchg.org>
1 parent 4843c33 commit e625305

File tree

2 files changed

+31
-30
lines changed

2 files changed

+31
-30
lines changed

‎include/linux/percpu-refcount.h

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ struct percpu_ref;
5555
typedef void (percpu_ref_func_t)(struct percpu_ref *);
5656

5757
struct percpu_ref {
58-
atomic_t count;
58+
atomic_long_t count;
5959
/*
6060
* The low bit of the pointer indicates whether the ref is in percpu
6161
* mode; if set, then get/put will manipulate the atomic_t.
@@ -97,7 +97,7 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
9797
* branches as it can't assume that @ref->pcpu_count is not NULL.
9898
*/
9999
static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
100-
unsigned __percpu **pcpu_countp)
100+
unsigned long __percpu **pcpu_countp)
101101
{
102102
unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr);
103103

@@ -107,7 +107,7 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
107107
if (unlikely(pcpu_ptr & PCPU_REF_DEAD))
108108
return false;
109109

110-
*pcpu_countp = (unsigned __percpu *)pcpu_ptr;
110+
*pcpu_countp = (unsigned long __percpu *)pcpu_ptr;
111111
return true;
112112
}
113113

@@ -119,14 +119,14 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
119119
*/
120120
static inline void percpu_ref_get(struct percpu_ref *ref)
121121
{
122-
unsigned __percpu *pcpu_count;
122+
unsigned long __percpu *pcpu_count;
123123

124124
rcu_read_lock_sched();
125125

126126
if (__pcpu_ref_alive(ref, &pcpu_count))
127127
this_cpu_inc(*pcpu_count);
128128
else
129-
atomic_inc(&ref->count);
129+
atomic_long_inc(&ref->count);
130130

131131
rcu_read_unlock_sched();
132132
}
@@ -142,7 +142,7 @@ static inline void percpu_ref_get(struct percpu_ref *ref)
142142
*/
143143
static inline bool percpu_ref_tryget(struct percpu_ref *ref)
144144
{
145-
unsigned __percpu *pcpu_count;
145+
unsigned long __percpu *pcpu_count;
146146
int ret = false;
147147

148148
rcu_read_lock_sched();
@@ -151,7 +151,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
151151
this_cpu_inc(*pcpu_count);
152152
ret = true;
153153
} else {
154-
ret = atomic_inc_not_zero(&ref->count);
154+
ret = atomic_long_inc_not_zero(&ref->count);
155155
}
156156

157157
rcu_read_unlock_sched();
@@ -175,7 +175,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
175175
*/
176176
static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
177177
{
178-
unsigned __percpu *pcpu_count;
178+
unsigned long __percpu *pcpu_count;
179179
int ret = false;
180180

181181
rcu_read_lock_sched();
@@ -199,13 +199,13 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
199199
*/
200200
static inline void percpu_ref_put(struct percpu_ref *ref)
201201
{
202-
unsigned __percpu *pcpu_count;
202+
unsigned long __percpu *pcpu_count;
203203

204204
rcu_read_lock_sched();
205205

206206
if (__pcpu_ref_alive(ref, &pcpu_count))
207207
this_cpu_dec(*pcpu_count);
208-
else if (unlikely(atomic_dec_and_test(&ref->count)))
208+
else if (unlikely(atomic_long_dec_and_test(&ref->count)))
209209
ref->release(ref);
210210

211211
rcu_read_unlock_sched();
@@ -219,11 +219,11 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
219219
*/
220220
static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
221221
{
222-
unsigned __percpu *pcpu_count;
222+
unsigned long __percpu *pcpu_count;
223223

224224
if (__pcpu_ref_alive(ref, &pcpu_count))
225225
return false;
226-
return !atomic_read(&ref->count);
226+
return !atomic_long_read(&ref->count);
227227
}
228228

229229
#endif

‎lib/percpu-refcount.c

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,15 @@
2525
* works.
2626
*
2727
* Converting to non percpu mode is done with some RCUish stuff in
28-
* percpu_ref_kill. Additionally, we need a bias value so that the atomic_t
29-
* can't hit 0 before we've added up all the percpu refs.
28+
* percpu_ref_kill. Additionally, we need a bias value so that the
29+
* atomic_long_t can't hit 0 before we've added up all the percpu refs.
3030
*/
3131

32-
#define PCPU_COUNT_BIAS (1U << 31)
32+
#define PCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1))
3333

34-
static unsigned __percpu *pcpu_count_ptr(struct percpu_ref *ref)
34+
static unsigned long __percpu *pcpu_count_ptr(struct percpu_ref *ref)
3535
{
36-
return (unsigned __percpu *)(ref->pcpu_count_ptr & ~PCPU_REF_DEAD);
36+
return (unsigned long __percpu *)(ref->pcpu_count_ptr & ~PCPU_REF_DEAD);
3737
}
3838

3939
/**
@@ -43,17 +43,17 @@ static unsigned __percpu *pcpu_count_ptr(struct percpu_ref *ref)
4343
* @gfp: allocation mask to use
4444
*
4545
* Initializes the refcount in single atomic counter mode with a refcount of 1;
46-
* analagous to atomic_set(ref, 1).
46+
* analagous to atomic_long_set(ref, 1).
4747
*
4848
* Note that @release must not sleep - it may potentially be called from RCU
4949
* callback context by percpu_ref_kill().
5050
*/
5151
int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release,
5252
gfp_t gfp)
5353
{
54-
atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);
54+
atomic_long_set(&ref->count, 1 + PCPU_COUNT_BIAS);
5555

56-
ref->pcpu_count_ptr = (unsigned long)alloc_percpu_gfp(unsigned, gfp);
56+
ref->pcpu_count_ptr = (unsigned long)alloc_percpu_gfp(unsigned long, gfp);
5757
if (!ref->pcpu_count_ptr)
5858
return -ENOMEM;
5959

@@ -75,13 +75,13 @@ EXPORT_SYMBOL_GPL(percpu_ref_init);
7575
*/
7676
void percpu_ref_reinit(struct percpu_ref *ref)
7777
{
78-
unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
78+
unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref);
7979
int cpu;
8080

8181
BUG_ON(!pcpu_count);
8282
WARN_ON(!percpu_ref_is_zero(ref));
8383

84-
atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);
84+
atomic_long_set(&ref->count, 1 + PCPU_COUNT_BIAS);
8585

8686
/*
8787
* Restore per-cpu operation. smp_store_release() is paired with
@@ -109,7 +109,7 @@ EXPORT_SYMBOL_GPL(percpu_ref_reinit);
109109
*/
110110
void percpu_ref_exit(struct percpu_ref *ref)
111111
{
112-
unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
112+
unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref);
113113

114114
if (pcpu_count) {
115115
free_percpu(pcpu_count);
@@ -121,14 +121,15 @@ EXPORT_SYMBOL_GPL(percpu_ref_exit);
121121
static void percpu_ref_kill_rcu(struct rcu_head *rcu)
122122
{
123123
struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
124-
unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
125-
unsigned count = 0;
124+
unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref);
125+
unsigned long count = 0;
126126
int cpu;
127127

128128
for_each_possible_cpu(cpu)
129129
count += *per_cpu_ptr(pcpu_count, cpu);
130130

131-
pr_debug("global %i pcpu %i", atomic_read(&ref->count), (int) count);
131+
pr_debug("global %ld pcpu %ld",
132+
atomic_long_read(&ref->count), (long)count);
132133

133134
/*
134135
* It's crucial that we sum the percpu counters _before_ adding the sum
@@ -143,11 +144,11 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu)
143144
* time is equivalent and saves us atomic operations:
144145
*/
145146

146-
atomic_add((int) count - PCPU_COUNT_BIAS, &ref->count);
147+
atomic_long_add((long)count - PCPU_COUNT_BIAS, &ref->count);
147148

148-
WARN_ONCE(atomic_read(&ref->count) <= 0,
149-
"percpu ref (%pf) <= 0 (%i) after killed",
150-
ref->release, atomic_read(&ref->count));
149+
WARN_ONCE(atomic_long_read(&ref->count) <= 0,
150+
"percpu ref (%pf) <= 0 (%ld) after killed",
151+
ref->release, atomic_long_read(&ref->count));
151152

152153
/* @ref is viewed as dead on all CPUs, send out kill confirmation */
153154
if (ref->confirm_kill)

0 commit comments

Comments
 (0)