Skip to content

Commit b0e4dad

Browse files
author
J. Bruce Fields
committed
SUNRPC: Allow cache lookups to use RCU protection rather than the r/w spinlock
Instead of the reader/writer spinlock, allow cache lookups to use RCU for looking up entries. This is more efficient since modifications can occur while other entries are being looked up. Note that for now, we keep the reader/writer spinlock until all users have been converted to use RCU-safe freeing of their cache entries. Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com> Signed-off-by: J. Bruce Fields <bfields@redhat.com>
1 parent b92a8fa commit b0e4dad

File tree

2 files changed

+91
-14
lines changed

2 files changed

+91
-14
lines changed

‎include/linux/sunrpc/cache.h

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,9 @@ extern const struct file_operations cache_file_operations_pipefs;
167167
extern const struct file_operations content_file_operations_pipefs;
168168
extern const struct file_operations cache_flush_operations_pipefs;
169169

170+
extern struct cache_head *
171+
sunrpc_cache_lookup_rcu(struct cache_detail *detail,
172+
struct cache_head *key, int hash);
170173
extern struct cache_head *
171174
sunrpc_cache_lookup(struct cache_detail *detail,
172175
struct cache_head *key, int hash);
@@ -186,6 +189,12 @@ static inline struct cache_head *cache_get(struct cache_head *h)
186189
return h;
187190
}
188191

192+
static inline struct cache_head *cache_get_rcu(struct cache_head *h)
193+
{
194+
if (kref_get_unless_zero(&h->ref))
195+
return h;
196+
return NULL;
197+
}
189198

190199
static inline void cache_put(struct cache_head *h, struct cache_detail *cd)
191200
{
@@ -227,6 +236,9 @@ extern void sunrpc_cache_unhash(struct cache_detail *, struct cache_head *);
227236
extern void *cache_seq_start(struct seq_file *file, loff_t *pos);
228237
extern void *cache_seq_next(struct seq_file *file, void *p, loff_t *pos);
229238
extern void cache_seq_stop(struct seq_file *file, void *p);
239+
extern void *cache_seq_start_rcu(struct seq_file *file, loff_t *pos);
240+
extern void *cache_seq_next_rcu(struct seq_file *file, void *p, loff_t *pos);
241+
extern void cache_seq_stop_rcu(struct seq_file *file, void *p);
230242

231243
extern void qword_add(char **bpp, int *lp, char *str);
232244
extern void qword_addhex(char **bpp, int *lp, char *buf, int blen);

‎net/sunrpc/cache.c

Lines changed: 79 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,34 @@ static void cache_init(struct cache_head *h, struct cache_detail *detail)
5454
h->last_refresh = now;
5555
}
5656

57+
static struct cache_head *sunrpc_cache_find_rcu(struct cache_detail *detail,
58+
struct cache_head *key,
59+
int hash)
60+
{
61+
struct hlist_head *head = &detail->hash_table[hash];
62+
struct cache_head *tmp;
63+
64+
rcu_read_lock();
65+
hlist_for_each_entry_rcu(tmp, head, cache_list) {
66+
if (detail->match(tmp, key)) {
67+
if (cache_is_expired(detail, tmp))
68+
continue;
69+
tmp = cache_get_rcu(tmp);
70+
rcu_read_unlock();
71+
return tmp;
72+
}
73+
}
74+
rcu_read_unlock();
75+
return NULL;
76+
}
77+
5778
static struct cache_head *sunrpc_cache_find(struct cache_detail *detail,
5879
struct cache_head *key, int hash)
5980
{
6081
struct hlist_head *head = &detail->hash_table[hash];
6182
struct cache_head *tmp;
6283

6384
read_lock(&detail->hash_lock);
64-
6585
hlist_for_each_entry(tmp, head, cache_list) {
6686
if (detail->match(tmp, key)) {
6787
if (cache_is_expired(detail, tmp))
@@ -96,10 +116,10 @@ static struct cache_head *sunrpc_cache_add_entry(struct cache_detail *detail,
96116
write_lock(&detail->hash_lock);
97117

98118
/* check if entry appeared while we slept */
99-
hlist_for_each_entry(tmp, head, cache_list) {
119+
hlist_for_each_entry_rcu(tmp, head, cache_list) {
100120
if (detail->match(tmp, key)) {
101121
if (cache_is_expired(detail, tmp)) {
102-
hlist_del_init(&tmp->cache_list);
122+
hlist_del_init_rcu(&tmp->cache_list);
103123
detail->entries --;
104124
freeme = tmp;
105125
break;
@@ -111,7 +131,7 @@ static struct cache_head *sunrpc_cache_add_entry(struct cache_detail *detail,
111131
}
112132
}
113133

114-
hlist_add_head(&new->cache_list, head);
134+
hlist_add_head_rcu(&new->cache_list, head);
115135
detail->entries++;
116136
cache_get(new);
117137
write_unlock(&detail->hash_lock);
@@ -121,6 +141,19 @@ static struct cache_head *sunrpc_cache_add_entry(struct cache_detail *detail,
121141
return new;
122142
}
123143

144+
struct cache_head *sunrpc_cache_lookup_rcu(struct cache_detail *detail,
145+
struct cache_head *key, int hash)
146+
{
147+
struct cache_head *ret;
148+
149+
ret = sunrpc_cache_find_rcu(detail, key, hash);
150+
if (ret)
151+
return ret;
152+
/* Didn't find anything, insert an empty entry */
153+
return sunrpc_cache_add_entry(detail, key, hash);
154+
}
155+
EXPORT_SYMBOL_GPL(sunrpc_cache_lookup_rcu);
156+
124157
struct cache_head *sunrpc_cache_lookup(struct cache_detail *detail,
125158
struct cache_head *key, int hash)
126159
{
@@ -134,6 +167,7 @@ struct cache_head *sunrpc_cache_lookup(struct cache_detail *detail,
134167
}
135168
EXPORT_SYMBOL_GPL(sunrpc_cache_lookup);
136169

170+
137171
static void cache_dequeue(struct cache_detail *detail, struct cache_head *ch);
138172

139173
static void cache_fresh_locked(struct cache_head *head, time_t expiry,
@@ -450,7 +484,7 @@ static int cache_clean(void)
450484
if (!cache_is_expired(current_detail, ch))
451485
continue;
452486

453-
hlist_del_init(&ch->cache_list);
487+
hlist_del_init_rcu(&ch->cache_list);
454488
current_detail->entries--;
455489
rv = 1;
456490
break;
@@ -521,7 +555,7 @@ void cache_purge(struct cache_detail *detail)
521555
for (i = 0; i < detail->hash_size; i++) {
522556
head = &detail->hash_table[i];
523557
hlist_for_each_entry_safe(ch, tmp, head, cache_list) {
524-
hlist_del_init(&ch->cache_list);
558+
hlist_del_init_rcu(&ch->cache_list);
525559
detail->entries--;
526560

527561
set_bit(CACHE_CLEANED, &ch->flags);
@@ -1306,21 +1340,19 @@ EXPORT_SYMBOL_GPL(qword_get);
13061340
* get a header, then pass each real item in the cache
13071341
*/
13081342

1309-
void *cache_seq_start(struct seq_file *m, loff_t *pos)
1310-
__acquires(cd->hash_lock)
1343+
static void *__cache_seq_start(struct seq_file *m, loff_t *pos)
13111344
{
13121345
loff_t n = *pos;
13131346
unsigned int hash, entry;
13141347
struct cache_head *ch;
13151348
struct cache_detail *cd = m->private;
13161349

1317-
read_lock(&cd->hash_lock);
13181350
if (!n--)
13191351
return SEQ_START_TOKEN;
13201352
hash = n >> 32;
13211353
entry = n & ((1LL<<32) - 1);
13221354

1323-
hlist_for_each_entry(ch, &cd->hash_table[hash], cache_list)
1355+
hlist_for_each_entry_rcu(ch, &cd->hash_table[hash], cache_list)
13241356
if (!entry--)
13251357
return ch;
13261358
n &= ~((1LL<<32) - 1);
@@ -1332,9 +1364,19 @@ void *cache_seq_start(struct seq_file *m, loff_t *pos)
13321364
if (hash >= cd->hash_size)
13331365
return NULL;
13341366
*pos = n+1;
1335-
return hlist_entry_safe(cd->hash_table[hash].first,
1367+
return hlist_entry_safe(rcu_dereference_raw(
1368+
hlist_first_rcu(&cd->hash_table[hash])),
13361369
struct cache_head, cache_list);
13371370
}
1371+
1372+
void *cache_seq_start(struct seq_file *m, loff_t *pos)
1373+
__acquires(cd->hash_lock)
1374+
{
1375+
struct cache_detail *cd = m->private;
1376+
1377+
read_lock(&cd->hash_lock);
1378+
return __cache_seq_start(m, pos);
1379+
}
13381380
EXPORT_SYMBOL_GPL(cache_seq_start);
13391381

13401382
void *cache_seq_next(struct seq_file *m, void *p, loff_t *pos)
@@ -1350,7 +1392,8 @@ void *cache_seq_next(struct seq_file *m, void *p, loff_t *pos)
13501392
*pos += 1LL<<32;
13511393
} else {
13521394
++*pos;
1353-
return hlist_entry_safe(ch->cache_list.next,
1395+
return hlist_entry_safe(rcu_dereference_raw(
1396+
hlist_next_rcu(&ch->cache_list)),
13541397
struct cache_head, cache_list);
13551398
}
13561399
*pos &= ~((1LL<<32) - 1);
@@ -1362,7 +1405,8 @@ void *cache_seq_next(struct seq_file *m, void *p, loff_t *pos)
13621405
if (hash >= cd->hash_size)
13631406
return NULL;
13641407
++*pos;
1365-
return hlist_entry_safe(cd->hash_table[hash].first,
1408+
return hlist_entry_safe(rcu_dereference_raw(
1409+
hlist_first_rcu(&cd->hash_table[hash])),
13661410
struct cache_head, cache_list);
13671411
}
13681412
EXPORT_SYMBOL_GPL(cache_seq_next);
@@ -1375,6 +1419,27 @@ void cache_seq_stop(struct seq_file *m, void *p)
13751419
}
13761420
EXPORT_SYMBOL_GPL(cache_seq_stop);
13771421

1422+
void *cache_seq_start_rcu(struct seq_file *m, loff_t *pos)
1423+
__acquires(RCU)
1424+
{
1425+
rcu_read_lock();
1426+
return __cache_seq_start(m, pos);
1427+
}
1428+
EXPORT_SYMBOL_GPL(cache_seq_start_rcu);
1429+
1430+
void *cache_seq_next_rcu(struct seq_file *file, void *p, loff_t *pos)
1431+
{
1432+
return cache_seq_next(file, p, pos);
1433+
}
1434+
EXPORT_SYMBOL_GPL(cache_seq_next_rcu);
1435+
1436+
void cache_seq_stop_rcu(struct seq_file *m, void *p)
1437+
__releases(RCU)
1438+
{
1439+
rcu_read_unlock();
1440+
}
1441+
EXPORT_SYMBOL_GPL(cache_seq_stop_rcu);
1442+
13781443
static int c_show(struct seq_file *m, void *p)
13791444
{
13801445
struct cache_head *cp = p;
@@ -1863,7 +1928,7 @@ void sunrpc_cache_unhash(struct cache_detail *cd, struct cache_head *h)
18631928
{
18641929
write_lock(&cd->hash_lock);
18651930
if (!hlist_unhashed(&h->cache_list)){
1866-
hlist_del_init(&h->cache_list);
1931+
hlist_del_init_rcu(&h->cache_list);
18671932
cd->entries--;
18681933
write_unlock(&cd->hash_lock);
18691934
cache_put(h, cd);

0 commit comments

Comments
 (0)