Skip to content

Commit ec0328e

Browse files
dhowellsdhowells
committed
fscache: Maintain a catalogue of allocated cookies
Maintain a catalogue of allocated cookies so that cookie collisions can be handled properly. For the moment, this just involves printing a warning and returning a NULL cookie to the caller of fscache_acquire_cookie(), but in future it might make sense to wait for the old cookie to finish being cleaned up. This requires the cookie key to be stored attached to the cookie so that we still have the key available if the netfs relinquishes the cookie. This is done by an earlier patch. The catalogue also renders redundant fscache_netfs_list (used for checking for duplicates), so that can be removed. Signed-off-by: David Howells <dhowells@redhat.com> Acked-by: Anna Schumaker <anna.schumaker@netapp.com> Tested-by: Steve Dickson <steved@redhat.com>
1 parent ee1235a commit ec0328e

File tree

5 files changed

+279
-119
lines changed

5 files changed

+279
-119
lines changed

‎fs/fscache/cookie.c

Lines changed: 238 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,54 @@ struct kmem_cache *fscache_cookie_jar;
2121

2222
static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
2323

24+
#define fscache_cookie_hash_shift 15
25+
static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
26+
2427
static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
2528
loff_t object_size);
2629
static int fscache_alloc_object(struct fscache_cache *cache,
2730
struct fscache_cookie *cookie);
2831
static int fscache_attach_object(struct fscache_cookie *cookie,
2932
struct fscache_object *object);
3033

34+
static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
35+
{
36+
struct hlist_node *object;
37+
const u8 *k;
38+
unsigned loop;
39+
40+
pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
41+
prefix, cookie, cookie->parent, cookie->flags,
42+
atomic_read(&cookie->n_children),
43+
atomic_read(&cookie->n_active));
44+
pr_err("%c-cookie d=%p n=%p\n",
45+
prefix, cookie->def, cookie->netfs_data);
46+
47+
object = READ_ONCE(cookie->backing_objects.first);
48+
if (object)
49+
pr_err("%c-cookie o=%p\n",
50+
prefix, hlist_entry(object, struct fscache_object, cookie_link));
51+
52+
pr_err("%c-key=[%u] '", prefix, cookie->key_len);
53+
k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
54+
cookie->inline_key : cookie->key;
55+
for (loop = 0; loop < cookie->key_len; loop++)
56+
pr_cont("%02x", k[loop]);
57+
pr_cont("'\n");
58+
}
59+
60+
void fscache_free_cookie(struct fscache_cookie *cookie)
61+
{
62+
if (cookie) {
63+
BUG_ON(!hlist_empty(&cookie->backing_objects));
64+
if (cookie->aux_len > sizeof(cookie->inline_aux))
65+
kfree(cookie->aux);
66+
if (cookie->key_len > sizeof(cookie->inline_key))
67+
kfree(cookie->key);
68+
kmem_cache_free(fscache_cookie_jar, cookie);
69+
}
70+
}
71+
3172
/*
3273
* initialise an cookie jar slab element prior to any use
3374
*/
@@ -41,6 +82,170 @@ void fscache_cookie_init_once(void *_cookie)
4182
INIT_HLIST_HEAD(&cookie->backing_objects);
4283
}
4384

85+
/*
86+
* Set the index key in a cookie. The cookie struct has space for a 12-byte
87+
* key plus length and hash, but if that's not big enough, it's instead a
88+
* pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
89+
* the key data.
90+
*/
91+
static int fscache_set_key(struct fscache_cookie *cookie,
92+
const void *index_key, size_t index_key_len)
93+
{
94+
unsigned long long h;
95+
u32 *buf;
96+
int i;
97+
98+
cookie->key_len = index_key_len;
99+
100+
if (index_key_len > sizeof(cookie->inline_key)) {
101+
buf = kzalloc(index_key_len, GFP_KERNEL);
102+
if (!buf)
103+
return -ENOMEM;
104+
cookie->key = buf;
105+
} else {
106+
buf = (u32 *)cookie->inline_key;
107+
buf[0] = 0;
108+
buf[1] = 0;
109+
buf[2] = 0;
110+
}
111+
112+
memcpy(buf, index_key, index_key_len);
113+
114+
/* Calculate a hash and combine this with the length in the first word
115+
* or first half word
116+
*/
117+
h = (unsigned long)cookie->parent;
118+
h += index_key_len + cookie->type;
119+
for (i = 0; i < (index_key_len + sizeof(u32) - 1) / sizeof(u32); i++)
120+
h += buf[i];
121+
122+
cookie->key_hash = h ^ (h >> 32);
123+
return 0;
124+
}
125+
126+
static long fscache_compare_cookie(const struct fscache_cookie *a,
127+
const struct fscache_cookie *b)
128+
{
129+
const void *ka, *kb;
130+
131+
if (a->key_hash != b->key_hash)
132+
return (long)a->key_hash - (long)b->key_hash;
133+
if (a->parent != b->parent)
134+
return (long)a->parent - (long)b->parent;
135+
if (a->key_len != b->key_len)
136+
return (long)a->key_len - (long)b->key_len;
137+
if (a->type != b->type)
138+
return (long)a->type - (long)b->type;
139+
140+
if (a->key_len <= sizeof(a->inline_key)) {
141+
ka = &a->inline_key;
142+
kb = &b->inline_key;
143+
} else {
144+
ka = a->key;
145+
kb = b->key;
146+
}
147+
return memcmp(ka, kb, a->key_len);
148+
}
149+
150+
/*
151+
* Allocate a cookie.
152+
*/
153+
struct fscache_cookie *fscache_alloc_cookie(
154+
struct fscache_cookie *parent,
155+
const struct fscache_cookie_def *def,
156+
const void *index_key, size_t index_key_len,
157+
const void *aux_data, size_t aux_data_len,
158+
void *netfs_data,
159+
loff_t object_size)
160+
{
161+
struct fscache_cookie *cookie;
162+
163+
/* allocate and initialise a cookie */
164+
cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
165+
if (!cookie)
166+
return NULL;
167+
168+
cookie->key_len = index_key_len;
169+
cookie->aux_len = aux_data_len;
170+
171+
if (fscache_set_key(cookie, index_key, index_key_len) < 0)
172+
goto nomem;
173+
174+
if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
175+
memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
176+
} else {
177+
cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
178+
if (!cookie->aux)
179+
goto nomem;
180+
}
181+
182+
atomic_set(&cookie->usage, 1);
183+
atomic_set(&cookie->n_children, 0);
184+
185+
/* We keep the active count elevated until relinquishment to prevent an
186+
* attempt to wake up every time the object operations queue quiesces.
187+
*/
188+
atomic_set(&cookie->n_active, 1);
189+
190+
cookie->def = def;
191+
cookie->parent = parent;
192+
cookie->netfs_data = netfs_data;
193+
cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET);
194+
cookie->type = def->type;
195+
196+
/* radix tree insertion won't use the preallocation pool unless it's
197+
* told it may not wait */
198+
INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
199+
return cookie;
200+
201+
nomem:
202+
fscache_free_cookie(cookie);
203+
return NULL;
204+
}
205+
206+
/*
207+
* Attempt to insert the new cookie into the hash. If there's a collision, we
208+
* return the old cookie if it's not in use and an error otherwise.
209+
*/
210+
struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
211+
{
212+
struct fscache_cookie *cursor;
213+
struct hlist_bl_head *h;
214+
struct hlist_bl_node *p;
215+
unsigned int bucket;
216+
217+
bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
218+
h = &fscache_cookie_hash[bucket];
219+
220+
hlist_bl_lock(h);
221+
hlist_bl_for_each_entry(cursor, p, h, hash_link) {
222+
if (fscache_compare_cookie(candidate, cursor) == 0)
223+
goto collision;
224+
}
225+
226+
__set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
227+
fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
228+
atomic_inc(&candidate->parent->n_children);
229+
hlist_bl_add_head(&candidate->hash_link, h);
230+
hlist_bl_unlock(h);
231+
return candidate;
232+
233+
collision:
234+
if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
235+
trace_fscache_cookie(cursor, fscache_cookie_collision,
236+
atomic_read(&cursor->usage));
237+
pr_err("Duplicate cookie detected\n");
238+
fscache_print_cookie(cursor, 'O');
239+
fscache_print_cookie(candidate, 'N');
240+
hlist_bl_unlock(h);
241+
return NULL;
242+
}
243+
244+
fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
245+
hlist_bl_unlock(h);
246+
return cursor;
247+
}
248+
44249
/*
45250
* request a cookie to represent an object (index, datafile, xattr, etc)
46251
* - parent specifies the parent object
@@ -65,7 +270,7 @@ struct fscache_cookie *__fscache_acquire_cookie(
65270
loff_t object_size,
66271
bool enable)
67272
{
68-
struct fscache_cookie *cookie;
273+
struct fscache_cookie *candidate, *cookie;
69274

70275
BUG_ON(!def);
71276

@@ -95,53 +300,24 @@ struct fscache_cookie *__fscache_acquire_cookie(
95300
BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
96301
parent->type != FSCACHE_COOKIE_TYPE_INDEX);
97302

98-
/* allocate and initialise a cookie */
99-
cookie = kmem_cache_alloc(fscache_cookie_jar, GFP_KERNEL);
100-
if (!cookie) {
303+
candidate = fscache_alloc_cookie(parent, def,
304+
index_key, index_key_len,
305+
aux_data, aux_data_len,
306+
netfs_data, object_size);
307+
if (!candidate) {
101308
fscache_stat(&fscache_n_acquires_oom);
102309
_leave(" [ENOMEM]");
103310
return NULL;
104311
}
105312

106-
cookie->key_len = index_key_len;
107-
cookie->aux_len = aux_data_len;
108-
109-
if (cookie->key_len <= sizeof(cookie->inline_key)) {
110-
memcpy(cookie->inline_key, index_key, cookie->key_len);
111-
} else {
112-
cookie->key = kmemdup(index_key, cookie->key_len, GFP_KERNEL);
113-
if (!cookie->key)
114-
goto nomem;
115-
}
116-
117-
if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
118-
memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
119-
} else {
120-
cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
121-
if (!cookie->aux)
122-
goto nomem;
313+
cookie = fscache_hash_cookie(candidate);
314+
if (!cookie) {
315+
trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
316+
goto out;
123317
}
124318

125-
atomic_set(&cookie->usage, 1);
126-
atomic_set(&cookie->n_children, 0);
127-
128-
/* We keep the active count elevated until relinquishment to prevent an
129-
* attempt to wake up every time the object operations queue quiesces.
130-
*/
131-
atomic_set(&cookie->n_active, 1);
132-
133-
fscache_cookie_get(parent, fscache_cookie_get_acquire_parent);
134-
atomic_inc(&parent->n_children);
135-
136-
cookie->def = def;
137-
cookie->parent = parent;
138-
cookie->netfs_data = netfs_data;
139-
cookie->flags = (1 << FSCACHE_COOKIE_NO_DATA_YET);
140-
cookie->type = def->type;
141-
142-
/* radix tree insertion won't use the preallocation pool unless it's
143-
* told it may not wait */
144-
INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
319+
if (cookie == candidate)
320+
candidate = NULL;
145321

146322
switch (cookie->type) {
147323
case FSCACHE_COOKIE_TYPE_INDEX:
@@ -178,16 +354,10 @@ struct fscache_cookie *__fscache_acquire_cookie(
178354
}
179355

180356
fscache_stat(&fscache_n_acquires_ok);
181-
_leave(" = %p", cookie);
182-
return cookie;
183357

184-
nomem:
185-
if (cookie->aux_len > sizeof(cookie->inline_aux))
186-
kfree(cookie->aux);
187-
if (cookie->key_len > sizeof(cookie->inline_key))
188-
kfree(cookie->key);
189-
kmem_cache_free(fscache_cookie_jar, cookie);
190-
return NULL;
358+
out:
359+
fscache_free_cookie(candidate);
360+
return cookie;
191361
}
192362
EXPORT_SYMBOL(__fscache_acquire_cookie);
193363

@@ -678,6 +848,22 @@ void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
678848
}
679849
EXPORT_SYMBOL(__fscache_relinquish_cookie);
680850

851+
/*
852+
* Remove a cookie from the hash table.
853+
*/
854+
static void fscache_unhash_cookie(struct fscache_cookie *cookie)
855+
{
856+
struct hlist_bl_head *h;
857+
unsigned int bucket;
858+
859+
bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
860+
h = &fscache_cookie_hash[bucket];
861+
862+
hlist_bl_lock(h);
863+
hlist_bl_del(&cookie->hash_link);
864+
hlist_bl_unlock(h);
865+
}
866+
681867
/*
682868
* Drop a reference to a cookie.
683869
*/
@@ -698,12 +884,8 @@ void fscache_cookie_put(struct fscache_cookie *cookie,
698884
BUG_ON(usage < 0);
699885

700886
parent = cookie->parent;
701-
BUG_ON(!hlist_empty(&cookie->backing_objects));
702-
if (cookie->aux_len > sizeof(cookie->inline_aux))
703-
kfree(cookie->aux);
704-
if (cookie->key_len > sizeof(cookie->inline_key))
705-
kfree(cookie->key);
706-
kmem_cache_free(fscache_cookie_jar, cookie);
887+
fscache_unhash_cookie(cookie);
888+
fscache_free_cookie(cookie);
707889

708890
cookie = parent;
709891
where = fscache_cookie_put_parent;

‎fs/fscache/internal.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,14 @@ extern struct fscache_cache *fscache_select_cache_for_object(
4949
*/
5050
extern struct kmem_cache *fscache_cookie_jar;
5151

52+
extern void fscache_free_cookie(struct fscache_cookie *);
5253
extern void fscache_cookie_init_once(void *);
54+
extern struct fscache_cookie *fscache_alloc_cookie(struct fscache_cookie *,
55+
const struct fscache_cookie_def *,
56+
const void *, size_t,
57+
const void *, size_t,
58+
void *, loff_t);
59+
extern struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *);
5360
extern void fscache_cookie_put(struct fscache_cookie *,
5461
enum fscache_cookie_trace);
5562

0 commit comments

Comments
 (0)