Skip to content

Commit ad3d6c7

Browse files
author
Matthew Wilcox
committed
xarray: Add XArray load operation
The xa_load function brings with it a lot of infrastructure; xa_empty(), xa_is_err(), and large chunks of the XArray advanced API that are used to implement xa_load. As the test-suite demonstrates, it is possible to use the XArray functions on a radix tree. The radix tree functions depend on the GFP flags being stored in the root of the tree, so it's not possible to use the radix tree functions on an XArray. Signed-off-by: Matthew Wilcox <willy@infradead.org>
1 parent 992a8e6 commit ad3d6c7

File tree

14 files changed

+661
-45
lines changed

14 files changed

+661
-45
lines changed

‎include/linux/xarray.h

Lines changed: 336 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
#include <linux/bug.h>
1313
#include <linux/compiler.h>
1414
#include <linux/kconfig.h>
15+
#include <linux/kernel.h>
16+
#include <linux/rcupdate.h>
1517
#include <linux/spinlock.h>
1618
#include <linux/types.h>
1719

@@ -30,6 +32,10 @@
3032
*
3133
* 0-62: Sibling entries
3234
* 256: Retry entry
35+
*
36+
* Errors are also represented as internal entries, but use the negative
37+
* space (-4094 to -2). They're never stored in the slots array; only
38+
* returned by the normal API.
3339
*/
3440

3541
#define BITS_PER_XA_VALUE (BITS_PER_LONG - 1)
@@ -155,6 +161,42 @@ static inline bool xa_is_internal(const void *entry)
155161
return ((unsigned long)entry & 3) == 2;
156162
}
157163

164+
/**
165+
* xa_is_err() - Report whether an XArray operation returned an error
166+
* @entry: Result from calling an XArray function
167+
*
168+
* If an XArray operation cannot complete an operation, it will return
169+
* a special value indicating an error. This function tells you
170+
* whether an error occurred; xa_err() tells you which error occurred.
171+
*
172+
* Context: Any context.
173+
* Return: %true if the entry indicates an error.
174+
*/
175+
static inline bool xa_is_err(const void *entry)
176+
{
177+
return unlikely(xa_is_internal(entry));
178+
}
179+
180+
/**
181+
* xa_err() - Turn an XArray result into an errno.
182+
* @entry: Result from calling an XArray function.
183+
*
184+
* If an XArray operation cannot complete an operation, it will return
185+
* a special pointer value which encodes an errno. This function extracts
186+
* the errno from the pointer value, or returns 0 if the pointer does not
187+
* represent an errno.
188+
*
189+
* Context: Any context.
190+
* Return: A negative errno or 0.
191+
*/
192+
static inline int xa_err(void *entry)
193+
{
194+
/* xa_to_internal() would not do sign extension. */
195+
if (xa_is_err(entry))
196+
return (long)entry >> 2;
197+
return 0;
198+
}
199+
158200
/**
159201
* struct xarray - The anchor of the XArray.
160202
* @xa_lock: Lock that protects the contents of the XArray.
@@ -209,6 +251,7 @@ struct xarray {
209251
#define DEFINE_XARRAY(name) DEFINE_XARRAY_FLAGS(name, 0)
210252

211253
void xa_init_flags(struct xarray *, gfp_t flags);
254+
void *xa_load(struct xarray *, unsigned long index);
212255

213256
/**
214257
* xa_init() - Initialise an empty XArray.
@@ -223,6 +266,18 @@ static inline void xa_init(struct xarray *xa)
223266
xa_init_flags(xa, 0);
224267
}
225268

269+
/**
270+
* xa_empty() - Determine if an array has any present entries.
271+
* @xa: XArray.
272+
*
273+
* Context: Any context.
274+
* Return: %true if the array contains only NULL pointers.
275+
*/
276+
static inline bool xa_empty(const struct xarray *xa)
277+
{
278+
return xa->xa_head == NULL;
279+
}
280+
226281
#define xa_trylock(xa) spin_trylock(&(xa)->xa_lock)
227282
#define xa_lock(xa) spin_lock(&(xa)->xa_lock)
228283
#define xa_unlock(xa) spin_unlock(&(xa)->xa_lock)
@@ -280,6 +335,65 @@ struct xa_node {
280335
};
281336
};
282337

338+
void xa_dump(const struct xarray *);
339+
void xa_dump_node(const struct xa_node *);
340+
341+
#ifdef XA_DEBUG
342+
#define XA_BUG_ON(xa, x) do { \
343+
if (x) { \
344+
xa_dump(xa); \
345+
BUG(); \
346+
} \
347+
} while (0)
348+
#define XA_NODE_BUG_ON(node, x) do { \
349+
if (x) { \
350+
if (node) xa_dump_node(node); \
351+
BUG(); \
352+
} \
353+
} while (0)
354+
#else
355+
#define XA_BUG_ON(xa, x) do { } while (0)
356+
#define XA_NODE_BUG_ON(node, x) do { } while (0)
357+
#endif
358+
359+
/* Private */
360+
static inline void *xa_head(const struct xarray *xa)
361+
{
362+
return rcu_dereference_check(xa->xa_head,
363+
lockdep_is_held(&xa->xa_lock));
364+
}
365+
366+
/* Private */
367+
static inline void *xa_head_locked(const struct xarray *xa)
368+
{
369+
return rcu_dereference_protected(xa->xa_head,
370+
lockdep_is_held(&xa->xa_lock));
371+
}
372+
373+
/* Private */
374+
static inline void *xa_entry(const struct xarray *xa,
375+
const struct xa_node *node, unsigned int offset)
376+
{
377+
XA_NODE_BUG_ON(node, offset >= XA_CHUNK_SIZE);
378+
return rcu_dereference_check(node->slots[offset],
379+
lockdep_is_held(&xa->xa_lock));
380+
}
381+
382+
/* Private */
383+
static inline void *xa_entry_locked(const struct xarray *xa,
384+
const struct xa_node *node, unsigned int offset)
385+
{
386+
XA_NODE_BUG_ON(node, offset >= XA_CHUNK_SIZE);
387+
return rcu_dereference_protected(node->slots[offset],
388+
lockdep_is_held(&xa->xa_lock));
389+
}
390+
391+
/* Private */
392+
static inline struct xa_node *xa_to_node(const void *entry)
393+
{
394+
return (struct xa_node *)((unsigned long)entry - 2);
395+
}
396+
283397
/* Private */
284398
static inline bool xa_is_node(const void *entry)
285399
{
@@ -312,4 +426,226 @@ static inline bool xa_is_sibling(const void *entry)
312426

313427
#define XA_RETRY_ENTRY xa_mk_internal(256)
314428

429+
/**
430+
* xa_is_retry() - Is the entry a retry entry?
431+
* @entry: Entry retrieved from the XArray
432+
*
433+
* Return: %true if the entry is a retry entry.
434+
*/
435+
static inline bool xa_is_retry(const void *entry)
436+
{
437+
return unlikely(entry == XA_RETRY_ENTRY);
438+
}
439+
440+
/**
441+
* typedef xa_update_node_t - A callback function from the XArray.
442+
* @node: The node which is being processed
443+
*
444+
* This function is called every time the XArray updates the count of
445+
* present and value entries in a node. It allows advanced users to
446+
* maintain the private_list in the node.
447+
*
448+
* Context: The xa_lock is held and interrupts may be disabled.
449+
* Implementations should not drop the xa_lock, nor re-enable
450+
* interrupts.
451+
*/
452+
typedef void (*xa_update_node_t)(struct xa_node *node);
453+
454+
/*
455+
* The xa_state is opaque to its users. It contains various different pieces
456+
* of state involved in the current operation on the XArray. It should be
457+
* declared on the stack and passed between the various internal routines.
458+
* The various elements in it should not be accessed directly, but only
459+
* through the provided accessor functions. The below documentation is for
460+
* the benefit of those working on the code, not for users of the XArray.
461+
*
462+
* @xa_node usually points to the xa_node containing the slot we're operating
463+
* on (and @xa_offset is the offset in the slots array). If there is a
464+
* single entry in the array at index 0, there are no allocated xa_nodes to
465+
* point to, and so we store %NULL in @xa_node. @xa_node is set to
466+
* the value %XAS_RESTART if the xa_state is not walked to the correct
467+
* position in the tree of nodes for this operation. If an error occurs
468+
* during an operation, it is set to an %XAS_ERROR value. If we run off the
469+
* end of the allocated nodes, it is set to %XAS_BOUNDS.
470+
*/
471+
struct xa_state {
472+
struct xarray *xa;
473+
unsigned long xa_index;
474+
unsigned char xa_shift;
475+
unsigned char xa_sibs;
476+
unsigned char xa_offset;
477+
unsigned char xa_pad; /* Helps gcc generate better code */
478+
struct xa_node *xa_node;
479+
struct xa_node *xa_alloc;
480+
xa_update_node_t xa_update;
481+
};
482+
483+
/*
484+
* We encode errnos in the xas->xa_node. If an error has happened, we need to
485+
* drop the lock to fix it, and once we've done so the xa_state is invalid.
486+
*/
487+
#define XA_ERROR(errno) ((struct xa_node *)(((unsigned long)errno << 2) | 2UL))
488+
#define XAS_BOUNDS ((struct xa_node *)1UL)
489+
#define XAS_RESTART ((struct xa_node *)3UL)
490+
491+
#define __XA_STATE(array, index, shift, sibs) { \
492+
.xa = array, \
493+
.xa_index = index, \
494+
.xa_shift = shift, \
495+
.xa_sibs = sibs, \
496+
.xa_offset = 0, \
497+
.xa_pad = 0, \
498+
.xa_node = XAS_RESTART, \
499+
.xa_alloc = NULL, \
500+
.xa_update = NULL \
501+
}
502+
503+
/**
504+
* XA_STATE() - Declare an XArray operation state.
505+
* @name: Name of this operation state (usually xas).
506+
* @array: Array to operate on.
507+
* @index: Initial index of interest.
508+
*
509+
* Declare and initialise an xa_state on the stack.
510+
*/
511+
#define XA_STATE(name, array, index) \
512+
struct xa_state name = __XA_STATE(array, index, 0, 0)
513+
514+
/**
515+
* XA_STATE_ORDER() - Declare an XArray operation state.
516+
* @name: Name of this operation state (usually xas).
517+
* @array: Array to operate on.
518+
* @index: Initial index of interest.
519+
* @order: Order of entry.
520+
*
521+
* Declare and initialise an xa_state on the stack. This variant of
522+
* XA_STATE() allows you to specify the 'order' of the element you
523+
* want to operate on.`
524+
*/
525+
#define XA_STATE_ORDER(name, array, index, order) \
526+
struct xa_state name = __XA_STATE(array, \
527+
(index >> order) << order, \
528+
order - (order % XA_CHUNK_SHIFT), \
529+
(1U << (order % XA_CHUNK_SHIFT)) - 1)
530+
531+
#define xas_marked(xas, mark) xa_marked((xas)->xa, (mark))
532+
#define xas_trylock(xas) xa_trylock((xas)->xa)
533+
#define xas_lock(xas) xa_lock((xas)->xa)
534+
#define xas_unlock(xas) xa_unlock((xas)->xa)
535+
#define xas_lock_bh(xas) xa_lock_bh((xas)->xa)
536+
#define xas_unlock_bh(xas) xa_unlock_bh((xas)->xa)
537+
#define xas_lock_irq(xas) xa_lock_irq((xas)->xa)
538+
#define xas_unlock_irq(xas) xa_unlock_irq((xas)->xa)
539+
#define xas_lock_irqsave(xas, flags) \
540+
xa_lock_irqsave((xas)->xa, flags)
541+
#define xas_unlock_irqrestore(xas, flags) \
542+
xa_unlock_irqrestore((xas)->xa, flags)
543+
544+
/**
545+
* xas_error() - Return an errno stored in the xa_state.
546+
* @xas: XArray operation state.
547+
*
548+
* Return: 0 if no error has been noted. A negative errno if one has.
549+
*/
550+
static inline int xas_error(const struct xa_state *xas)
551+
{
552+
return xa_err(xas->xa_node);
553+
}
554+
555+
/**
556+
* xas_set_err() - Note an error in the xa_state.
557+
* @xas: XArray operation state.
558+
* @err: Negative error number.
559+
*
560+
* Only call this function with a negative @err; zero or positive errors
561+
* will probably not behave the way you think they should. If you want
562+
* to clear the error from an xa_state, use xas_reset().
563+
*/
564+
static inline void xas_set_err(struct xa_state *xas, long err)
565+
{
566+
xas->xa_node = XA_ERROR(err);
567+
}
568+
569+
/**
570+
* xas_invalid() - Is the xas in a retry or error state?
571+
* @xas: XArray operation state.
572+
*
573+
* Return: %true if the xas cannot be used for operations.
574+
*/
575+
static inline bool xas_invalid(const struct xa_state *xas)
576+
{
577+
return (unsigned long)xas->xa_node & 3;
578+
}
579+
580+
/**
581+
* xas_valid() - Is the xas a valid cursor into the array?
582+
* @xas: XArray operation state.
583+
*
584+
* Return: %true if the xas can be used for operations.
585+
*/
586+
static inline bool xas_valid(const struct xa_state *xas)
587+
{
588+
return !xas_invalid(xas);
589+
}
590+
591+
/**
592+
* xas_reset() - Reset an XArray operation state.
593+
* @xas: XArray operation state.
594+
*
595+
* Resets the error or walk state of the @xas so future walks of the
596+
* array will start from the root. Use this if you have dropped the
597+
* xarray lock and want to reuse the xa_state.
598+
*
599+
* Context: Any context.
600+
*/
601+
static inline void xas_reset(struct xa_state *xas)
602+
{
603+
xas->xa_node = XAS_RESTART;
604+
}
605+
606+
/**
607+
* xas_retry() - Retry the operation if appropriate.
608+
* @xas: XArray operation state.
609+
* @entry: Entry from xarray.
610+
*
611+
* The advanced functions may sometimes return an internal entry, such as
612+
* a retry entry or a zero entry. This function sets up the @xas to restart
613+
* the walk from the head of the array if needed.
614+
*
615+
* Context: Any context.
616+
* Return: true if the operation needs to be retried.
617+
*/
618+
static inline bool xas_retry(struct xa_state *xas, const void *entry)
619+
{
620+
if (!xa_is_retry(entry))
621+
return false;
622+
xas_reset(xas);
623+
return true;
624+
}
625+
626+
void *xas_load(struct xa_state *);
627+
628+
/**
629+
* xas_reload() - Refetch an entry from the xarray.
630+
* @xas: XArray operation state.
631+
*
632+
* Use this function to check that a previously loaded entry still has
633+
* the same value. This is useful for the lockless pagecache lookup where
634+
* we walk the array with only the RCU lock to protect us, lock the page,
635+
* then check that the page hasn't moved since we looked it up.
636+
*
637+
* The caller guarantees that @xas is still valid. If it may be in an
638+
* error or restart state, call xas_load() instead.
639+
*
640+
* Return: The entry at this location in the xarray.
641+
*/
642+
static inline void *xas_reload(struct xa_state *xas)
643+
{
644+
struct xa_node *node = xas->xa_node;
645+
646+
if (node)
647+
return xa_entry(xas->xa, node, xas->xa_offset);
648+
return xa_head(xas->xa);
649+
}
650+
315651
#endif /* _LINUX_XARRAY_H */

‎lib/Kconfig.debug

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1813,6 +1813,9 @@ config TEST_BITFIELD
18131813
config TEST_UUID
18141814
tristate "Test functions located in the uuid module at runtime"
18151815

1816+
config TEST_XARRAY
1817+
tristate "Test the XArray code at runtime"
1818+
18161819
config TEST_OVERFLOW
18171820
tristate "Test check_*_overflow() functions at runtime"
18181821

0 commit comments

Comments
 (0)