rculist: Use WRITE_ONCE() when deleting from reader-visible list

paulmck · paulmck · commit 7f5f873c6a07 · 2015-10-06T11:16:42.000-07:00
The various RCU list-deletion macros (list_del_rcu(),
hlist_del_init_rcu(), hlist_del_rcu(), hlist_bl_del_init_rcu(),
hlist_bl_del_rcu(), hlist_nulls_del_init_rcu(), and hlist_nulls_del_rcu())
do plain stores into the -&gt;next pointer of the preceding list elemment.
Unfortunately, the compiler is within its rights to (for example) use
byte-at-a-time writes to update the pointer, which would fatally confuse
concurrent readers.  This patch therefore adds the needed WRITE_ONCE()
macros.

KernelThreadSanitizer (KTSAN) reported the __hlist_del() issue, which
is a problem when __hlist_del() is invoked by hlist_del_rcu().

Reported-by: Dmitry Vyukov &lt;dvyukov@google.com&gt;
Signed-off-by: Paul E. McKenney &lt;paulmck@linux.vnet.ibm.com&gt;
Reviewed-by: Josh Triplett &lt;josh@joshtriplett.org&gt;
diff --git a/include/linux/list.h b/include/linux/list.h
@@ -87,7 +87,7 @@ static inline void list_add_tail(struct list_head *new, struct list_head *head)
 static inline void __list_del(struct list_head * prev, struct list_head * next)
 {
 	next->prev = prev;
-	prev->next = next;
+	WRITE_ONCE(prev->next, next);
 }
 
 /**
@@ -615,7 +615,8 @@ static inline void __hlist_del(struct hlist_node *n)
 {
 	struct hlist_node *next = n->next;
 	struct hlist_node **pprev = n->pprev;
-	*pprev = next;
+
+	WRITE_ONCE(*pprev, next);
 	if (next)
 		next->pprev = pprev;
 }
diff --git a/include/linux/list_bl.h b/include/linux/list_bl.h
@@ -93,9 +93,10 @@ static inline void __hlist_bl_del(struct hlist_bl_node *n)
 	LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK);
 
 	/* pprev may be `first`, so be careful not to lose the lock bit */
-	*pprev = (struct hlist_bl_node *)
+	WRITE_ONCE(*pprev,
+		   (struct hlist_bl_node *)
 			((unsigned long)next |
-			 ((unsigned long)*pprev & LIST_BL_LOCKMASK));
+			 ((unsigned long)*pprev & LIST_BL_LOCKMASK)));
 	if (next)
 		next->pprev = pprev;
 }
diff --git a/include/linux/list_nulls.h b/include/linux/list_nulls.h
@@ -76,7 +76,8 @@ static inline void __hlist_nulls_del(struct hlist_nulls_node *n)
 {
 	struct hlist_nulls_node *next = n->next;
 	struct hlist_nulls_node **pprev = n->pprev;
-	*pprev = next;
+
+	WRITE_ONCE(*pprev, next);
 	if (!is_a_nulls(next))
 		next->pprev = pprev;
 }

Original file line number	Diff line number	Diff line change
`@@ -87,7 +87,7 @@ static inline void list_add_tail(struct list_head new, struct list_head head)`
`87`	`87`	`static inline void __list_del(struct list_head * prev, struct list_head * next)`
`88`	`88`	`{`
`89`	`89`	`next->prev = prev;`
`90`		`- prev->next = next;`
	`90`	`+ WRITE_ONCE(prev->next, next);`
`91`	`91`	`}`
`92`	`92`
`93`	`93`	`/**`
`@@ -615,7 +615,8 @@ static inline void __hlist_del(struct hlist_node *n)`
`615`	`615`	`{`
`616`	`616`	`struct hlist_node *next = n->next;`
`617`	`617`	`struct hlist_node **pprev = n->pprev;`
`618`		`- *pprev = next;`
	`618`	`+`
	`619`	`+ WRITE_ONCE(*pprev, next);`
`619`	`620`	`if (next)`
`620`	`621`	`next->pprev = pprev;`
`621`	`622`	`}`
Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,8 @@ static inline void __hlist_nulls_del(struct hlist_nulls_node *n)`
`76`	`76`	`{`
`77`	`77`	`struct hlist_nulls_node *next = n->next;`
`78`	`78`	`struct hlist_nulls_node **pprev = n->pprev;`
`79`		`- *pprev = next;`
	`79`	`+`
	`80`	`+ WRITE_ONCE(*pprev, next);`
`80`	`81`	`if (!is_a_nulls(next))`
`81`	`82`	`next->pprev = pprev;`
`82`	`83`	`}`