block: partition: convert percpu ref

ming1 · axboe · commit 6c71013ecb7e · 2015-07-17T08:41:53.000-06:00
Percpu refcount is the perfect match for partition's case,
and the conversion is quite straight.

With the convertion, one pair of atomic inc/dec can be saved
for accounting block I/O, which is run in hot path of block I/O.

Signed-off-by: Ming Lei &lt;tom.leiming@gmail.com&gt;
Acked-by: Tejun Heo &lt;tj@kernel.org&gt;
Signed-off-by: Jens Axboe &lt;axboe@fb.com&gt;
diff --git a/block/genhd.c b/block/genhd.c
@@ -1284,7 +1284,11 @@ struct gendisk *alloc_disk_node(int minors, int node_id)
 		 * converted to make use of bd_mutex and sequence counters.
 		 */
 		seqcount_init(&disk->part0.nr_sects_seq);
-		hd_ref_init(&disk->part0);
+		if (hd_ref_init(&disk->part0)) {
+			hd_free_part(&disk->part0);
+			kfree(disk);
+			return NULL;
+		}
 
 		disk->minors = minors;
 		rand_initialize_disk(disk);
diff --git a/block/partition-generic.c b/block/partition-generic.c
@@ -232,8 +232,9 @@ static void delete_partition_rcu_cb(struct rcu_head *head)
 	put_device(part_to_dev(part));
 }
 
-void __delete_partition(struct hd_struct *part)
+void __delete_partition(struct percpu_ref *ref)
 {
+	struct hd_struct *part = container_of(ref, struct hd_struct, ref);
 	call_rcu(&part->rcu_head, delete_partition_rcu_cb);
 }
 
@@ -254,7 +255,7 @@ void delete_partition(struct gendisk *disk, int partno)
 	kobject_put(part->holder_dir);
 	device_del(part_to_dev(part));
 
-	hd_struct_put(part);
+	hd_struct_kill(part);
 }
 
 static ssize_t whole_disk_show(struct device *dev,
@@ -355,8 +356,8 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno,
 	if (!dev_get_uevent_suppress(ddev))
 		kobject_uevent(&pdev->kobj, KOBJ_ADD);
 
-	hd_ref_init(p);
-	return p;
+	if (!hd_ref_init(p))
+		return p;
 
 out_free_info:
 	free_part_info(p);
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
@@ -13,6 +13,7 @@
 #include <linux/kdev_t.h>
 #include <linux/rcupdate.h>
 #include <linux/slab.h>
+#include <linux/percpu-refcount.h>
 
 #ifdef CONFIG_BLOCK
 
@@ -124,7 +125,7 @@ struct hd_struct {
 #else
 	struct disk_stats dkstats;
 #endif
-	atomic_t ref;
+	struct percpu_ref ref;
 	struct rcu_head rcu_head;
 };
 
@@ -611,7 +612,7 @@ extern struct hd_struct * __must_check add_partition(struct gendisk *disk,
 						     sector_t len, int flags,
 						     struct partition_meta_info
 						       *info);
-extern void __delete_partition(struct hd_struct *);
+extern void __delete_partition(struct percpu_ref *);
 extern void delete_partition(struct gendisk *, int);
 extern void printk_all_partitions(void);
 
@@ -640,33 +641,39 @@ extern ssize_t part_fail_store(struct device *dev,
 			       const char *buf, size_t count);
 #endif /* CONFIG_FAIL_MAKE_REQUEST */
 
-static inline void hd_ref_init(struct hd_struct *part)
+static inline int hd_ref_init(struct hd_struct *part)
 {
-	atomic_set(&part->ref, 1);
-	smp_mb();
+	if (percpu_ref_init(&part->ref, __delete_partition, 0,
+				GFP_KERNEL))
+		return -ENOMEM;
+	return 0;
 }
 
 static inline void hd_struct_get(struct hd_struct *part)
 {
-	atomic_inc(&part->ref);
-	smp_mb__after_atomic();
+	percpu_ref_get(&part->ref);
 }
 
 static inline int hd_struct_try_get(struct hd_struct *part)
 {
-	return atomic_inc_not_zero(&part->ref);
+	return percpu_ref_tryget_live(&part->ref);
 }
 
 static inline void hd_struct_put(struct hd_struct *part)
 {
-	if (atomic_dec_and_test(&part->ref))
-		__delete_partition(part);
+	percpu_ref_put(&part->ref);
+}
+
+static inline void hd_struct_kill(struct hd_struct *part)
+{
+	percpu_ref_kill(&part->ref);
 }
 
 static inline void hd_free_part(struct hd_struct *part)
 {
 	free_part_stats(part);
 	free_part_info(part);
+	percpu_ref_exit(&part->ref);
 }
 
 /*

Original file line number	Diff line number	Diff line change
`@@ -232,8 +232,9 @@ static void delete_partition_rcu_cb(struct rcu_head *head)`
`232`	`232`	`put_device(part_to_dev(part));`
`233`	`233`	`}`
`234`	`234`
`235`		`-void __delete_partition(struct hd_struct *part)`
	`235`	`+void __delete_partition(struct percpu_ref *ref)`
`236`	`236`	`{`
	`237`	`+ struct hd_struct *part = container_of(ref, struct hd_struct, ref);`
`237`	`238`	`call_rcu(&part->rcu_head, delete_partition_rcu_cb);`
`238`	`239`	`}`
`239`	`240`
`@@ -254,7 +255,7 @@ void delete_partition(struct gendisk *disk, int partno)`
`254`	`255`	`kobject_put(part->holder_dir);`
`255`	`256`	`device_del(part_to_dev(part));`
`256`	`257`
`257`		`- hd_struct_put(part);`
	`258`	`+ hd_struct_kill(part);`
`258`	`259`	`}`
`259`	`260`
`260`	`261`	`static ssize_t whole_disk_show(struct device *dev,`
`@@ -355,8 +356,8 @@ struct hd_struct add_partition(struct gendisk disk, int partno,`
`355`	`356`	`if (!dev_get_uevent_suppress(ddev))`
`356`	`357`	`kobject_uevent(&pdev->kobj, KOBJ_ADD);`
`357`	`358`
`358`		`- hd_ref_init(p);`
`359`		`- return p;`
	`359`	`+ if (!hd_ref_init(p))`
	`360`	`+ return p;`
`360`	`361`
`361`	`362`	`out_free_info:`
`362`	`363`	`free_part_info(p);`