md-cluster: release RESYNC lock after the last resync message

GuoqingJiang · shligit · commit 41a950411265 · 2018-08-31T17:38:10.000-07:00
All the RESYNC messages are sent with resync lock held, the only
exception is resync_finish which releases resync_lockres before
send the last resync message, this should be changed as well.
Otherwise, we can see deadlock issue as follows:

clustermd2-gqjiang2:~ # cat /proc/mdstat
Personalities : [raid10] [raid1]
md0 : active raid1 sdg[0] sdf[1]
      134144 blocks super 1.2 [2/2] [UU]
      [===================&gt;.]  resync = 99.6% (134144/134144) finish=0.0min speed=26K/sec
      bitmap: 1/1 pages [4KB], 65536KB chunk

unused devices: &lt;none&gt;
clustermd2-gqjiang2:~ # ps aux|grep md|grep D
root     20497  0.0  0.0      0     0 ?        D    16:00   0:00 [md0_raid1]
clustermd2-gqjiang2:~ # cat /proc/20497/stack
[&lt;ffffffffc05ff51e&gt;] dlm_lock_sync+0x8e/0xc0 [md_cluster]
[&lt;ffffffffc05ff7e8&gt;] __sendmsg+0x98/0x130 [md_cluster]
[&lt;ffffffffc05ff900&gt;] sendmsg+0x20/0x30 [md_cluster]
[&lt;ffffffffc05ffc35&gt;] resync_info_update+0xb5/0xc0 [md_cluster]
[&lt;ffffffffc0593e84&gt;] md_reap_sync_thread+0x134/0x170 [md_mod]
[&lt;ffffffffc059514c&gt;] md_check_recovery+0x28c/0x510 [md_mod]
[&lt;ffffffffc060c882&gt;] raid1d+0x42/0x800 [raid1]
[&lt;ffffffffc058ab61&gt;] md_thread+0x121/0x150 [md_mod]
[&lt;ffffffff9a0a5b3f&gt;] kthread+0xff/0x140
[&lt;ffffffff9a800235&gt;] ret_from_fork+0x35/0x40
[&lt;ffffffffffffffff&gt;] 0xffffffffffffffff

clustermd-gqjiang1:~ # ps aux|grep md|grep D
root     20531  0.0  0.0      0     0 ?        D    16:00   0:00 [md0_raid1]
root     20537  0.0  0.0      0     0 ?        D    16:00   0:00 [md0_cluster_rec]
root     20676  0.0  0.0      0     0 ?        D    16:01   0:00 [md0_resync]
clustermd-gqjiang1:~ # cat /proc/mdstat
Personalities : [raid10] [raid1]
md0 : active raid1 sdf[1] sdg[0]
      134144 blocks super 1.2 [2/2] [UU]
      [===================&gt;.]  resync = 97.3% (131072/134144) finish=8076.8min speed=0K/sec
      bitmap: 1/1 pages [4KB], 65536KB chunk

unused devices: &lt;none&gt;
clustermd-gqjiang1:~ # cat /proc/20531/stack
[&lt;ffffffffc080974d&gt;] metadata_update_start+0xcd/0xd0 [md_cluster]
[&lt;ffffffffc079c897&gt;] md_update_sb.part.61+0x97/0x820 [md_mod]
[&lt;ffffffffc079f15b&gt;] md_check_recovery+0x29b/0x510 [md_mod]
[&lt;ffffffffc0816882&gt;] raid1d+0x42/0x800 [raid1]
[&lt;ffffffffc0794b61&gt;] md_thread+0x121/0x150 [md_mod]
[&lt;ffffffff9e0a5b3f&gt;] kthread+0xff/0x140
[&lt;ffffffff9e800235&gt;] ret_from_fork+0x35/0x40
[&lt;ffffffffffffffff&gt;] 0xffffffffffffffff
clustermd-gqjiang1:~ # cat /proc/20537/stack
[&lt;ffffffffc0813222&gt;] freeze_array+0xf2/0x140 [raid1]
[&lt;ffffffffc080a56e&gt;] recv_daemon+0x41e/0x580 [md_cluster]
[&lt;ffffffffc0794b61&gt;] md_thread+0x121/0x150 [md_mod]
[&lt;ffffffff9e0a5b3f&gt;] kthread+0xff/0x140
[&lt;ffffffff9e800235&gt;] ret_from_fork+0x35/0x40
[&lt;ffffffffffffffff&gt;] 0xffffffffffffffff
clustermd-gqjiang1:~ # cat /proc/20676/stack
[&lt;ffffffffc080951e&gt;] dlm_lock_sync+0x8e/0xc0 [md_cluster]
[&lt;ffffffffc080957f&gt;] lock_token+0x2f/0xa0 [md_cluster]
[&lt;ffffffffc0809622&gt;] lock_comm+0x32/0x90 [md_cluster]
[&lt;ffffffffc08098f5&gt;] sendmsg+0x15/0x30 [md_cluster]
[&lt;ffffffffc0809c0a&gt;] resync_info_update+0x8a/0xc0 [md_cluster]
[&lt;ffffffffc08130ba&gt;] raid1_sync_request+0xa9a/0xb10 [raid1]
[&lt;ffffffffc079b8ea&gt;] md_do_sync+0xbaa/0xf90 [md_mod]
[&lt;ffffffffc0794b61&gt;] md_thread+0x121/0x150 [md_mod]
[&lt;ffffffff9e0a5b3f&gt;] kthread+0xff/0x140
[&lt;ffffffff9e800235&gt;] ret_from_fork+0x35/0x40
[&lt;ffffffffffffffff&gt;] 0xffffffffffffffff

Reviewed-by: NeilBrown &lt;neilb@suse.com&gt;
Signed-off-by: Guoqing Jiang &lt;gqjiang@suse.com&gt;
Signed-off-by: Shaohua Li &lt;shli@fb.com&gt;
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
@@ -1276,18 +1276,18 @@ static int resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi)
 static int resync_finish(struct mddev *mddev)
 {
 	struct md_cluster_info *cinfo = mddev->cluster_info;
+	int ret = 0;
 
 	clear_bit(MD_RESYNCING_REMOTE, &mddev->recovery);
-	dlm_unlock_sync(cinfo->resync_lockres);
 
 	/*
 	 * If resync thread is interrupted so we can't say resync is finished,
 	 * another node will launch resync thread to continue.
 	 */
-	if (test_bit(MD_CLOSING, &mddev->flags))
-		return 0;
-	else
-		return resync_info_update(mddev, 0, 0);
+	if (!test_bit(MD_CLOSING, &mddev->flags))
+		ret = resync_info_update(mddev, 0, 0);
+	dlm_unlock_sync(cinfo->resync_lockres);
+	return ret;
 }
 
 static int area_resyncing(struct mddev *mddev, int direction,
