Bug #32800020 EXPLORATORY - INVESTIGATE ODD NON-DATA DEADLOCK

sachinagarwal1111 · sachinagarwal1111 · commit be9954a5233b · 2021-05-13T22:46:01.000+05:30
Problem: The server terminates abnormally because of a deadlock between user thread (UPDATE) & purge thread. Analysis: 1. In 5.6, all undo logs reside either in system tablespace or in dedicated undo tablespaces. 2. In 5.7, new type of undo logs were introduced for temporary tables. One that is not redo logged and resides in the temp-tablespace. we reserved [1-32] rollback segment slots for temp-tables. (See [http://wl.mysql.oraclecorp.com/worklog/InnoDB-Sprint/?tid=6915]) 3. Upgrade from 5.6 ->5.7, server notices valid rsegs slots in range [1..32] and tries to move these rsegs from in slots 1..32 of `rseg_array[]` to `pending_purge_rseg_array[]`, so the slots can be used for non-redo rsegs (temp-table). 4. This is just an in-memory movement, as the relevant on-disk trx system page is not updated. So when the next time the server will start, will repeat [step-3]. 5. Till this point, there is no problem because all rsegs in `rseg_array[]` or in `pending_purge_rseg_array[]` are attached with unique <space_id, Page_no>. 6. Now when undo tablespace truncate is triggered: a. UNDO tablespace truncate collects all rsegs in a undo tablespace into truncate vector `m_rseg_for_trunc`. b. Scan over each rseg in `m_rseg_for_trunc` and ensure that it doesn't hold any active trx. c. If there is any active trx then stop truncation otherwise truncate undo tablespace file. d. Again scan over each rseg in `m_rseg_for_trunc` and reinitialize rollback segment and assign page start with number 3. 7. Here comes the problem. we forgot to reinitialize rollback segments in `trx_sys->pending_purge_rseg_array`. These rsegs are still attached with <space_id, Page_no>. So now two rollback segments are attched with same <Space_id, Page_no>. One rseg in `rseg_array[]` and the other in `pending_purge_rseg_array[]`. 8. And regs slots in trx system page are still have <space_id, Page_no> info for rsegs in `trx_sys->pending_purge_rseg_array` so even after restart this problem exist. [mentioned in step-7]. 9. In this situation, deadlock might occur between user thread & purge thread. when a. User thread (during trx_commit(), adding undo log to history list) has acquired x-latch over undo log page and waiting for x-latch over rseg page. b. Purge thread (during truncate undo log & history list for `pending_purge_rseg_array`) has acquired x-latch over rseg page and waiting for x-latch over undo log page. c. If user thread & purge thread are in race for same undo log page & rseg page then they can deadlock either other. Fix: During undo tablespace truncate, scan all rollback segments in `pending_purge_rseg_array[]` and free all rsegs, those exist in same undo tablespace. RB: 26376 Reviewed by : Jakub Lopuszanski <jakub.lopuszanski@oracle.com>
diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h
@@ -627,7 +627,7 @@ struct trx_sys_t {
 					transactions), protected by
 					rseg->mutex */
 
-	trx_rseg_t*	const pending_purge_rseg_array[TRX_SYS_N_RSEGS];
+	trx_rseg_t*	pending_purge_rseg_array[TRX_SYS_N_RSEGS];
 					/*!< Pointer array to rollback segments
 					between slot-1..slot-srv_tmp_undo_logs
 					that are now replaced by non-redo
diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc
@@ -310,11 +310,9 @@ trx_rseg_schedule_pending_purge(
 
 		ut_ad(found);
 
-		trx_rseg_t** rseg_array =
-			((trx_rseg_t**) trx_sys->pending_purge_rseg_array);
 		rseg = trx_rseg_mem_create(
 			slot, space, page_no, page_size,
-			purge_queue, rseg_array, mtr);
+			purge_queue, trx_sys->pending_purge_rseg_array, mtr);
 
 		ut_a(rseg->id == slot);
 	}
diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc
@@ -1208,15 +1208,14 @@ trx_sys_close(void)
 		}
 	}
 
-	rseg_array = ((trx_rseg_t**) trx_sys->pending_purge_rseg_array);
-
 	for (ulint i = 0; i < TRX_SYS_N_RSEGS; ++i) {
 		trx_rseg_t*	rseg;
 
 		rseg = trx_sys->pending_purge_rseg_array[i];
 
 		if (rseg != NULL) {
-			trx_rseg_mem_free(rseg, rseg_array);
+			trx_rseg_mem_free(rseg,
+				trx_sys->pending_purge_rseg_array);
 		}
 	}
 
diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc
@@ -2181,6 +2181,24 @@ trx_undo_truncate_tablespace(
 		rseg->last_trx_no = 0;
 		rseg->last_del_marks = FALSE;
 	}
+
+	/* During Upgrade, existing rsegs in range from slot-1....slot-32
+	were added into the array pending_purge_rseg_array[]. These rsegs also
+	reside in system or undo tablespace. */
+	trx_sysf_t* sys_header = trx_sysf_get(&mtr);
+	for (ulint i = 0; i < TRX_SYS_N_RSEGS; ++i) {
+		trx_rseg_t*	rseg = trx_sys->pending_purge_rseg_array[i];
+		if(rseg != NULL
+			&& rseg->space == undo_trunc->get_marked_space_id()) {
+			/* Reset the rollback segment slot in the trx
+			system header */
+			trx_sysf_rseg_set_page_no(
+				sys_header, rseg->id, FIL_NULL, &mtr);
+			/* Free a pending rollback segment instance in memory */
+			trx_rseg_mem_free(rseg,
+				trx_sys->pending_purge_rseg_array);
+		}
+	}
 	mtr_commit(&mtr);
 
 	return(success);

Original file line number	Diff line number	Diff line change
`@@ -1208,15 +1208,14 @@ trx_sys_close(void)`
`1208`	`1208`	`}`
`1209`	`1209`	`}`
`1210`	`1210`
`1211`		`- rseg_array = ((trx_rseg_t**) trx_sys->pending_purge_rseg_array);`
`1212`		`-`
`1213`	`1211`	`for (ulint i = 0; i < TRX_SYS_N_RSEGS; ++i) {`
`1214`	`1212`	`trx_rseg_t* rseg;`
`1215`	`1213`
`1216`	`1214`	`rseg = trx_sys->pending_purge_rseg_array[i];`
`1217`	`1215`
`1218`	`1216`	`if (rseg != NULL) {`
`1219`		`- trx_rseg_mem_free(rseg, rseg_array);`
	`1217`	`+ trx_rseg_mem_free(rseg,`
	`1218`	`+ trx_sys->pending_purge_rseg_array);`
`1220`	`1219`	`}`
`1221`	`1220`	`}`
`1222`	`1221`