BUG#35949017 Schema dist setup lockup

blaudden · blaudden · commit 2fff6a3db31d · 2023-11-23T13:50:16.000+01:00
Problem:
Schema event handler hangs waiting for GSL to become available when
trying to setup util tables.

Analysis:
The schema event handler will try to setup the util tables when the
physical tables have been dropped from NDB or connection lost otherwise.
This setup check is intended to be done periodically every second but
since the first step is to acquire the GSL the check will lockup waiting
for the GSL. During this time the schema event handler and thus the
binlog thread will not be actively handling other events.

Solution:
Use try lock when attempting to acquire the GSL, thus causing another
setup check attempt to be made at a later time if GSl is not available.

This change has the effect that check setup is active during a slightly
longer then before, this means the util_tables_drop test has to be
modified to wait for all servers completing the setup before continuing
with next step in test. Previoulsy the test only waited for servers that
would be used for subsequent SQL queries.

Change-Id: Ie8e90b65781d94737eafc0e2ae0b6f3c18599f1d
diff --git a/mysql-test/suite/ndb_ddl/check_util_table_drop.inc b/mysql-test/suite/ndb_ddl/check_util_table_drop.inc
@@ -26,17 +26,15 @@ RESET MASTER;
 --echo # Drop mysql.$util_table_name from NDB
 --exec $NDB_DROP_TABLE -d mysql $util_table_name
 
---connection mysqld1
---echo # mysqld1: Wait until ready again...
---source include/ndb_not_readonly.inc
-
---connection mysqld3
---echo # mysqld3: Wait until ready again...
---source include/ndb_not_readonly.inc
-
---connection mysqld5
---echo # mysqld5: Wait until ready again...
---source include/ndb_not_readonly.inc
+# Wait for all servers connected again
+let $i = 1;
+while($i <= $NUM_MYSQLDS)
+{
+  --connection mysqld$i
+  --echo # mysqld$i: Wait until ready again...
+  --source include/ndb_not_readonly.inc
+  inc $i;
+}
 
 --connection mysqld5
 # NOTE! Neither drop of ndb_schema or ndb_schema_result should have a LOST_EVENT
diff --git a/mysql-test/suite/ndb_ddl/util_tables_drop.result b/mysql-test/suite/ndb_ddl/util_tables_drop.result
@@ -14,8 +14,11 @@ Dropping table ndb_schema...OK
 NDBT_ProgramExit: 0 - OK
 
 # mysqld1: Wait until ready again...
+# mysqld2: Wait until ready again...
 # mysqld3: Wait until ready again...
+# mysqld4: Wait until ready again...
 # mysqld5: Wait until ready again...
+# mysqld6: Wait until ready again...
 # Check if a LOST_EVENTS event was written to the binlog
 include/show_binlog_events.inc
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
@@ -44,8 +47,11 @@ Dropping table ndb_apply_status...OK
 NDBT_ProgramExit: 0 - OK
 
 # mysqld1: Wait until ready again...
+# mysqld2: Wait until ready again...
 # mysqld3: Wait until ready again...
+# mysqld4: Wait until ready again...
 # mysqld5: Wait until ready again...
+# mysqld6: Wait until ready again...
 # Check if a LOST_EVENTS event was written to the binlog
 include/show_binlog_events.inc
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
@@ -76,8 +82,11 @@ Dropping table ndb_schema_result...OK
 NDBT_ProgramExit: 0 - OK
 
 # mysqld1: Wait until ready again...
+# mysqld2: Wait until ready again...
 # mysqld3: Wait until ready again...
+# mysqld4: Wait until ready again...
 # mysqld5: Wait until ready again...
+# mysqld6: Wait until ready again...
 # Check if a LOST_EVENTS event was written to the binlog
 include/show_binlog_events.inc
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
diff --git a/storage/ndb/plugin/ha_ndbcluster_binlog.cc b/storage/ndb/plugin/ha_ndbcluster_binlog.cc
@@ -4266,7 +4266,7 @@ class Ndb_schema_event_handler {
 
     // Protect the setup with GSL(Global Schema Lock)
     Ndb_global_schema_lock_guard global_schema_lock_guard(m_thd);
-    if (global_schema_lock_guard.lock()) {
+    if (!global_schema_lock_guard.try_lock()) {
       ndb_log_info(" - failed to lock GSL");
       return true;
     }

Original file line number	Diff line number	Diff line change
`@@ -4266,7 +4266,7 @@ class Ndb_schema_event_handler {`
`4266`	`4266`
`4267`	`4267`	`// Protect the setup with GSL(Global Schema Lock)`
`4268`	`4268`	`Ndb_global_schema_lock_guard global_schema_lock_guard(m_thd);`
`4269`		`- if (global_schema_lock_guard.lock()) {`
	`4269`	`+ if (!global_schema_lock_guard.try_lock()) {`
`4270`	`4270`	`ndb_log_info(" - failed to lock GSL");`
`4271`	`4271`	`return true;`
`4272`	`4272`	`}`