BUG#27625172 Step 3

Mikael Ronström · Hery Ramilison · commit 56f78c36e90a · 2018-03-20T14:21:10.000+01:00
-------------------
Added a new test in testNodeRestart -n LCP_with_many_parts.
It requires running with table T17, I used 10000 rows
that requires about 100 MByte of DataMemory. It requires
that only one 1 LDM is used to work as planned.

(cherry picked from commit 4e5c98fe470d7a7e3114aa98f1f7b52960c5f24b)
diff --git a/storage/ndb/src/kernel/blocks/ERROR_codes.txt b/storage/ndb/src/kernel/blocks/ERROR_codes.txt
@@ -23,7 +23,7 @@ Next DBDICT 6223
 Next DBDIH 7249
 Next DBTC 8117
 Next CMVMI 9000
-Next BACKUP 10048
+Next BACKUP 10049
 Next PGMAN 11010
 Next DBTUX 12010
 Next SUMA 13049
@@ -620,6 +620,7 @@ Backup Stuff:
 10045: Crash insertion at FSAPPENDREF
 10046: DIH scan tab around Get next fragment
 10047: Delay start of scan for LCPs
+10048: Ensure that LCP runs with exactly 1 part
 
 11001: Send UTIL_SEQUENCE_REF (in master)
 
diff --git a/storage/ndb/src/kernel/blocks/backup/Backup.cpp b/storage/ndb/src/kernel/blocks/backup/Backup.cpp
@@ -12385,6 +12385,16 @@ Backup::calculate_number_of_parts(BackupRecordPtr ptr)
                      MAX(min_parts_rule3,
                      MAX(min_parts_rule4, min_parts_rule5)));
 
+  if (ERROR_INSERTED(10048) && min_parts_rule4 == 0)
+  {
+    /**
+     * We need this in test cases to ensure that we can create a situation
+     * with 1 part per LCP and having more than 980 parts and even close to
+     * 2048 LCPs to restore a LCP.
+     */
+    jam();
+    parts = 1;
+  }
 #ifdef DEBUG_LCP_STAT
   TablePtr debTabPtr;
   FragmentPtr fragPtr;
@@ -14156,6 +14166,11 @@ Backup::execEND_LCPREQ(Signal* signal)
   ptr.p->slaveState.setState(DEFINING);
   ptr.p->slaveState.setState(DEFINED);
 
+  if (ERROR_INSERTED(10048))
+  {
+    CLEAR_ERROR_INSERT_VALUE;
+  }
+
   DEB_LCP(("(%u)TAGE Send SYNC_EXTENT_PAGES_REQ", instance()));
   /**
    * As part of ending the LCP we need to ensure that the extent pages
diff --git a/storage/ndb/test/ndbapi/testNodeRestart.cpp b/storage/ndb/test/ndbapi/testNodeRestart.cpp
@@ -8714,6 +8714,139 @@ int runTestStartNode(NDBT_Context* ctx, NDBT_Step* step){
   return NDBT_OK;
 }
 
+/**
+ * In Partial LCP we need many LCPs to restore a checkpoint. The
+ * maximum number of LCPs we need in order to restore a checkpoint
+ * is 2048. This test uses error insert 10048 to ensure that each
+ * LCP only stores 1 part completely. This means that this test
+ * can generate checkpoints that have to write LCP control files
+ * consisting of close to 2048 parts and similarly to restore those.
+ * 
+ * The test loops for more than 2048 times to ensure that we come
+ * to a situation with a large number of parts in each LCP and in
+ * particular for the last one that we are to restore. The number
+ * 2058 is somewhat arbitrarily choosen to ensure this.
+ *
+ * The test case is hardcoded to make those special LCPs in node 2.
+ *
+ * Between each LCP we perform a random amount of updates to ensure
+ * that each part of this table will create a non-empty LCP. We
+ * insert a number of random LCPs that are empty as well to ensure
+ * that we generate empty LCPs correctly as well even if there are
+ * many parts in the LCP.
+ */
+int run_PLCP_many_parts(NDBT_Context *ctx, NDBT_Step *step)
+{
+  Ndb *pNdb = GETNDB(step);
+  int loops = 2108;
+  int result = NDBT_OK;
+  int records = ctx->getNumRecords();
+  HugoTransactions hugoTrans(*ctx->getTab());
+  NdbRestarter restarter;
+  int i = 0;
+  const Uint32 nodeCount = restarter.getNumDbNodes();
+  int nodeId = 2;
+  HugoOperations hugoOps(*ctx->getTab());
+  if (nodeCount < 2)
+  {
+    return NDBT_OK; /* Requires at least 2 nodes to run */
+  }
+  if (hugoTrans.loadTable(pNdb, records) != NDBT_OK)
+  {
+    g_err << "Failed to load table" << endl;
+    return NDBT_FAILED;
+  }
+
+  g_err << "Executing " << loops << " loops" << endl;
+  while(++i <= loops && result != NDBT_FAILED)
+  {
+    g_err << "Start loop " << i << endl;
+    ndbout << "Start an LCP" << endl;
+    {
+      if (restarter.insertErrorInNode(nodeId, 10048) != 0)
+      {
+        g_err << "ERROR: Error insert 10048 failed" << endl;
+        return NDBT_FAILED;
+      }
+      int val = DumpStateOrd::DihStartLcpImmediately;
+      if(restarter.dumpStateAllNodes(&val, 1) != 0)
+      {
+        g_err << "ERR: "<< step->getName()
+              << " failed on line " << __LINE__ << endl;
+        return NDBT_FAILED;
+      }
+    }
+    bool skip = false;
+    if ((i % 50) == 0)
+    {
+      skip = true;
+    }
+    Uint32 batch = 4;
+    Uint32 row;
+    if (!skip)
+    {
+      row = rand() % records;
+      if(row + batch > (Uint32)records)
+        batch = records - row;
+
+      if ((hugoOps.startTransaction(pNdb) != 0) ||
+          (hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0) ||
+          (hugoOps.execute_Commit(pNdb)) ||
+          (hugoOps.closeTransaction(pNdb)))
+      {
+        g_err << "Update failed" << endl;
+        //return NDBT_FAILED;
+      }
+    }
+    NdbSleep_SecSleep(1);
+    if (!skip)
+    {
+      row = rand() % records;
+      if(row + batch > (Uint32)records)
+        batch = records - row;
+      if ((hugoOps.startTransaction(pNdb) != 0) ||
+          (hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0) ||
+          (hugoOps.execute_Commit(pNdb)) ||
+          (hugoOps.closeTransaction(pNdb)))
+      {
+        g_err << "Update failed" << endl;
+        //return NDBT_FAILED;
+      }
+    }
+  }
+  /**
+   * Finally after creating a complex restore situation we test this
+   * by restarting node 2 to ensure that we can also recover the
+   * complex LCP setup.
+   */
+  ndbout << "Restart node 2" << endl;
+  if (restarter.restartOneDbNode(nodeId,
+                                 false, /* initial */
+                                 true,  /* nostart  */
+                                 false, /* abort */
+                                 false  /* force */) != 0)
+  {
+    g_err << "Restart failed" << endl;
+    return NDBT_FAILED;
+  }
+  ndbout << "Wait for NoStart state" << endl;
+  restarter.waitNodesNoStart(&nodeId, 1);
+  ndbout << "Start node" << endl;
+  if (restarter.startNodes(&nodeId, 1) != 0)
+  {
+    g_err << "Start failed" << endl;
+    return NDBT_FAILED;
+  }
+  ndbout << "Waiting for node to start" << endl;
+  if (restarter.waitNodesStarted(&nodeId, 1) != 0)
+  {
+    g_err << "Wait node start failed" << endl;
+    return NDBT_FAILED;
+  }
+  ndbout << "Test complete" << endl;
+  return NDBT_OK;
+}
+
 int run_PLCP_I1(NDBT_Context *ctx, NDBT_Step *step)
 {
   Ndb *pNdb = GETNDB(step);
@@ -9591,6 +9724,11 @@ TESTCASE("MultiCrashTest",
   STEP(runMultiCrashTest);
   FINALIZER(runClearTable);
 }
+TESTCASE("LCP_with_many_parts",
+         "Ensure that LCP has many parts")
+{
+  INITIALIZER(run_PLCP_many_parts);
+}
 TESTCASE("PLCP_I1",
          "Initial node restart while deleting rows")
 {
