Recovery in progress

knizhnik · knizhnik · commit e6faa29dc7fd · 2016-03-29T14:00:13.000+03:00
diff --git a/contrib/mmts/arbiter.c b/contrib/mmts/arbiter.c
@@ -376,7 +376,7 @@ static void MtmOpenConnections()
 	}
 	if (Mtm->nNodes < MtmNodes/2+1) { /* no quorum */
 		elog(WARNING, "Node is out of quorum: only %d nodes from %d are accssible", Mtm->nNodes, MtmNodes);
-		Mtm->status = MTM_IN_MINORITY;
+		MtmSwitchClusterMode(MTM_IN_MINORITY);
 	} else if (Mtm->status == MTM_INITIALIZATION) { 
 		MtmSwitchClusterMode(MTM_CONNECTED);
 	}
diff --git a/contrib/mmts/multimaster.c b/contrib/mmts/multimaster.c
@@ -973,7 +973,13 @@ static int64 MtmGetSlotLag(int nodeId)
  */
 bool MtmIsRecoveredNode(int nodeId)
 {
-	if (BIT_CHECK(Mtm->disabledNodeMask, nodeId-1)) { 
+	return BIT_CHECK(Mtm->disabledNodeMask, nodeId-1));
+}
+
+
+void MtmRecoveryPorgress(XLogRecPtr lsn)
+{
+	
 		Assert(MyWalSnd != NULL); /* This function is called by WAL-sender, so it should not be NULL */
 		if (!BIT_CHECK(Mtm->nodeLockerMask, nodeId-1)
 			&& MyWalSnd->sentPtr + MtmMinRecoveryLag > GetXLogInsertRecPtr()) 
@@ -1135,7 +1141,7 @@ bool MtmRefreshClusterStatus(bool nowait)
 		}
 	} else { 
 		elog(WARNING, "Clique %lx has no quorum", clique);
-		Mtm->status = MTM_IN_MINORITY;
+		MtmSwitchClusterMode(MTM_IN_MINORITY);
 	}
 	return true;
 }
@@ -1145,12 +1151,12 @@ void MtmCheckQuorum(void)
 	if (Mtm->nNodes < MtmNodes/2+1) {
 		if (Mtm->status == MTM_ONLINE) { /* out of quorum */
 			elog(WARNING, "Node is in minority: disabled mask %lx", Mtm->disabledNodeMask);
-			Mtm->status = MTM_IN_MINORITY;
+			MtmSwitchClusterMode(MTM_IN_MINORITY);
 		}
 	} else {
 		if (Mtm->status == MTM_IN_MINORITY) { 
 			elog(WARNING, "Node is in majority: dissbled mask %lx", Mtm->disabledNodeMask);
-			Mtm->status = MTM_ONLINE;
+			MtmSwitchClusterMode(MTM_ONLINE);
 		}
 	}
 }
@@ -1696,6 +1702,19 @@ void MtmDropNode(int nodeId, bool dropSlot)
 	}
 }
 
+static void 
+MtmReplicationStartupHook(struct PGLogicalStartupHookArgs* args)
+{
+	MtmLock(LW_EXCLUSIVE);
+	if (BIT_CHECK(Mtm->disabledNodeMask,  MtmReplicationNodeId-1)) {
+		elog(WARNING, "Recovery of node %d is completed: start normal replication", MtmReplicationNodeId); 
+		BIT_CLEAR(Mtm->disabledNodeMask,  MtmReplicationNodeId-1);
+		Mtm->nNodes += 1;
+		MtmCheckQuorum();
+	}
+	MtmUnlock();
+}
+
 static void 
 MtmReplicationShutdownHook(struct PGLogicalShutdownHookArgs* args)
 {
@@ -1715,6 +1734,7 @@ MtmReplicationTxnFilterHook(struct PGLogicalTxnFilterArgs* args)
 
 void MtmSetupReplicationHooks(struct PGLogicalHooks* hooks)
 {
+	hooks->startup_hook = MtmReplicationStartupHook;
 	hooks->shutdown_hook = MtmReplicationShutdownHook;
 	hooks->txn_filter_hook = MtmReplicationTxnFilterHook;
 }

Original file line number	Diff line number	Diff line change
`@@ -376,7 +376,7 @@ static void MtmOpenConnections()`
`376`	`376`	`}`
`377`	`377`	`if (Mtm->nNodes < MtmNodes/2+1) { /* no quorum */`
`378`	`378`	`elog(WARNING, "Node is out of quorum: only %d nodes from %d are accssible", Mtm->nNodes, MtmNodes);`
`379`		`- Mtm->status = MTM_IN_MINORITY;`
	`379`	`+ MtmSwitchClusterMode(MTM_IN_MINORITY);`
`380`	`380`	`} else if (Mtm->status == MTM_INITIALIZATION) {`
`381`	`381`	`MtmSwitchClusterMode(MTM_CONNECTED);`
`382`	`382`	`}`
Original file line number	Diff line number	Diff line change
`@@ -973,7 +973,13 @@ static int64 MtmGetSlotLag(int nodeId)`
`973`	`973`	`*/`
`974`	`974`	`bool MtmIsRecoveredNode(int nodeId)`
`975`	`975`	`{`
`976`		`- if (BIT_CHECK(Mtm->disabledNodeMask, nodeId-1)) {`
	`976`	`+ return BIT_CHECK(Mtm->disabledNodeMask, nodeId-1));`
	`977`	`+}`
	`978`	`+`
	`979`	`+`
	`980`	`+void MtmRecoveryPorgress(XLogRecPtr lsn)`
	`981`	`+{`
	`982`	`+`
`977`	`983`	`Assert(MyWalSnd != NULL); /* This function is called by WAL-sender, so it should not be NULL */`
`978`	`984`	`if (!BIT_CHECK(Mtm->nodeLockerMask, nodeId-1)`
`979`	`985`	`&& MyWalSnd->sentPtr + MtmMinRecoveryLag > GetXLogInsertRecPtr())`
`@@ -1135,7 +1141,7 @@ bool MtmRefreshClusterStatus(bool nowait)`
`1135`	`1141`	`}`
`1136`	`1142`	`} else {`
`1137`	`1143`	`elog(WARNING, "Clique %lx has no quorum", clique);`
`1138`		`- Mtm->status = MTM_IN_MINORITY;`
	`1144`	`+ MtmSwitchClusterMode(MTM_IN_MINORITY);`
`1139`	`1145`	`}`
`1140`	`1146`	`return true;`
`1141`	`1147`	`}`
`@@ -1145,12 +1151,12 @@ void MtmCheckQuorum(void)`
`1145`	`1151`	`if (Mtm->nNodes < MtmNodes/2+1) {`
`1146`	`1152`	`if (Mtm->status == MTM_ONLINE) { /* out of quorum */`
`1147`	`1153`	`elog(WARNING, "Node is in minority: disabled mask %lx", Mtm->disabledNodeMask);`
`1148`		`- Mtm->status = MTM_IN_MINORITY;`
	`1154`	`+ MtmSwitchClusterMode(MTM_IN_MINORITY);`
`1149`	`1155`	`}`
`1150`	`1156`	`} else {`
`1151`	`1157`	`if (Mtm->status == MTM_IN_MINORITY) {`
`1152`	`1158`	`elog(WARNING, "Node is in majority: dissbled mask %lx", Mtm->disabledNodeMask);`
`1153`		`- Mtm->status = MTM_ONLINE;`
	`1159`	`+ MtmSwitchClusterMode(MTM_ONLINE);`
`1154`	`1160`	`}`
`1155`	`1161`	`}`
`1156`	`1162`	`}`
`@@ -1696,6 +1702,19 @@ void MtmDropNode(int nodeId, bool dropSlot)`
`1696`	`1702`	`}`
`1697`	`1703`	`}`
`1698`	`1704`
	`1705`	`+static void`
	`1706`	`+MtmReplicationStartupHook(struct PGLogicalStartupHookArgs* args)`
	`1707`	`+{`
	`1708`	`+ MtmLock(LW_EXCLUSIVE);`
	`1709`	`+ if (BIT_CHECK(Mtm->disabledNodeMask, MtmReplicationNodeId-1)) {`
	`1710`	`+ elog(WARNING, "Recovery of node %d is completed: start normal replication", MtmReplicationNodeId);`
	`1711`	`+ BIT_CLEAR(Mtm->disabledNodeMask, MtmReplicationNodeId-1);`
	`1712`	`+ Mtm->nNodes += 1;`
	`1713`	`+ MtmCheckQuorum();`
	`1714`	`+ }`
	`1715`	`+ MtmUnlock();`
	`1716`	`+}`
	`1717`	`+`
`1699`	`1718`	`static void`
`1700`	`1719`	`MtmReplicationShutdownHook(struct PGLogicalShutdownHookArgs* args)`
`1701`	`1720`	`{`
`@@ -1715,6 +1734,7 @@ MtmReplicationTxnFilterHook(struct PGLogicalTxnFilterArgs* args)`
`1715`	`1734`
`1716`	`1735`	`void MtmSetupReplicationHooks(struct PGLogicalHooks* hooks)`
`1717`	`1736`	`{`
	`1737`	`+ hooks->startup_hook = MtmReplicationStartupHook;`
`1718`	`1738`	`hooks->shutdown_hook = MtmReplicationShutdownHook;`
`1719`	`1739`	`hooks->txn_filter_hook = MtmReplicationTxnFilterHook;`
`1720`	`1740`	`}`