Obtain timeline ID with control file and not XLOG system function

Michael Paquier · Michael Paquier · commit 308d00b80c4f · 2013-12-13T03:55:39.000+09:00
The system function used up to now was pg_xlogfile_name_offset, which
cannot be used on a node in recovery, and it was the only way present
to fetch the timeline ID of a backup, either incremental or full. So
instead scan the control file of server and fetch the timeline from
that. This also removes the restriction on which a backup could not
be taken on a standby node. The next step being to have the possibility
to take backups from streams.
diff --git a/Makefile b/Makefile
@@ -5,6 +5,7 @@ SRCS = \
 	data.c \
 	delete.c \
 	dir.c \
+	fetch.c \
 	init.c \
 	parray.c \
 	pg_rman.c \
diff --git a/backup.c b/backup.c
@@ -17,6 +17,7 @@
 #include <dirent.h>
 #include <time.h>
 
+#include "catalog/pg_control.h"
 #include "libpq/pqsignal.h"
 #include "pgut/pgut-port.h"
 
@@ -48,7 +49,7 @@ static void confirm_block_size(const char *name, int blcksz);
 static void pg_start_backup(const char *label, bool smooth, pgBackup *backup);
 static void pg_stop_backup(pgBackup *backup);
 static void pg_switch_xlog(pgBackup *backup);
-static void get_lsn(PGresult *res, TimeLineID *timeline, XLogRecPtr *lsn);
+static void get_lsn(PGresult *res, XLogRecPtr *lsn);
 static void get_xid(PGresult *res, uint32 *xid);
 static bool execute_restartpoint(pgBackupOption bkupopt);
 
@@ -60,6 +61,7 @@ static bool dirExists(const char *path);
 static void add_files(parray *files, const char *root, bool add_root, bool is_pgdata);
 static int strCompare(const void *str1, const void *str2);
 static void create_file_list(parray *files, const char *root, const char *prefix, bool is_append);
+static TimeLineID get_current_timeline(void);
 
 /*
  * Take a backup of database.
@@ -108,6 +110,13 @@ do_backup_database(parray *backup_list, pgBackupOption bkupopt)
 	current.total_data_bytes = 0;
 	current.read_data_bytes = 0;
 
+	/*
+	 * Obtain current timeline by scanning control file, theh LSN
+	 * obtained at output of pg_start_backup or pg_stop_backup does
+	 * not contain this information.
+	 */
+	current.tli = get_current_timeline();
+
 	/* notify start of backup to PostgreSQL server */
 	time2iso(label, lengthof(label), current.start_time);
 	strncat(label, " with pg_rman", lengthof(label));
@@ -492,8 +501,8 @@ do_backup_arclog(parray *backup_list)
 		pg_switch_xlog(&current);
 
 	/*
-	 * To take incremental backup, the file list of the last completed database
-	 * backup is needed.
+	 * To take incremental backup, the file list of the last completed
+	 * database backup is needed.
 	 */
 	prev_backup = catalog_get_last_arclog_backup(backup_list);
 	if (verbose && prev_backup == NULL)
@@ -985,10 +994,10 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup)
 
 	/* 2nd argument is 'fast'*/
 	params[1] = smooth ? "false" : "true";
-	res = execute("SELECT * from pg_xlogfile_name_offset(pg_start_backup($1, $2))", 2, params);
+	res = execute("SELECT pg_start_backup($1, $2)", 2, params);
 
 	if (backup != NULL)
-		get_lsn(res, &backup->tli, &backup->start_lsn);
+		get_lsn(res, &backup->start_lsn);
 	PQclear(res);
 	disconnect();
 }
@@ -998,22 +1007,42 @@ wait_for_archive(pgBackup *backup, const char *sql)
 {
 	PGresult	   *res;
 	char			ready_path[MAXPGPATH];
+	char			file_name[MAXFNAMELEN];
 	int				try_count;
+	XLogRecPtr		lsn;
+	TimeLineID		tli;
 
 	reconnect();
 	res = execute(sql, 0, NULL);
+
+	/* Get LSN from execution result */
+	get_lsn(res, &lsn);
+
+	/*
+	 * Enforce TLI obtention if backup is not present as this code
+	 * path can be taken as a callback at exit.
+	 */
+	if (backup != NULL)
+		tli = backup->tli;
+	else
+		tli = get_current_timeline();
+
+	/* Fill in fields if backup exists */
 	if (backup != NULL)
 	{
-		get_lsn(res, &backup->tli, &backup->stop_lsn);
+		backup->stop_lsn = lsn;
 		elog(LOG, _("%s(): tli=%X lsn=%X/%08X"),
 			 __FUNCTION__, backup->tli,
 			 (uint32) (backup->stop_lsn >> 32),
 			 (uint32) backup->stop_lsn);
 	}
 
-	/* get filename from the result of pg_xlogfile_name_offset() */
+	/* As well as WAL file name */
+	XLogFileName(file_name, tli, lsn);
+
 	snprintf(ready_path, lengthof(ready_path),
-		"%s/pg_xlog/archive_status/%s.ready", pgdata, PQgetvalue(res, 0, 0));
+		"%s/pg_xlog/archive_status/%s.ready", pgdata,
+			 file_name);
 	elog(LOG, "%s() wait for %s", __FUNCTION__, ready_path);
 
 	PQclear(res);
@@ -1049,32 +1078,41 @@ static void
 pg_stop_backup(pgBackup *backup)
 {
 	wait_for_archive(backup,
-		"SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup())");
+		"SELECT * FROM pg_stop_backup()");
 }
 
 /*
- * Force switch to a new transaction log file and update backup->tli.
+ * Force switch to a new transaction log file
  */
 static void
 pg_switch_xlog(pgBackup *backup)
 {
 	wait_for_archive(backup,
-		"SELECT * FROM pg_xlogfile_name_offset(pg_switch_xlog())");
+		"SELECT * FROM pg_switch_xlog())");
 }
 
 /*
- * Get TimeLineID and LSN from result of pg_xlogfile_name_offset().
+ * Get LSN from result of pg_start_backup() or pg_stop_backup().
  */
 static void
-get_lsn(PGresult *res, TimeLineID *timeline, XLogRecPtr *lsn)
+get_lsn(PGresult *res, XLogRecPtr *lsn)
 {
-	if (res == NULL || PQntuples(res) != 1 || PQnfields(res) != 2)
+	uint32	xlogid;
+	uint32	xrecoff;
+
+	if (res == NULL || PQntuples(res) != 1 || PQnfields(res) != 1)
 		elog(ERROR_PG_COMMAND,
-			_("result of pg_xlogfile_name_offset() is invalid: %s"),
+			_("result of backup command is invalid: %s"),
 			PQerrorMessage(connection));
 
-	/* Extract timeline and LSN from result of pg_stop_backup() */
-	XLogFromFileName(PQgetvalue(res, 0, 0), timeline, lsn);
+	/*
+	 * Extract timeline and LSN from results of pg_stop_backup()
+	 * and friends.
+	 */
+	XLogDataFromLSN(PQgetvalue(res, 0, 0), &xlogid, &xrecoff);
+
+	/* Calculate LSN */
+	*lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff;
 }
 
 /*
@@ -1598,3 +1636,27 @@ create_file_list(parray *files, const char *root, const char *prefix, bool is_ap
 		fclose(fp);
 	}
 }
+
+/*
+ * Scan control file of given cluster at obtain the current timeline
+ * since last checkpoint that occurred on it.
+ */
+static TimeLineID
+get_current_timeline(void)
+{
+	char	   *buffer;
+	size_t		size;
+	ControlFileData control_file;
+
+	/* First fetch file... */
+	buffer = slurpFile(pgdata, "global/pg_control", &size);
+
+	/* .. Then interpret it */
+    if (size != PG_CONTROL_SIZE)
+		elog(ERROR_CORRUPTED, "unexpected control file size %d, expected %d\n",
+			 (int) size, PG_CONTROL_SIZE);
+	memcpy(&control_file, buffer, sizeof(ControlFileData));
+
+	/* Finally return the timeline wanted */
+	return control_file.checkPointCopy.ThisTimeLineID;
+}
diff --git a/fetch.c b/fetch.c
@@ -0,0 +1,69 @@
+/*-------------------------------------------------------------------------
+ *
+ * fetch.c
+ *    Functions for fetching files from PostgreSQL data directory
+ *
+ * Portions Copyright (c) 1996-2012, PostgreSQL Global Development Group
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "catalog/catalog.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "pg_rman.h"
+
+/*
+ * Read a file into memory. The file to be read is <datadir>/<path>.
+ * The file contents are returned in a malloc'd buffer, and *filesize
+ * is set to the length of the file.
+ *
+ * The returned buffer is always zero-terminated; the size of the returned
+ * buffer is actually *filesize + 1. That's handy when reading a text file.
+ * This function can be used to read binary files as well, you can just
+ * ignore the zero-terminator in that case.
+ *
+ */
+char *
+slurpFile(const char *datadir, const char *path, size_t *filesize)
+{
+	int		 fd;
+	char	   *buffer;
+	struct stat statbuf;
+	char		fullpath[MAXPGPATH];
+	int		 len;
+	snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path);
+
+	if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1)
+		elog(ERROR_CORRUPTED, _("could not open file \"%s\" for reading: %s\n"),
+				fullpath, strerror(errno));
+
+	if (fstat(fd, &statbuf) < 0)
+		elog(ERROR_CORRUPTED, _("could not open file \"%s\" for reading: %s\n"),
+			 fullpath, strerror(errno));
+
+	len = statbuf.st_size;
+
+	buffer = pg_malloc(len + 1);
+
+	if (read(fd, buffer, len) != len)
+		elog(ERROR_CORRUPTED, _("could not read file \"%s\": %s\n"),
+			 fullpath, strerror(errno));
+
+	close(fd);
+
+	/* Zero-terminate the buffer. */
+	buffer[len] = '\0';
+
+	if (filesize)
+		*filesize = len;
+	return buffer;
+}
diff --git a/pg_rman.h b/pg_rman.h
@@ -214,6 +214,13 @@ typedef enum CompressionMode
 #define JoinPathEnd(str, prefix) \
 	((strlen(str) <= strlen(prefix)) ? "" : str + strlen(prefix) + 1)
 
+/*
+ * Return timeline, xlog ID and record offset from an LSN of the type
+ * 0/B000188, usual result from pg_stop_backup() and friends.
+ */
+#define XLogDataFromLSN(data, xlogid, xrecoff)		\
+	sscanf(data, "%X/%X", xlogid, xrecoff)
+
 /* path configuration */
 extern char *backup_path;
 extern char *pgdata;
@@ -253,6 +260,11 @@ extern int do_show(pgBackupRange *range, bool show_all);
 extern int do_delete(pgBackupRange *range, bool force);
 extern void pgBackupDelete(int keep_generations, int keep_days);
 
+/* in fetch.c */
+extern char *slurpFile(const char *datadir,
+					   const char *path,
+					   size_t *filesize);
+
 /* in validate.c */
 extern int do_validate(pgBackupRange *range);
 extern void pgBackupValidate(pgBackup *backup, bool size_only, bool for_get_timeline, bool with_database);
