Skip to content

Commit cf131fa

Browse files
nathan-bossartnathan-bossart
committed
initdb: Add --no-sync-data-files.
This new option instructs initdb to skip synchronizing any files in database directories, the database directories themselves, and the tablespace directories, i.e., everything in the base/ subdirectory and any other tablespace directories. Other files, such as those in pg_wal/ and pg_xact/, will still be synchronized unless --no-sync is also specified. --no-sync-data-files is primarily intended for internal use by tools that separately ensure the skipped files are synchronized to disk. A follow-up commit will use this to help optimize pg_upgrade's file transfer step. The --sync-method=fsync implementation of this option makes use of a new exclude_dir parameter for walkdir(). When not NULL, exclude_dir specifies a directory to skip processing. The --sync-method=syncfs implementation of this option just skips synchronizing the non-default tablespace directories. This means that initdb will still synchronize some or all of the database files, but there's not much we can do about that. Discussion: https://postgr.es/m/Zyvop-LxLXBLrZil%40nathan
1 parent 650ab8a commit cf131fa

File tree

9 files changed

+96
-37
lines changed

9 files changed

+96
-37
lines changed

‎doc/src/sgml/ref/initdb.sgml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,33 @@ PostgreSQL documentation
527527
</listitem>
528528
</varlistentry>
529529

530+
<varlistentry id="app-initdb-option-no-sync-data-files">
531+
<term><option>--no-sync-data-files</option></term>
532+
<listitem>
533+
<para>
534+
By default, <command>initdb</command> safely writes all database files
535+
to disk. This option instructs <command>initdb</command> to skip
536+
synchronizing all files in the individual database directories, the
537+
database directories themselves, and the tablespace directories, i.e.,
538+
everything in the <filename>base</filename> subdirectory and any other
539+
tablespace directories. Other files, such as those in
540+
<literal>pg_wal</literal> and <literal>pg_xact</literal>, will still be
541+
synchronized unless the <option>--no-sync</option> option is also
542+
specified.
543+
</para>
544+
<para>
545+
Note that if <option>--no-sync-data-files</option> is used in
546+
conjuction with <option>--sync-method=syncfs</option>, some or all of
547+
the aforementioned files and directories will be synchronized because
548+
<literal>syncfs</literal> processes entire file systems.
549+
</para>
550+
<para>
551+
This option is primarily intended for internal use by tools that
552+
separately ensure the skipped files are synchronized to disk.
553+
</para>
554+
</listitem>
555+
</varlistentry>
556+
530557
<varlistentry id="app-initdb-option-no-instructions">
531558
<term><option>--no-instructions</option></term>
532559
<listitem>

‎src/bin/initdb/initdb.c

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ static bool data_checksums = true;
168168
static char *xlog_dir = NULL;
169169
static int wal_segment_size_mb = (DEFAULT_XLOG_SEG_SIZE) / (1024 * 1024);
170170
static DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
171+
static bool sync_data_files = true;
171172

172173

173174
/* internal vars */
@@ -2566,6 +2567,7 @@ usage(const char *progname)
25662567
printf(_(" -L DIRECTORY where to find the input files\n"));
25672568
printf(_(" -n, --no-clean do not clean up after errors\n"));
25682569
printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
2570+
printf(_(" --no-sync-data-files do not sync files within database directories\n"));
25692571
printf(_(" --no-instructions do not print instructions for next steps\n"));
25702572
printf(_(" -s, --show show internal settings, then exit\n"));
25712573
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
@@ -3208,6 +3210,7 @@ main(int argc, char *argv[])
32083210
{"icu-rules", required_argument, NULL, 18},
32093211
{"sync-method", required_argument, NULL, 19},
32103212
{"no-data-checksums", no_argument, NULL, 20},
3213+
{"no-sync-data-files", no_argument, NULL, 21},
32113214
{NULL, 0, NULL, 0}
32123215
};
32133216

@@ -3402,6 +3405,9 @@ main(int argc, char *argv[])
34023405
case 20:
34033406
data_checksums = false;
34043407
break;
3408+
case 21:
3409+
sync_data_files = false;
3410+
break;
34053411
default:
34063412
/* getopt_long already emitted a complaint */
34073413
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -3453,7 +3459,7 @@ main(int argc, char *argv[])
34533459

34543460
fputs(_("syncing data to disk ... "), stdout);
34553461
fflush(stdout);
3456-
sync_pgdata(pg_data, PG_VERSION_NUM, sync_method);
3462+
sync_pgdata(pg_data, PG_VERSION_NUM, sync_method, sync_data_files);
34573463
check_ok();
34583464
return 0;
34593465
}
@@ -3516,7 +3522,7 @@ main(int argc, char *argv[])
35163522
{
35173523
fputs(_("syncing data to disk ... "), stdout);
35183524
fflush(stdout);
3519-
sync_pgdata(pg_data, PG_VERSION_NUM, sync_method);
3525+
sync_pgdata(pg_data, PG_VERSION_NUM, sync_method, sync_data_files);
35203526
check_ok();
35213527
}
35223528
else

‎src/bin/initdb/t/001_initdb.pl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
'checksums are enabled in control file');
7777

7878
command_ok([ 'initdb', '--sync-only', $datadir ], 'sync only');
79+
command_ok([ 'initdb', '--sync-only', '--no-sync-data-files', $datadir ], '--no-sync-data-files');
7980
command_fails([ 'initdb', $datadir ], 'existing data directory');
8081

8182
if ($supports_syncfs)

‎src/bin/pg_basebackup/pg_basebackup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2310,7 +2310,7 @@ BaseBackup(char *compression_algorithm, char *compression_detail,
23102310
}
23112311
else
23122312
{
2313-
(void) sync_pgdata(basedir, serverVersion, sync_method);
2313+
(void) sync_pgdata(basedir, serverVersion, sync_method, true);
23142314
}
23152315
}
23162316

‎src/bin/pg_checksums/pg_checksums.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -633,7 +633,7 @@ main(int argc, char *argv[])
633633
if (do_sync)
634634
{
635635
pg_log_info("syncing data directory");
636-
sync_pgdata(DataDir, PG_VERSION_NUM, sync_method);
636+
sync_pgdata(DataDir, PG_VERSION_NUM, sync_method, true);
637637
}
638638

639639
pg_log_info("updating control file");

‎src/bin/pg_combinebackup/pg_combinebackup.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,7 @@ main(int argc, char *argv[])
424424
else
425425
{
426426
pg_log_debug("recursively fsyncing \"%s\"", opt.output);
427-
sync_pgdata(opt.output, version * 10000, opt.sync_method);
427+
sync_pgdata(opt.output, version * 10000, opt.sync_method, true);
428428
}
429429
}
430430

‎src/bin/pg_rewind/file_ops.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,7 @@ sync_target_dir(void)
296296
if (!do_sync || dry_run)
297297
return;
298298

299-
sync_pgdata(datadir_target, PG_VERSION_NUM, sync_method);
299+
sync_pgdata(datadir_target, PG_VERSION_NUM, sync_method, true);
300300
}
301301

302302

‎src/common/file_utils.c

Lines changed: 55 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ static int pre_sync_fname(const char *fname, bool isdir);
5050
#endif
5151
static void walkdir(const char *path,
5252
int (*action) (const char *fname, bool isdir),
53-
bool process_symlinks);
53+
bool process_symlinks,
54+
const char *exclude_dir);
5455

5556
#ifdef HAVE_SYNCFS
5657

@@ -93,11 +94,15 @@ do_syncfs(const char *path)
9394
* syncing, and might not have privileges to write at all.
9495
*
9596
* serverVersion indicates the version of the server to be sync'd.
97+
*
98+
* If sync_data_files is false, this function skips syncing "base/" and any
99+
* other tablespace directories.
96100
*/
97101
void
98102
sync_pgdata(const char *pg_data,
99103
int serverVersion,
100-
DataDirSyncMethod sync_method)
104+
DataDirSyncMethod sync_method,
105+
bool sync_data_files)
101106
{
102107
bool xlog_is_symlink;
103108
char pg_wal[MAXPGPATH];
@@ -147,30 +152,33 @@ sync_pgdata(const char *pg_data,
147152
do_syncfs(pg_data);
148153

149154
/* If any tablespaces are configured, sync each of those. */
150-
dir = opendir(pg_tblspc);
151-
if (dir == NULL)
152-
pg_log_error("could not open directory \"%s\": %m",
153-
pg_tblspc);
154-
else
155+
if (sync_data_files)
155156
{
156-
while (errno = 0, (de = readdir(dir)) != NULL)
157+
dir = opendir(pg_tblspc);
158+
if (dir == NULL)
159+
pg_log_error("could not open directory \"%s\": %m",
160+
pg_tblspc);
161+
else
157162
{
158-
char subpath[MAXPGPATH * 2];
163+
while (errno = 0, (de = readdir(dir)) != NULL)
164+
{
165+
char subpath[MAXPGPATH * 2];
159166

160-
if (strcmp(de->d_name, ".") == 0 ||
161-
strcmp(de->d_name, "..") == 0)
162-
continue;
167+
if (strcmp(de->d_name, ".") == 0 ||
168+
strcmp(de->d_name, "..") == 0)
169+
continue;
163170

164-
snprintf(subpath, sizeof(subpath), "%s/%s",
165-
pg_tblspc, de->d_name);
166-
do_syncfs(subpath);
167-
}
171+
snprintf(subpath, sizeof(subpath), "%s/%s",
172+
pg_tblspc, de->d_name);
173+
do_syncfs(subpath);
174+
}
168175

169-
if (errno)
170-
pg_log_error("could not read directory \"%s\": %m",
171-
pg_tblspc);
176+
if (errno)
177+
pg_log_error("could not read directory \"%s\": %m",
178+
pg_tblspc);
172179

173-
(void) closedir(dir);
180+
(void) closedir(dir);
181+
}
174182
}
175183

176184
/* If pg_wal is a symlink, process that too. */
@@ -182,15 +190,21 @@ sync_pgdata(const char *pg_data,
182190

183191
case DATA_DIR_SYNC_METHOD_FSYNC:
184192
{
193+
char *exclude_dir = NULL;
194+
195+
if (!sync_data_files)
196+
exclude_dir = psprintf("%s/base", pg_data);
197+
185198
/*
186199
* If possible, hint to the kernel that we're soon going to
187200
* fsync the data directory and its contents.
188201
*/
189202
#ifdef PG_FLUSH_DATA_WORKS
190-
walkdir(pg_data, pre_sync_fname, false);
203+
walkdir(pg_data, pre_sync_fname, false, exclude_dir);
191204
if (xlog_is_symlink)
192-
walkdir(pg_wal, pre_sync_fname, false);
193-
walkdir(pg_tblspc, pre_sync_fname, true);
205+
walkdir(pg_wal, pre_sync_fname, false, NULL);
206+
if (sync_data_files)
207+
walkdir(pg_tblspc, pre_sync_fname, true, NULL);
194208
#endif
195209

196210
/*
@@ -203,10 +217,14 @@ sync_pgdata(const char *pg_data,
203217
* get fsync'd twice. That's not an expected case so we don't
204218
* worry about optimizing it.
205219
*/
206-
walkdir(pg_data, fsync_fname, false);
220+
walkdir(pg_data, fsync_fname, false, exclude_dir);
207221
if (xlog_is_symlink)
208-
walkdir(pg_wal, fsync_fname, false);
209-
walkdir(pg_tblspc, fsync_fname, true);
222+
walkdir(pg_wal, fsync_fname, false, NULL);
223+
if (sync_data_files)
224+
walkdir(pg_tblspc, fsync_fname, true, NULL);
225+
226+
if (exclude_dir)
227+
pfree(exclude_dir);
210228
}
211229
break;
212230
}
@@ -245,10 +263,10 @@ sync_dir_recurse(const char *dir, DataDirSyncMethod sync_method)
245263
* fsync the data directory and its contents.
246264
*/
247265
#ifdef PG_FLUSH_DATA_WORKS
248-
walkdir(dir, pre_sync_fname, false);
266+
walkdir(dir, pre_sync_fname, false, NULL);
249267
#endif
250268

251-
walkdir(dir, fsync_fname, false);
269+
walkdir(dir, fsync_fname, false, NULL);
252270
}
253271
break;
254272
}
@@ -264,18 +282,25 @@ sync_dir_recurse(const char *dir, DataDirSyncMethod sync_method)
264282
* ignored in subdirectories, ie we intentionally don't pass down the
265283
* process_symlinks flag to recursive calls.
266284
*
285+
* If exclude_dir is not NULL, it specifies a directory path to skip
286+
* processing.
287+
*
267288
* Errors are reported but not considered fatal.
268289
*
269290
* See also walkdir in fd.c, which is a backend version of this logic.
270291
*/
271292
static void
272293
walkdir(const char *path,
273294
int (*action) (const char *fname, bool isdir),
274-
bool process_symlinks)
295+
bool process_symlinks,
296+
const char *exclude_dir)
275297
{
276298
DIR *dir;
277299
struct dirent *de;
278300

301+
if (exclude_dir && strcmp(exclude_dir, path) == 0)
302+
return;
303+
279304
dir = opendir(path);
280305
if (dir == NULL)
281306
{
@@ -299,7 +324,7 @@ walkdir(const char *path,
299324
(*action) (subpath, false);
300325
break;
301326
case PGFILETYPE_DIR:
302-
walkdir(subpath, action, false);
327+
walkdir(subpath, action, false, exclude_dir);
303328
break;
304329
default:
305330

‎src/include/common/file_utils.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ struct iovec; /* avoid including port/pg_iovec.h here */
3535
#ifdef FRONTEND
3636
extern int fsync_fname(const char *fname, bool isdir);
3737
extern void sync_pgdata(const char *pg_data, int serverVersion,
38-
DataDirSyncMethod sync_method);
38+
DataDirSyncMethod sync_method, bool sync_data_files);
3939
extern void sync_dir_recurse(const char *dir, DataDirSyncMethod sync_method);
4040
extern int durable_rename(const char *oldfile, const char *newfile);
4141
extern int fsync_parent_path(const char *fname);

0 commit comments

Comments
 (0)