Skip to content

Commit 861c6e7

Browse files
macdicemacdice
committed
Skip unnecessary stat() calls in walkdir().
Some kernels can tell us the type of a "dirent", so we can avoid a call to stat() or lstat() in many cases. Define a new function get_dirent_type() to contain that logic, for use by the backend and frontend versions of walkdir(), and perhaps other callers in future. Reviewed-by: Tom Lane <tgl@sss.pgh.pa.us> Reviewed-by: Juan José Santamaría Flecha <juanjo.santamaria@gmail.com> Discussion: https://postgr.es/m/CA%2BhUKG%2BFzxupGGN4GpUdbzZN%2Btn6FQPHo8w0Q%2BAPH5Wz8RG%2Bww%40mail.gmail.com
1 parent f0942b1 commit 861c6e7

File tree

6 files changed

+129
-40
lines changed

6 files changed

+129
-40
lines changed

‎src/backend/storage/file/fd.c

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
#include "access/xlog.h"
9090
#include "catalog/pg_tablespace.h"
9191
#include "common/file_perm.h"
92+
#include "common/file_utils.h"
9293
#include "miscadmin.h"
9394
#include "pgstat.h"
9495
#include "portability/mem.h"
@@ -3340,8 +3341,6 @@ walkdir(const char *path,
33403341
while ((de = ReadDirExtended(dir, path, elevel)) != NULL)
33413342
{
33423343
char subpath[MAXPGPATH * 2];
3343-
struct stat fst;
3344-
int sret;
33453344

33463345
CHECK_FOR_INTERRUPTS();
33473346

@@ -3351,23 +3350,23 @@ walkdir(const char *path,
33513350

33523351
snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name);
33533352

3354-
if (process_symlinks)
3355-
sret = stat(subpath, &fst);
3356-
else
3357-
sret = lstat(subpath, &fst);
3358-
3359-
if (sret < 0)
3353+
switch (get_dirent_type(subpath, de, process_symlinks, elevel))
33603354
{
3361-
ereport(elevel,
3362-
(errcode_for_file_access(),
3363-
errmsg("could not stat file \"%s\": %m", subpath)));
3364-
continue;
3365-
}
3355+
case PGFILETYPE_REG:
3356+
(*action) (subpath, false, elevel);
3357+
break;
3358+
case PGFILETYPE_DIR:
3359+
walkdir(subpath, action, false, elevel);
3360+
break;
3361+
default:
33663362

3367-
if (S_ISREG(fst.st_mode))
3368-
(*action) (subpath, false, elevel);
3369-
else if (S_ISDIR(fst.st_mode))
3370-
walkdir(subpath, action, false, elevel);
3363+
/*
3364+
* Errors are already reported directly by get_dirent_type(),
3365+
* and any remaining symlinks and unknown file types are
3366+
* ignored.
3367+
*/
3368+
break;
3369+
}
33713370
}
33723371

33733372
FreeDir(dir); /* we ignore any error here */

‎src/common/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ OBJS_COMMON = \
5656
exec.o \
5757
f2s.o \
5858
file_perm.o \
59+
file_utils.o \
5960
hashfn.o \
6061
ip.o \
6162
jsonapi.o \
@@ -91,7 +92,6 @@ endif
9192
OBJS_FRONTEND = \
9293
$(OBJS_COMMON) \
9394
fe_memutils.o \
94-
file_utils.o \
9595
logging.o \
9696
restricted_token.o \
9797
sprompt.o

‎src/common/file_utils.c

Lines changed: 91 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -14,19 +14,22 @@
1414
*/
1515

1616
#ifndef FRONTEND
17-
#error "This file is not expected to be compiled for backend code"
18-
#endif
19-
17+
#include "postgres.h"
18+
#else
2019
#include "postgres_fe.h"
20+
#endif
2121

2222
#include <dirent.h>
2323
#include <fcntl.h>
2424
#include <sys/stat.h>
2525
#include <unistd.h>
2626

2727
#include "common/file_utils.h"
28+
#ifdef FRONTEND
2829
#include "common/logging.h"
30+
#endif
2931

32+
#ifdef FRONTEND
3033

3134
/* Define PG_FLUSH_DATA_WORKS if we have an implementation for pg_flush_data */
3235
#if defined(HAVE_SYNC_FILE_RANGE)
@@ -167,30 +170,30 @@ walkdir(const char *path,
167170
while (errno = 0, (de = readdir(dir)) != NULL)
168171
{
169172
char subpath[MAXPGPATH * 2];
170-
struct stat fst;
171-
int sret;
172173

173174
if (strcmp(de->d_name, ".") == 0 ||
174175
strcmp(de->d_name, "..") == 0)
175176
continue;
176177

177178
snprintf(subpath, sizeof(subpath), "%s/%s", path, de->d_name);
178179

179-
if (process_symlinks)
180-
sret = stat(subpath, &fst);
181-
else
182-
sret = lstat(subpath, &fst);
183-
184-
if (sret < 0)
180+
switch (get_dirent_type(subpath, de, process_symlinks, PG_LOG_ERROR))
185181
{
186-
pg_log_error("could not stat file \"%s\": %m", subpath);
187-
continue;
182+
case PGFILETYPE_REG:
183+
(*action) (subpath, false);
184+
break;
185+
case PGFILETYPE_DIR:
186+
walkdir(subpath, action, false);
187+
break;
188+
default:
189+
190+
/*
191+
* Errors are already reported directly by get_dirent_type(),
192+
* and any remaining symlinks and unknown file types are
193+
* ignored.
194+
*/
195+
break;
188196
}
189-
190-
if (S_ISREG(fst.st_mode))
191-
(*action) (subpath, false);
192-
else if (S_ISDIR(fst.st_mode))
193-
walkdir(subpath, action, false);
194197
}
195198

196199
if (errno)
@@ -394,3 +397,73 @@ durable_rename(const char *oldfile, const char *newfile)
394397

395398
return 0;
396399
}
400+
401+
#endif /* FRONTEND */
402+
403+
/*
404+
* Return the type of a directory entry.
405+
*
406+
* In frontend code, elevel should be a level from logging.h; in backend code
407+
* it should be a level from elog.h.
408+
*/
409+
PGFileType
410+
get_dirent_type(const char *path,
411+
const struct dirent *de,
412+
bool look_through_symlinks,
413+
int elevel)
414+
{
415+
PGFileType result;
416+
417+
/*
418+
* Some systems tell us the type directly in the dirent struct, but that's
419+
* a BSD and Linux extension not required by POSIX. Even when the
420+
* interface is present, sometimes the type is unknown, depending on the
421+
* filesystem.
422+
*/
423+
#if defined(DT_REG) && defined(DT_DIR) && defined(DT_LNK)
424+
if (de->d_type == DT_REG)
425+
result = PGFILETYPE_REG;
426+
else if (de->d_type == DT_DIR)
427+
result = PGFILETYPE_DIR;
428+
else if (de->d_type == DT_LNK && !look_through_symlinks)
429+
result = PGFILETYPE_LNK;
430+
else
431+
result = PGFILETYPE_UNKNOWN;
432+
#else
433+
result = PGFILETYPE_UNKNOWN;
434+
#endif
435+
436+
if (result == PGFILETYPE_UNKNOWN)
437+
{
438+
struct stat fst;
439+
int sret;
440+
441+
442+
if (look_through_symlinks)
443+
sret = stat(path, &fst);
444+
else
445+
sret = lstat(path, &fst);
446+
447+
if (sret < 0)
448+
{
449+
result = PGFILETYPE_ERROR;
450+
#ifdef FRONTEND
451+
pg_log_generic(elevel, "could not stat file \"%s\": %m", path);
452+
#else
453+
ereport(elevel,
454+
(errcode_for_file_access(),
455+
errmsg("could not stat file \"%s\": %m", path)));
456+
#endif
457+
}
458+
else if (S_ISREG(fst.st_mode))
459+
result = PGFILETYPE_REG;
460+
else if (S_ISDIR(fst.st_mode))
461+
result = PGFILETYPE_DIR;
462+
#ifdef S_ISLNK
463+
else if (S_ISLNK(fst.st_mode))
464+
result = PGFILETYPE_LNK;
465+
#endif
466+
}
467+
468+
return result;
469+
}

‎src/include/common/file_utils.h

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
/*-------------------------------------------------------------------------
2-
*
3-
* File-processing utility routines for frontend code
42
*
53
* Assorted utility functions to work on files.
64
*
@@ -15,10 +13,28 @@
1513
#ifndef FILE_UTILS_H
1614
#define FILE_UTILS_H
1715

16+
#include <dirent.h>
17+
18+
typedef enum PGFileType
19+
{
20+
PGFILETYPE_ERROR,
21+
PGFILETYPE_UNKNOWN,
22+
PGFILETYPE_REG,
23+
PGFILETYPE_DIR,
24+
PGFILETYPE_LNK
25+
} PGFileType;
26+
27+
#ifdef FRONTEND
1828
extern int fsync_fname(const char *fname, bool isdir);
1929
extern void fsync_pgdata(const char *pg_data, int serverVersion);
2030
extern void fsync_dir_recurse(const char *dir);
2131
extern int durable_rename(const char *oldfile, const char *newfile);
2232
extern int fsync_parent_path(const char *fname);
33+
#endif
34+
35+
extern PGFileType get_dirent_type(const char *path,
36+
const struct dirent *de,
37+
bool look_through_symlinks,
38+
int elevel);
2339

2440
#endif /* FILE_UTILS_H */

‎src/tools/msvc/Mkvcbuild.pm

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ sub mkvcbuild
121121
our @pgcommonallfiles = qw(
122122
archive.c base64.c checksum_helper.c
123123
config_info.c controldata_utils.c d2s.c encnames.c exec.c
124-
f2s.c file_perm.c hashfn.c ip.c jsonapi.c
124+
f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c
125125
keywords.c kwlookup.c link-canary.c md5.c
126126
pg_get_line.c pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c
127127
saslprep.c scram-common.c string.c stringinfo.c unicode_norm.c username.c
@@ -138,7 +138,7 @@ sub mkvcbuild
138138
}
139139

140140
our @pgcommonfrontendfiles = (
141-
@pgcommonallfiles, qw(fe_memutils.c file_utils.c
141+
@pgcommonallfiles, qw(fe_memutils.c
142142
logging.c restricted_token.c sprompt.c));
143143

144144
our @pgcommonbkndfiles = @pgcommonallfiles;

‎src/tools/pgindent/typedefs.list

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1515,6 +1515,7 @@ PGEventResultCopy
15151515
PGEventResultCreate
15161516
PGEventResultDestroy
15171517
PGFInfoFunction
1518+
PGFileType
15181519
PGFunction
15191520
PGLZ_HistEntry
15201521
PGLZ_Strategy

0 commit comments

Comments
 (0)