Fix issues with Windows' stat() for files pending on deletion

michaelpq · adunstan · commit f1cf64167fc9 · 2024-11-08T09:41:47.000+10:30
The code introduced by bed9075 to enhance the stat() implementation on Windows for file sizes larger than 4GB fails to properly detect files pending for deletion with its method based on NtQueryInformationFile() or GetFileInformationByHandleEx(), as proved by Alexander Lakhin in a custom TAP test of his own. The method used in the implementation of open() to sleep and loop when when failing on ERROR_ACCESS_DENIED (EACCES) is showing much more stability, so switch to this method. This could still lead to issues if the permission problem stays around for much longer than the timeout of 1 second used, but that should (hopefully) never happen in performance-critical paths. Still, there could be a point in increasing the timeouts for the sake of machines that handle heavy loads. Note that WIN32's open() now uses microsoft_native_stat() as it should be similar to stat() when working around issues with concurrent file deletions. I have spent some time testing this patch with pgbench in combination of the SQL functions from genfile.c, as well as running the TAP test provided on the thread with MSVC builds, and this looks much more stable than the previous method. Author: Alexander Lakhin Reviewed-by: Tom Lane, Michael Paquier, Justin Pryzby Discussion: https://postgr.es/m/c3427edf-d7c0-ff57-90f6-b5de3bb62709@gmail.com Backpatch-through: 14 (cherry picked from commit 54fb8c7) Author: Alexandra Wang <alexandra.wang.oss@gmail.com>
diff --git a/src/port/open.c b/src/port/open.c
@@ -157,9 +157,9 @@ pgwin32_open(const char *fileName, int fileFlags,...)
 		{
 			if (loops < 10)
 			{
-				struct stat st;
+				struct microsoft_native_stat st;
 
-				if (stat(fileName, &st) != 0)
+				if (microsoft_native_stat(fileName, &st) != 0)
 				{
 					pg_usleep(100000);
 					loops++;
diff --git a/src/port/win32stat.c b/src/port/win32stat.c
@@ -18,55 +18,6 @@
 #include "c.h"
 #include <windows.h>
 
-/*
- * In order to support MinGW and MSVC2013 we use NtQueryInformationFile as an
- * alternative for GetFileInformationByHandleEx. It is loaded from the ntdll
- * library.
- */
-#if _WIN32_WINNT < 0x0600
-#include <winternl.h>
-
-#if !defined(__MINGW32__) && !defined(__MINGW64__)
-/* MinGW includes this in <winternl.h>, but it is missing in MSVC */
-typedef struct _FILE_STANDARD_INFORMATION
-{
-	LARGE_INTEGER AllocationSize;
-	LARGE_INTEGER EndOfFile;
-	ULONG		NumberOfLinks;
-	BOOLEAN		DeletePending;
-	BOOLEAN		Directory;
-} FILE_STANDARD_INFORMATION;
-#define FileStandardInformation 5
-#endif							/* !defined(__MINGW32__) &&
-								 * !defined(__MINGW64__) */
-
-typedef NTSTATUS(NTAPI * PFN_NTQUERYINFORMATIONFILE)
-(
- IN HANDLE FileHandle,
- OUT PIO_STATUS_BLOCK IoStatusBlock,
- OUT PVOID FileInformation,
- IN ULONG Length,
- IN FILE_INFORMATION_CLASS FileInformationClass
-);
-
-static PFN_NTQUERYINFORMATIONFILE _NtQueryInformationFile = NULL;
-
-static HMODULE ntdll = NULL;
-
-/*
- * Load DLL file just once regardless of how many functions we load/call in it.
- */
-static void
-LoadNtdll(void)
-{
-	if (ntdll != NULL)
-		return;
-	ntdll = LoadLibraryEx("ntdll.dll", NULL, 0);
-}
-
-#endif							/* _WIN32_WINNT < 0x0600 */
-
-
 /*
  * Convert a FILETIME struct into a 64 bit time_t.
  */
@@ -165,105 +116,79 @@ _pgstat64(const char *name, struct stat *buf)
 {
 	/*
 	 * We must use a handle so lstat() returns the information of the target
-	 * file.  To have a reliable test for ERROR_DELETE_PENDING, we use
-	 * NtQueryInformationFile from Windows 2000 or
-	 * GetFileInformationByHandleEx from Server 2008 / Vista.
+	 * file.  To have a reliable test for ERROR_DELETE_PENDING, this uses a
+	 * method similar to open() with a loop using stat() and some waits when
+	 * facing ERROR_ACCESS_DENIED.
 	 */
 	SECURITY_ATTRIBUTES sa;
 	HANDLE		hFile;
 	int			ret;
-#if _WIN32_WINNT < 0x0600
-	IO_STATUS_BLOCK ioStatus;
-	FILE_STANDARD_INFORMATION standardInfo;
-#else
-	FILE_STANDARD_INFO standardInfo;
-#endif
+	int			loops = 0;
 
 	if (name == NULL || buf == NULL)
 	{
 		errno = EINVAL;
 		return -1;
 	}
-
 	/* fast not-exists check */
 	if (GetFileAttributes(name) == INVALID_FILE_ATTRIBUTES)
 	{
-		_dosmaperr(GetLastError());
-		return -1;
+		DWORD		err = GetLastError();
+
+		if (err != ERROR_ACCESS_DENIED)
+		{
+			_dosmaperr(err);
+			return -1;
+		}
 	}
 
 	/* get a file handle as lightweight as we can */
 	sa.nLength = sizeof(SECURITY_ATTRIBUTES);
 	sa.bInheritHandle = TRUE;
 	sa.lpSecurityDescriptor = NULL;
-	hFile = CreateFile(name,
-					   GENERIC_READ,
-					   (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE),
-					   &sa,
-					   OPEN_EXISTING,
-					   (FILE_FLAG_NO_BUFFERING | FILE_FLAG_BACKUP_SEMANTICS |
-						FILE_FLAG_OVERLAPPED),
-					   NULL);
-	if (hFile == INVALID_HANDLE_VALUE)
+	while ((hFile = CreateFile(name,
+							   GENERIC_READ,
+							   (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE),
+							   &sa,
+							   OPEN_EXISTING,
+							   (FILE_FLAG_NO_BUFFERING | FILE_FLAG_BACKUP_SEMANTICS |
+								FILE_FLAG_OVERLAPPED),
+							   NULL)) == INVALID_HANDLE_VALUE)
 	{
-		CloseHandle(hFile);
-		errno = ENOENT;
-		return -1;
-	}
-
-	memset(&standardInfo, 0, sizeof(standardInfo));
+		DWORD		err = GetLastError();
 
-#if _WIN32_WINNT < 0x0600
-	if (_NtQueryInformationFile == NULL)
-	{
-		/* First time through: load ntdll.dll and find NtQueryInformationFile */
-		LoadNtdll();
-		if (ntdll == NULL)
-		{
-			_dosmaperr(GetLastError());
-			CloseHandle(hFile);
-			return -1;
-		}
-
-		_NtQueryInformationFile = (PFN_NTQUERYINFORMATIONFILE)
-			GetProcAddress(ntdll, "NtQueryInformationFile");
-		if (_NtQueryInformationFile == NULL)
+		/*
+		 * ERROR_ACCESS_DENIED is returned if the file is deleted but not yet
+		 * gone (Windows NT status code is STATUS_DELETE_PENDING).  In that
+		 * case we want to wait a bit and try again, giving up after 1 second
+		 * (since this condition should never persist very long).  However,
+		 * there are other commonly-hit cases that return ERROR_ACCESS_DENIED,
+		 * so care is needed.  In particular that happens if we try to open a
+		 * directory, or of course if there's an actual file-permissions
+		 * problem.  To distinguish these cases, try a stat().  In the
+		 * delete-pending case, it will either also get STATUS_DELETE_PENDING,
+		 * or it will see the file as gone and fail with ENOENT.  In other
+		 * cases it will usually succeed.  The only somewhat-likely case where
+		 * this coding will uselessly wait is if there's a permissions problem
+		 * with a containing directory, which we hope will never happen in any
+		 * performance-critical code paths.
+		 */
+		if (err == ERROR_ACCESS_DENIED)
 		{
-			_dosmaperr(GetLastError());
-			CloseHandle(hFile);
-			return -1;
+			if (loops < 10)
+			{
+				struct microsoft_native_stat st;
+
+				if (microsoft_native_stat(name, &st) != 0)
+				{
+					pg_usleep(100000);
+					loops++;
+					continue;
+				}
+			}
 		}
-	}
 
-	if (!NT_SUCCESS(_NtQueryInformationFile(hFile, &ioStatus, &standardInfo,
-											sizeof(standardInfo),
-											FileStandardInformation)))
-	{
-		_dosmaperr(GetLastError());
-		CloseHandle(hFile);
-		return -1;
-	}
-#else
-	if (!GetFileInformationByHandleEx(hFile, FileStandardInfo, &standardInfo,
-									  sizeof(standardInfo)))
-	{
-		_dosmaperr(GetLastError());
-		CloseHandle(hFile);
-		return -1;
-	}
-#endif							/* _WIN32_WINNT < 0x0600 */
-
-	if (standardInfo.DeletePending)
-	{
-		/*
-		 * File has been deleted, but is not gone from the filesystem yet.
-		 * This can happen when some process with FILE_SHARE_DELETE has it
-		 * open, and it will be fully removed once that handle is closed.
-		 * Meanwhile, we can't open it, so indicate that the file just doesn't
-		 * exist.
-		 */
-		CloseHandle(hFile);
-		errno = ENOENT;
+		_dosmaperr(err);
 		return -1;
 	}
 

Original file line number	Diff line number	Diff line change
`@@ -157,9 +157,9 @@ pgwin32_open(const char *fileName, int fileFlags,...)`
`157`	`157`	`{`
`158`	`158`	`if (loops < 10)`
`159`	`159`	`{`
`160`		`- struct stat st;`
	`160`	`+ struct microsoft_native_stat st;`
`161`	`161`
`162`		`- if (stat(fileName, &st) != 0)`
	`162`	`+ if (microsoft_native_stat(fileName, &st) != 0)`
`163`	`163`	`{`
`164`	`164`	`pg_usleep(100000);`
`165`	`165`	`loops++;`