From dffb615f04dcaef2c74396ddbe54789cd0b96cf4 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 29 May 2023 11:39:08 +0100 Subject: [PATCH 1/4] gh-88745: Add _winapi.CopyFile2 and update shutil.copy2 to use it --- .../pycore_global_objects_fini_generated.h | 4 + Include/internal/pycore_global_strings.h | 4 + .../internal/pycore_runtime_init_generated.h | 4 + .../internal/pycore_unicodeobject_generated.h | 12 +++ Lib/shutil.py | 26 ++++++ ...3-05-29-11-38-53.gh-issue-88745.cldf9G.rst | 3 + Modules/_winapi.c | 85 +++++++++++++++++++ Modules/clinic/_winapi.c.h | 72 +++++++++++++++- 8 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-05-29-11-38-53.gh-issue-88745.cldf9G.rst diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index a83f8fc49fc5ef..1200e65335a55c 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -570,10 +570,14 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ExistingFileName)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(FINISHED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(False)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(JSONDecodeError)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(NewFileName)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(PENDING)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ProgressRoutine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Py_Repr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(TextIOWrapper)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index dd6a62f53a9989..78a3683dd6c0e1 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -58,10 +58,14 @@ struct _Py_global_strings { struct { STRUCT_FOR_ID(CANCELLED) + STRUCT_FOR_ID(ExistingFileName) STRUCT_FOR_ID(FINISHED) STRUCT_FOR_ID(False) + STRUCT_FOR_ID(Flags) STRUCT_FOR_ID(JSONDecodeError) + STRUCT_FOR_ID(NewFileName) STRUCT_FOR_ID(PENDING) + STRUCT_FOR_ID(ProgressRoutine) STRUCT_FOR_ID(Py_Repr) STRUCT_FOR_ID(TextIOWrapper) STRUCT_FOR_ID(True) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index d689f717eaf94f..702c7ee5f7ef19 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -564,10 +564,14 @@ extern "C" { #define _Py_str_identifiers_INIT { \ INIT_ID(CANCELLED), \ + INIT_ID(ExistingFileName), \ INIT_ID(FINISHED), \ INIT_ID(False), \ + INIT_ID(Flags), \ INIT_ID(JSONDecodeError), \ + INIT_ID(NewFileName), \ INIT_ID(PENDING), \ + INIT_ID(ProgressRoutine), \ INIT_ID(Py_Repr), \ INIT_ID(TextIOWrapper), \ INIT_ID(True), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index db6a157ee7afbf..036a8115824261 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -15,18 +15,30 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(CANCELLED); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(ExistingFileName); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(FINISHED); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(False); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(Flags); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(JSONDecodeError); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(NewFileName); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(PENDING); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(ProgressRoutine); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(Py_Repr); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/shutil.py b/Lib/shutil.py index 7d1a3d00011f37..0d1e09845297b6 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -42,6 +42,8 @@ if sys.platform == 'win32': import _winapi +else: + _winapi = None COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 # This should never be removed, see rationale in: @@ -435,6 +437,30 @@ def copy2(src, dst, *, follow_symlinks=True): """ if os.path.isdir(dst): dst = os.path.join(dst, os.path.basename(src)) + + if hasattr(_winapi, "CopyFile2"): + src_ = os.fsdecode(src) + dst_ = os.fsdecode(dst) + flags = _winapi.COPY_FILE_ALLOW_DECRYPTED_DESTINATION # for compat + if not follow_symlinks: + flags |= _winapi.COPY_FILE_COPY_SYMLINK + retry = False + try: + _winapi.CopyFile2(src_, dst_, flags) + return dst + except OSError as exc: + if (exc.winerror == _winapi.ERROR_PRIVILEGE_NOT_HELD + and not follow_symlinks): + # Likely encountered a symlink we aren't allowed to create. + # Fall back on the old code + pass + elif exc.winerror == _winapi.ERROR_ACCESS_DENIED: + # Possibly encountered a hidden or readonly file we can't + # overwrite. Fall back on old code + pass + else: + raise + copyfile(src, dst, follow_symlinks=follow_symlinks) copystat(src, dst, follow_symlinks=follow_symlinks) return dst diff --git a/Misc/NEWS.d/next/Windows/2023-05-29-11-38-53.gh-issue-88745.cldf9G.rst b/Misc/NEWS.d/next/Windows/2023-05-29-11-38-53.gh-issue-88745.cldf9G.rst new file mode 100644 index 00000000000000..258eb89d50d9f5 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-05-29-11-38-53.gh-issue-88745.cldf9G.rst @@ -0,0 +1,3 @@ +Improve performance of :func:`shutil.copy2` by using the operating system's +``CopyFile2`` function. This may result in subtle changes to metadata copied +along with some files, bringing them in line with normal OS behavior. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 1e02dbc1a4bfd1..e69cdf0a8bda3d 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1947,6 +1947,7 @@ _winapi_GetFileType_impl(PyObject *module, HANDLE handle) return result; } + /*[clinic input] _winapi._mimetypes_read_windows_registry @@ -2075,6 +2076,59 @@ _winapi_NeedCurrentDirectoryForExePath_impl(PyObject *module, return result; } + +/*[clinic input] +_winapi.CopyFile2 + + ExistingFileName: LPCWSTR + NewFileName: LPCWSTR + Flags: DWORD + ProgressRoutine: object = None + +Copies a file from one name to a new name. + +This is implemented using the CopyFile2 API, which preserves all stat +and metadata information apart from security attributes. + +ProgressRoutine is reserved for future use, but is currently not +implemented. Its value is ignored. +[clinic start generated code]*/ + +static PyObject * +_winapi_CopyFile2_impl(PyObject *module, LPCWSTR ExistingFileName, + LPCWSTR NewFileName, DWORD Flags, + PyObject *ProgressRoutine) +/*[clinic end generated code: output=f3f588380da0000a input=13cf59b6e1a70fe6]*/ +{ + HRESULT hr; + COPYFILE2_EXTENDED_PARAMETERS params = { sizeof(COPYFILE2_EXTENDED_PARAMETERS) }; + params.dwCopyFlags = Flags; + /* For future implementation. We ignore the value for now so that + users only have to test for 'CopyFile2' existing and not whether + the additional parameter exists. + if (ProgressRoutine != Py_None) { + params.pProgressRoutine = _winapi_CopyFile2ProgressRoutine; + params.pvCallbackContext = Py_NewRef(ProgressRoutine); + } + */ + Py_BEGIN_ALLOW_THREADS; + hr = CopyFile2(ExistingFileName, NewFileName, ¶ms); + Py_END_ALLOW_THREADS; + /* For future implementation. + Py_XDECREF(ProgressRoutine); + */ + if (FAILED(hr)) { + if ((hr & 0xFFFF0000) == 0x80070000) { + PyErr_SetFromWindowsErr(hr & 0xFFFF); + } else { + PyErr_SetFromWindowsErr(hr); + } + return NULL; + } + Py_RETURN_NONE; +} + + static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF _WINAPI_CONNECTNAMEDPIPE_METHODDEF @@ -2110,6 +2164,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI_GETFILETYPE_METHODDEF _WINAPI__MIMETYPES_READ_WINDOWS_REGISTRY_METHODDEF _WINAPI_NEEDCURRENTDIRECTORYFOREXEPATH_METHODDEF + _WINAPI_COPYFILE2_METHODDEF {NULL, NULL} }; @@ -2146,6 +2201,7 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, CREATE_NEW_PROCESS_GROUP); WINAPI_CONSTANT(F_DWORD, DUPLICATE_SAME_ACCESS); WINAPI_CONSTANT(F_DWORD, DUPLICATE_CLOSE_SOURCE); + WINAPI_CONSTANT(F_DWORD, ERROR_ACCESS_DENIED); WINAPI_CONSTANT(F_DWORD, ERROR_ALREADY_EXISTS); WINAPI_CONSTANT(F_DWORD, ERROR_BROKEN_PIPE); WINAPI_CONSTANT(F_DWORD, ERROR_IO_PENDING); @@ -2159,6 +2215,7 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, ERROR_OPERATION_ABORTED); WINAPI_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); WINAPI_CONSTANT(F_DWORD, ERROR_PIPE_CONNECTED); + WINAPI_CONSTANT(F_DWORD, ERROR_PRIVILEGE_NOT_HELD); WINAPI_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); WINAPI_CONSTANT(F_DWORD, FILE_FLAG_FIRST_PIPE_INSTANCE); WINAPI_CONSTANT(F_DWORD, FILE_FLAG_OVERLAPPED); @@ -2252,6 +2309,34 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, LCMAP_TRADITIONAL_CHINESE); WINAPI_CONSTANT(F_DWORD, LCMAP_UPPERCASE); + WINAPI_CONSTANT(F_DWORD, COPY_FILE_ALLOW_DECRYPTED_DESTINATION); + WINAPI_CONSTANT(F_DWORD, COPY_FILE_COPY_SYMLINK); + WINAPI_CONSTANT(F_DWORD, COPY_FILE_FAIL_IF_EXISTS); + WINAPI_CONSTANT(F_DWORD, COPY_FILE_NO_BUFFERING); + WINAPI_CONSTANT(F_DWORD, COPY_FILE_NO_OFFLOAD); + WINAPI_CONSTANT(F_DWORD, COPY_FILE_OPEN_SOURCE_FOR_WRITE); + WINAPI_CONSTANT(F_DWORD, COPY_FILE_RESTARTABLE); + WINAPI_CONSTANT(F_DWORD, COPY_FILE_REQUEST_SECURITY_PRIVILEGES); + WINAPI_CONSTANT(F_DWORD, COPY_FILE_RESUME_FROM_PAUSE); +#ifndef COPY_FILE_REQUEST_COMPRESSED_TRAFFIC + // Only defined in newer WinSDKs + #define COPY_FILE_REQUEST_COMPRESSED_TRAFFIC 0x10000000 +#endif + WINAPI_CONSTANT(F_DWORD, COPY_FILE_REQUEST_COMPRESSED_TRAFFIC); + + WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_CHUNK_STARTED); + WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_CHUNK_FINISHED); + WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_STREAM_STARTED); + WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_STREAM_FINISHED); + WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_POLL_CONTINUE); + WINAPI_CONSTANT(F_DWORD, COPYFILE2_CALLBACK_ERROR); + + WINAPI_CONSTANT(F_DWORD, COPYFILE2_PROGRESS_CONTINUE); + WINAPI_CONSTANT(F_DWORD, COPYFILE2_PROGRESS_CANCEL); + WINAPI_CONSTANT(F_DWORD, COPYFILE2_PROGRESS_STOP); + WINAPI_CONSTANT(F_DWORD, COPYFILE2_PROGRESS_QUIET); + WINAPI_CONSTANT(F_DWORD, COPYFILE2_PROGRESS_PAUSE); + WINAPI_CONSTANT("i", NULL); return 0; diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 7bc63e612be348..c75d863b1728b1 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -1313,6 +1313,76 @@ _winapi_GetFileType(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P return return_value; } +PyDoc_STRVAR(_winapi_CopyFile2__doc__, +"CopyFile2($module, /, ExistingFileName, NewFileName, Flags,\n" +" ProgressRoutine=None)\n" +"--\n" +"\n" +"Copies a file from one name to a new name.\n" +"\n" +"This is implemented using the CopyFile2 API, which preserves all stat\n" +"and metadata information apart from security attributes.\n" +"\n" +"ProgressRoutine is reserved for future use, but is currently not\n" +"implemented. Its value is ignored."); + +#define _WINAPI_COPYFILE2_METHODDEF \ + {"CopyFile2", _PyCFunction_CAST(_winapi_CopyFile2), METH_FASTCALL|METH_KEYWORDS, _winapi_CopyFile2__doc__}, + +static PyObject * +_winapi_CopyFile2_impl(PyObject *module, LPCWSTR ExistingFileName, + LPCWSTR NewFileName, DWORD Flags, + PyObject *ProgressRoutine); + +static PyObject * +_winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(ExistingFileName), &_Py_ID(NewFileName), &_Py_ID(Flags), &_Py_ID(ProgressRoutine), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"ExistingFileName", "NewFileName", "Flags", "ProgressRoutine", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "O&O&k|O:CopyFile2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + LPCWSTR ExistingFileName = NULL; + LPCWSTR NewFileName = NULL; + DWORD Flags; + PyObject *ProgressRoutine = Py_None; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + _PyUnicode_WideCharString_Converter, &ExistingFileName, _PyUnicode_WideCharString_Converter, &NewFileName, &Flags, &ProgressRoutine)) { + goto exit; + } + return_value = _winapi_CopyFile2_impl(module, ExistingFileName, NewFileName, Flags, ProgressRoutine); + +exit: + /* Cleanup for ExistingFileName */ + PyMem_Free((void *)ExistingFileName); + /* Cleanup for NewFileName */ + PyMem_Free((void *)NewFileName); + + return return_value; +} + PyDoc_STRVAR(_winapi__mimetypes_read_windows_registry__doc__, "_mimetypes_read_windows_registry($module, /, on_type_read)\n" "--\n" @@ -1411,4 +1481,4 @@ _winapi_NeedCurrentDirectoryForExePath(PyObject *module, PyObject *arg) return return_value; } -/*[clinic end generated code: output=96ea65ece7912d0a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5e8927548ec12266 input=a9049054013a1b77]*/ From 5b8788da7adbfcecb74ff93a2e72d9d9bd7be2a0 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 29 May 2023 11:51:54 +0100 Subject: [PATCH 2/4] Update clinic again --- Modules/clinic/_winapi.c.h | 142 ++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index c75d863b1728b1..a72a7224dbc596 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -1313,76 +1313,6 @@ _winapi_GetFileType(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P return return_value; } -PyDoc_STRVAR(_winapi_CopyFile2__doc__, -"CopyFile2($module, /, ExistingFileName, NewFileName, Flags,\n" -" ProgressRoutine=None)\n" -"--\n" -"\n" -"Copies a file from one name to a new name.\n" -"\n" -"This is implemented using the CopyFile2 API, which preserves all stat\n" -"and metadata information apart from security attributes.\n" -"\n" -"ProgressRoutine is reserved for future use, but is currently not\n" -"implemented. Its value is ignored."); - -#define _WINAPI_COPYFILE2_METHODDEF \ - {"CopyFile2", _PyCFunction_CAST(_winapi_CopyFile2), METH_FASTCALL|METH_KEYWORDS, _winapi_CopyFile2__doc__}, - -static PyObject * -_winapi_CopyFile2_impl(PyObject *module, LPCWSTR ExistingFileName, - LPCWSTR NewFileName, DWORD Flags, - PyObject *ProgressRoutine); - -static PyObject * -_winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(ExistingFileName), &_Py_ID(NewFileName), &_Py_ID(Flags), &_Py_ID(ProgressRoutine), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"ExistingFileName", "NewFileName", "Flags", "ProgressRoutine", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .format = "O&O&k|O:CopyFile2", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - LPCWSTR ExistingFileName = NULL; - LPCWSTR NewFileName = NULL; - DWORD Flags; - PyObject *ProgressRoutine = Py_None; - - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - _PyUnicode_WideCharString_Converter, &ExistingFileName, _PyUnicode_WideCharString_Converter, &NewFileName, &Flags, &ProgressRoutine)) { - goto exit; - } - return_value = _winapi_CopyFile2_impl(module, ExistingFileName, NewFileName, Flags, ProgressRoutine); - -exit: - /* Cleanup for ExistingFileName */ - PyMem_Free((void *)ExistingFileName); - /* Cleanup for NewFileName */ - PyMem_Free((void *)NewFileName); - - return return_value; -} - PyDoc_STRVAR(_winapi__mimetypes_read_windows_registry__doc__, "_mimetypes_read_windows_registry($module, /, on_type_read)\n" "--\n" @@ -1481,4 +1411,74 @@ _winapi_NeedCurrentDirectoryForExePath(PyObject *module, PyObject *arg) return return_value; } -/*[clinic end generated code: output=5e8927548ec12266 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_winapi_CopyFile2__doc__, +"CopyFile2($module, /, ExistingFileName, NewFileName, Flags,\n" +" ProgressRoutine=None)\n" +"--\n" +"\n" +"Copies a file from one name to a new name.\n" +"\n" +"This is implemented using the CopyFile2 API, which preserves all stat\n" +"and metadata information apart from security attributes.\n" +"\n" +"ProgressRoutine is reserved for future use, but is currently not\n" +"implemented. Its value is ignored."); + +#define _WINAPI_COPYFILE2_METHODDEF \ + {"CopyFile2", _PyCFunction_CAST(_winapi_CopyFile2), METH_FASTCALL|METH_KEYWORDS, _winapi_CopyFile2__doc__}, + +static PyObject * +_winapi_CopyFile2_impl(PyObject *module, LPCWSTR ExistingFileName, + LPCWSTR NewFileName, DWORD Flags, + PyObject *ProgressRoutine); + +static PyObject * +_winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(ExistingFileName), &_Py_ID(NewFileName), &_Py_ID(Flags), &_Py_ID(ProgressRoutine), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"ExistingFileName", "NewFileName", "Flags", "ProgressRoutine", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "O&O&k|O:CopyFile2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + LPCWSTR ExistingFileName = NULL; + LPCWSTR NewFileName = NULL; + DWORD Flags; + PyObject *ProgressRoutine = Py_None; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + _PyUnicode_WideCharString_Converter, &ExistingFileName, _PyUnicode_WideCharString_Converter, &NewFileName, &Flags, &ProgressRoutine)) { + goto exit; + } + return_value = _winapi_CopyFile2_impl(module, ExistingFileName, NewFileName, Flags, ProgressRoutine); + +exit: + /* Cleanup for ExistingFileName */ + PyMem_Free((void *)ExistingFileName); + /* Cleanup for NewFileName */ + PyMem_Free((void *)NewFileName); + + return return_value; +} +/*[clinic end generated code: output=2c13b124ebdfbc43 input=a9049054013a1b77]*/ From 76f379a1b00ec359eb504cc3f93d3efdc0f5ce6b Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 29 May 2023 18:01:17 +0100 Subject: [PATCH 3/4] Remove dead code --- Lib/shutil.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/shutil.py b/Lib/shutil.py index 0d1e09845297b6..3f2864af517e7d 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -444,7 +444,6 @@ def copy2(src, dst, *, follow_symlinks=True): flags = _winapi.COPY_FILE_ALLOW_DECRYPTED_DESTINATION # for compat if not follow_symlinks: flags |= _winapi.COPY_FILE_COPY_SYMLINK - retry = False try: _winapi.CopyFile2(src_, dst_, flags) return dst From f829448606b446bc42d06cdd9c7b2d829cb19356 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 30 May 2023 10:18:46 +0100 Subject: [PATCH 4/4] Pythonize the names --- .../pycore_global_objects_fini_generated.h | 7 ++-- Include/internal/pycore_global_strings.h | 7 ++-- .../internal/pycore_runtime_init_generated.h | 7 ++-- .../internal/pycore_unicodeobject_generated.h | 21 +++++----- Modules/_winapi.c | 36 +++++++++++------- Modules/clinic/_winapi.c.h | 38 +++++++++---------- 6 files changed, 59 insertions(+), 57 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 1200e65335a55c..546ba6d4c5520f 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -570,14 +570,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ExistingFileName)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(FINISHED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(False)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(JSONDecodeError)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(NewFileName)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(PENDING)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ProgressRoutine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Py_Repr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(TextIOWrapper)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True)); @@ -923,6 +919,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(existing_file_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens)); @@ -1075,6 +1072,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(narg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_limit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(newline)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(newlines)); @@ -1129,6 +1127,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_handler)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_routine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(proto)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(protocol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps1)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 78a3683dd6c0e1..088bd96c756b49 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -58,14 +58,10 @@ struct _Py_global_strings { struct { STRUCT_FOR_ID(CANCELLED) - STRUCT_FOR_ID(ExistingFileName) STRUCT_FOR_ID(FINISHED) STRUCT_FOR_ID(False) - STRUCT_FOR_ID(Flags) STRUCT_FOR_ID(JSONDecodeError) - STRUCT_FOR_ID(NewFileName) STRUCT_FOR_ID(PENDING) - STRUCT_FOR_ID(ProgressRoutine) STRUCT_FOR_ID(Py_Repr) STRUCT_FOR_ID(TextIOWrapper) STRUCT_FOR_ID(True) @@ -411,6 +407,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) + STRUCT_FOR_ID(existing_file_name) STRUCT_FOR_ID(exp) STRUCT_FOR_ID(extend) STRUCT_FOR_ID(extra_tokens) @@ -563,6 +560,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(namespaces) STRUCT_FOR_ID(narg) STRUCT_FOR_ID(ndigits) + STRUCT_FOR_ID(new_file_name) STRUCT_FOR_ID(new_limit) STRUCT_FOR_ID(newline) STRUCT_FOR_ID(newlines) @@ -617,6 +615,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) STRUCT_FOR_ID(progress_handler) + STRUCT_FOR_ID(progress_routine) STRUCT_FOR_ID(proto) STRUCT_FOR_ID(protocol) STRUCT_FOR_ID(ps1) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 702c7ee5f7ef19..2963423f607108 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -564,14 +564,10 @@ extern "C" { #define _Py_str_identifiers_INIT { \ INIT_ID(CANCELLED), \ - INIT_ID(ExistingFileName), \ INIT_ID(FINISHED), \ INIT_ID(False), \ - INIT_ID(Flags), \ INIT_ID(JSONDecodeError), \ - INIT_ID(NewFileName), \ INIT_ID(PENDING), \ - INIT_ID(ProgressRoutine), \ INIT_ID(Py_Repr), \ INIT_ID(TextIOWrapper), \ INIT_ID(True), \ @@ -917,6 +913,7 @@ extern "C" { INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ + INIT_ID(existing_file_name), \ INIT_ID(exp), \ INIT_ID(extend), \ INIT_ID(extra_tokens), \ @@ -1069,6 +1066,7 @@ extern "C" { INIT_ID(namespaces), \ INIT_ID(narg), \ INIT_ID(ndigits), \ + INIT_ID(new_file_name), \ INIT_ID(new_limit), \ INIT_ID(newline), \ INIT_ID(newlines), \ @@ -1123,6 +1121,7 @@ extern "C" { INIT_ID(priority), \ INIT_ID(progress), \ INIT_ID(progress_handler), \ + INIT_ID(progress_routine), \ INIT_ID(proto), \ INIT_ID(protocol), \ INIT_ID(ps1), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 036a8115824261..9e13a9491b7da5 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -15,30 +15,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(CANCELLED); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(ExistingFileName); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(FINISHED); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(False); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(Flags); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(JSONDecodeError); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(NewFileName); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(PENDING); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(ProgressRoutine); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(Py_Repr); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1074,6 +1062,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(exception); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(existing_file_name); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(exp); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1530,6 +1521,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(ndigits); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(new_file_name); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(new_limit); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1692,6 +1686,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(progress_handler); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(progress_routine); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(proto); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Modules/_winapi.c b/Modules/_winapi.c index e69cdf0a8bda3d..bbc9facd227c9e 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -2080,42 +2080,50 @@ _winapi_NeedCurrentDirectoryForExePath_impl(PyObject *module, /*[clinic input] _winapi.CopyFile2 - ExistingFileName: LPCWSTR - NewFileName: LPCWSTR - Flags: DWORD - ProgressRoutine: object = None + existing_file_name: LPCWSTR + new_file_name: LPCWSTR + flags: DWORD + progress_routine: object = None Copies a file from one name to a new name. This is implemented using the CopyFile2 API, which preserves all stat and metadata information apart from security attributes. -ProgressRoutine is reserved for future use, but is currently not +progress_routine is reserved for future use, but is currently not implemented. Its value is ignored. [clinic start generated code]*/ static PyObject * -_winapi_CopyFile2_impl(PyObject *module, LPCWSTR ExistingFileName, - LPCWSTR NewFileName, DWORD Flags, - PyObject *ProgressRoutine) -/*[clinic end generated code: output=f3f588380da0000a input=13cf59b6e1a70fe6]*/ +_winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, + LPCWSTR new_file_name, DWORD flags, + PyObject *progress_routine) +/*[clinic end generated code: output=43d960d9df73d984 input=fb976b8d1492d130]*/ { HRESULT hr; COPYFILE2_EXTENDED_PARAMETERS params = { sizeof(COPYFILE2_EXTENDED_PARAMETERS) }; - params.dwCopyFlags = Flags; + + if (PySys_Audit("_winapi.CopyFile2", "uuI", + existing_file_name, new_file_name, flags) < 0) { + return NULL; + } + + params.dwCopyFlags = flags; /* For future implementation. We ignore the value for now so that users only have to test for 'CopyFile2' existing and not whether the additional parameter exists. - if (ProgressRoutine != Py_None) { + if (progress_routine != Py_None) { params.pProgressRoutine = _winapi_CopyFile2ProgressRoutine; - params.pvCallbackContext = Py_NewRef(ProgressRoutine); + params.pvCallbackContext = Py_NewRef(progress_routine); } */ Py_BEGIN_ALLOW_THREADS; - hr = CopyFile2(ExistingFileName, NewFileName, ¶ms); + hr = CopyFile2(existing_file_name, new_file_name, ¶ms); Py_END_ALLOW_THREADS; /* For future implementation. - Py_XDECREF(ProgressRoutine); + if (progress_routine != Py_None) { + Py_DECREF(progress_routine); + } */ if (FAILED(hr)) { if ((hr & 0xFFFF0000) == 0x80070000) { diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index a72a7224dbc596..3767b19d76db05 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -1413,8 +1413,8 @@ _winapi_NeedCurrentDirectoryForExePath(PyObject *module, PyObject *arg) } PyDoc_STRVAR(_winapi_CopyFile2__doc__, -"CopyFile2($module, /, ExistingFileName, NewFileName, Flags,\n" -" ProgressRoutine=None)\n" +"CopyFile2($module, /, existing_file_name, new_file_name, flags,\n" +" progress_routine=None)\n" "--\n" "\n" "Copies a file from one name to a new name.\n" @@ -1422,16 +1422,16 @@ PyDoc_STRVAR(_winapi_CopyFile2__doc__, "This is implemented using the CopyFile2 API, which preserves all stat\n" "and metadata information apart from security attributes.\n" "\n" -"ProgressRoutine is reserved for future use, but is currently not\n" +"progress_routine is reserved for future use, but is currently not\n" "implemented. Its value is ignored."); #define _WINAPI_COPYFILE2_METHODDEF \ {"CopyFile2", _PyCFunction_CAST(_winapi_CopyFile2), METH_FASTCALL|METH_KEYWORDS, _winapi_CopyFile2__doc__}, static PyObject * -_winapi_CopyFile2_impl(PyObject *module, LPCWSTR ExistingFileName, - LPCWSTR NewFileName, DWORD Flags, - PyObject *ProgressRoutine); +_winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, + LPCWSTR new_file_name, DWORD flags, + PyObject *progress_routine); static PyObject * _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1446,7 +1446,7 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(ExistingFileName), &_Py_ID(NewFileName), &_Py_ID(Flags), &_Py_ID(ProgressRoutine), }, + .ob_item = { &_Py_ID(existing_file_name), &_Py_ID(new_file_name), &_Py_ID(flags), &_Py_ID(progress_routine), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1455,30 +1455,30 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"ExistingFileName", "NewFileName", "Flags", "ProgressRoutine", NULL}; + static const char * const _keywords[] = {"existing_file_name", "new_file_name", "flags", "progress_routine", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .format = "O&O&k|O:CopyFile2", .kwtuple = KWTUPLE, }; #undef KWTUPLE - LPCWSTR ExistingFileName = NULL; - LPCWSTR NewFileName = NULL; - DWORD Flags; - PyObject *ProgressRoutine = Py_None; + LPCWSTR existing_file_name = NULL; + LPCWSTR new_file_name = NULL; + DWORD flags; + PyObject *progress_routine = Py_None; if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - _PyUnicode_WideCharString_Converter, &ExistingFileName, _PyUnicode_WideCharString_Converter, &NewFileName, &Flags, &ProgressRoutine)) { + _PyUnicode_WideCharString_Converter, &existing_file_name, _PyUnicode_WideCharString_Converter, &new_file_name, &flags, &progress_routine)) { goto exit; } - return_value = _winapi_CopyFile2_impl(module, ExistingFileName, NewFileName, Flags, ProgressRoutine); + return_value = _winapi_CopyFile2_impl(module, existing_file_name, new_file_name, flags, progress_routine); exit: - /* Cleanup for ExistingFileName */ - PyMem_Free((void *)ExistingFileName); - /* Cleanup for NewFileName */ - PyMem_Free((void *)NewFileName); + /* Cleanup for existing_file_name */ + PyMem_Free((void *)existing_file_name); + /* Cleanup for new_file_name */ + PyMem_Free((void *)new_file_name); return return_value; } -/*[clinic end generated code: output=2c13b124ebdfbc43 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=be1343b3759e0c96 input=a9049054013a1b77]*/