From 24eb52155dcfe1ae49f225e2eb8bb290b155c9e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:15:09 +0100 Subject: [PATCH 01/43] allow to use translated exception messages --- Include/internal/pycore_pyerrors.h | 19 +++++++++++++++++++ Python/errors.c | 17 +++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 02945f0e71a145..035c1df36ea020 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -130,6 +130,25 @@ PyAPI_FUNC(void) _PyErr_SetString( PyObject *exception, const char *string); +/* + * Similar to _PyErr_SetString but decodes 'string' into the current locale. + * + * Exceptions occurring in decoding take priority over the desired exception. + */ +extern void _PyErr_SetLocaleStringTstate( + PyThreadState *tstate, + PyObject *exception, + const char *string); + +/* + * Similar to PyErr_SetString but decodes 'string' into the current locale. + * + * Exceptions occurring in decoding take priority over the desired exception. + */ +extern void _PyErr_SetLocaleString( + PyObject *exception, + const char *string); + PyAPI_FUNC(PyObject*) _PyErr_Format( PyThreadState *tstate, PyObject *exception, diff --git a/Python/errors.c b/Python/errors.c index 7f3b4aabc432d7..825ad285bcac9a 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -301,6 +301,23 @@ PyErr_SetString(PyObject *exception, const char *string) _PyErr_SetString(tstate, exception, string); } +void +_PyErr_SetLocaleStringTstate(PyThreadState *tstate, PyObject *exception, + const char *string) +{ + PyObject *value = PyUnicode_DecodeLocale(string, "strict"); + if (value != NULL) { + _PyErr_SetObject(tstate, exception, value); + Py_DECREF(value); + } +} + +void +_PyErr_SetLocaleString(PyObject *exception, const char *string) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyErr_SetLocaleStringTstate(tstate, exception, string); +} PyObject* _Py_HOT_FUNCTION PyErr_Occurred(void) From 4c0f85bd968d7cb5361c40e857135af117c5f3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 12 Nov 2024 16:01:49 +0100 Subject: [PATCH 02/43] Use 'surrogateescape' handler instead of 'strict' handler. --- Python/errors.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/errors.c b/Python/errors.c index 825ad285bcac9a..e1f065671451a9 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -305,7 +305,7 @@ void _PyErr_SetLocaleStringTstate(PyThreadState *tstate, PyObject *exception, const char *string) { - PyObject *value = PyUnicode_DecodeLocale(string, "strict"); + PyObject *value = PyUnicode_DecodeLocale(string, "surrogateescape"); if (value != NULL) { _PyErr_SetObject(tstate, exception, value); Py_DECREF(value); From f432dc916a126fe8b8bdcaeb632058d2f45b8a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 20 Nov 2024 16:22:45 +0100 Subject: [PATCH 03/43] fix typos --- Include/internal/pycore_pyerrors.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 035c1df36ea020..37f1b7b98e4afd 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -131,7 +131,7 @@ PyAPI_FUNC(void) _PyErr_SetString( const char *string); /* - * Similar to _PyErr_SetString but decodes 'string' into the current locale. + * Similar to _PyErr_SetString but decodes 'string' from the current locale. * * Exceptions occurring in decoding take priority over the desired exception. */ @@ -141,7 +141,7 @@ extern void _PyErr_SetLocaleStringTstate( const char *string); /* - * Similar to PyErr_SetString but decodes 'string' into the current locale. + * Similar to PyErr_SetString but decodes 'string' from the current locale. * * Exceptions occurring in decoding take priority over the desired exception. */ From 48238fc8a5629793501be644289618d7f2266a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:52:29 +0100 Subject: [PATCH 04/43] Allow to set localized error string. --- Include/internal/pycore_pyerrors.h | 15 ++++----------- Include/pyerrors.h | 18 ++++++++++++++---- Python/errors.c | 15 +++++++++------ 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 37f1b7b98e4afd..7806a245cef4b2 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -133,22 +133,15 @@ PyAPI_FUNC(void) _PyErr_SetString( /* * Similar to _PyErr_SetString but decodes 'string' from the current locale. * - * Exceptions occurring in decoding take priority over the desired exception. + * Exceptions occurring in decoding take priority over the desired exception, + * in which case, this returns -1. Otherwise this returns 0 if the localized + * exception has been successfully set. */ -extern void _PyErr_SetLocaleStringTstate( +extern int _PyErr_SetLocaleString( PyThreadState *tstate, PyObject *exception, const char *string); -/* - * Similar to PyErr_SetString but decodes 'string' from the current locale. - * - * Exceptions occurring in decoding take priority over the desired exception. - */ -extern void _PyErr_SetLocaleString( - PyObject *exception, - const char *string); - PyAPI_FUNC(PyObject*) _PyErr_Format( PyThreadState *tstate, PyObject *exception, diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 5d0028c116e2d8..d814261abc17fe 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -8,10 +8,20 @@ extern "C" { PyAPI_FUNC(void) PyErr_SetNone(PyObject *); PyAPI_FUNC(void) PyErr_SetObject(PyObject *, PyObject *); -PyAPI_FUNC(void) PyErr_SetString( - PyObject *exception, - const char *string /* decoded from utf-8 */ - ); +/* + * Set an exception with the error message decoded from UTF-8. + * + * Exceptions occurring in decoding take priority over the desired exception. + */ +PyAPI_FUNC(void) PyErr_SetString(PyObject *exception, const char *string); +/* + * Set an exception with the error message decoded from the current locale. + * + * Exceptions occurring in decoding take priority over the desired exception, + * in which case, this returns -1. Otherwise this returns 0 if the localized + * exception has been successfully set. + */ +PyAPI_FUNC(int) PyErr_SetLocaleString(PyObject *, const char *string); PyAPI_FUNC(PyObject *) PyErr_Occurred(void); PyAPI_FUNC(void) PyErr_Clear(void); PyAPI_FUNC(void) PyErr_Fetch(PyObject **, PyObject **, PyObject **); diff --git a/Python/errors.c b/Python/errors.c index e1f065671451a9..ddfd7ebad73c7a 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -301,22 +301,25 @@ PyErr_SetString(PyObject *exception, const char *string) _PyErr_SetString(tstate, exception, string); } -void -_PyErr_SetLocaleStringTstate(PyThreadState *tstate, PyObject *exception, - const char *string) +int +_PyErr_SetLocaleString(PyThreadState *tstate, + PyObject *exception, + const char *string) { PyObject *value = PyUnicode_DecodeLocale(string, "surrogateescape"); if (value != NULL) { _PyErr_SetObject(tstate, exception, value); Py_DECREF(value); + return 0; } + return -1; } -void -_PyErr_SetLocaleString(PyObject *exception, const char *string) +int +PyErr_SetLocaleString(PyObject *exception, const char *string) { PyThreadState *tstate = _PyThreadState_GET(); - _PyErr_SetLocaleStringTstate(tstate, exception, string); + return _PyErr_SetLocaleString(tstate, exception, string); } PyObject* _Py_HOT_FUNCTION From a38105b6e48c0257baee414427c1304e00fdf410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:52:49 +0100 Subject: [PATCH 05/43] Use `PyErr_SetLocaleString` when possible --- Modules/_ctypes/_ctypes.c | 16 ++++---------- Modules/_ctypes/callproc.c | 41 ++++++++++++++++++++--------------- Modules/_gdbmmodule.c | 44 +++++++++++++++++++++++++++++--------- 3 files changed, 62 insertions(+), 39 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 34529bce496d88..6c193578c65a72 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -984,14 +984,10 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); - if (message) { - PyErr_SetObject(PyExc_ValueError, message); - Py_DECREF(message); + if (PyErr_SetLocaleString(PyExc_ValueError, dlerr) == 0) { return NULL; } - // Ignore errors from PyUnicode_DecodeLocale, - // fall back to the generic error below. + // Ignore decoding errors and fall back to the generic error. PyErr_Clear(); } #endif @@ -3808,15 +3804,11 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); - if (message) { - PyErr_SetObject(PyExc_AttributeError, message); + if (PyErr_SetLocaleString(PyExc_AttributeError, dlerr) == 0) { Py_DECREF(ftuple); - Py_DECREF(message); return NULL; } - // Ignore errors from PyUnicode_DecodeLocale, - // fall back to the generic error below. + // Ignore decoding errors and fall back to the generic error. PyErr_Clear(); } #endif diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 218c3a9c81e05f..0d6d2e3b2fed6f 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1588,10 +1588,14 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) Py_XDECREF(name2); if (!handle) { const char *errmsg = dlerror(); - if (!errmsg) - errmsg = "dlopen() error"; - PyErr_SetString(PyExc_OSError, - errmsg); + if (errmsg) { + if (PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { + return NULL; + } + // Ignore decoding errors and fall back to the generic error. + PyErr_Clear(); + } + PyErr_SetString(PyExc_OSError, "dlopen() error"); return NULL; } return PyLong_FromVoidPtr(handle); @@ -1604,8 +1608,15 @@ static PyObject *py_dl_close(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O&:dlclose", &_parse_voidp, &handle)) return NULL; if (dlclose(handle)) { - PyErr_SetString(PyExc_OSError, - dlerror()); + const char *errmsg = dlerror(); + if (errmsg) { + if (PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { + return NULL; + } + // Ignore decoding errors and fall back to the generic error. + PyErr_Clear(); + } + PyErr_SetString(PyExc_OSError, "dlclose() error"); return NULL; } Py_RETURN_NONE; @@ -1639,21 +1650,17 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) if (ptr) { return PyLong_FromVoidPtr(ptr); } - #ifdef USE_DLERROR - const char *dlerr = dlerror(); - if (dlerr) { - PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); - if (message) { - PyErr_SetObject(PyExc_OSError, message); - Py_DECREF(message); + #ifdef USE_DLERROR + const char *errmsg = dlerror(); + if (errmsg) { + if (PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { return NULL; } - // Ignore errors from PyUnicode_DecodeLocale, - // fall back to the generic error below. + // Ignore decoding errors and fall back to the generic error below. PyErr_Clear(); } - #endif - #undef USE_DLERROR + #endif + #undef USE_DLERROR PyErr_Format(PyExc_OSError, "symbol '%s' not found", name); return NULL; } diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index df7fba67810ed0..61e752110726dc 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -33,6 +33,27 @@ get_gdbm_state(PyObject *module) return (_gdbm_state *)state; } +/* + * Set the gdbm error obtained by gdbm_strerror(gdbm_errno). + * + * If the error message cannot be decoded from the current locale + * or if none exists, a generic (UTF-8) error message is used. + */ +static void +set_gdbm_error(_gdbm_state *state, const char *generic_error) +{ + const char *gdbm_errmsg = gdbm_strerror(gdbm_errno); + if (gdbm_errmsg) { + if (PyErr_SetLocaleString(state->gdbm_error, gdbm_errmsg) == 0) { + return; + } + // Ignore decoding errors and fall back to the generic error. + PyErr_Clear(); + } + assert(!PyErr_Occurred()); + PyErr_SetString(state->gdbm_error, generic_error); +} + /*[clinic input] module _gdbm class _gdbm.gdbm "gdbmobject *" "&Gdbmtype" @@ -91,7 +112,7 @@ newgdbmobject(_gdbm_state *state, const char *file, int flags, int mode) PyErr_SetFromErrnoWithFilename(state->gdbm_error, file); } else { - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + set_gdbm_error(state, "gdbm_open() error"); } Py_DECREF(dp); return NULL; @@ -136,7 +157,7 @@ gdbm_length(gdbmobject *dp) PyErr_SetFromErrno(state->gdbm_error); } else { - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + set_gdbm_error(state, "gdbm_count() error"); } return -1; } @@ -286,7 +307,7 @@ gdbm_ass_sub(gdbmobject *dp, PyObject *v, PyObject *w) PyErr_SetObject(PyExc_KeyError, v); } else { - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + set_gdbm_error(state, "gdbm_delete() error"); } return -1; } @@ -297,11 +318,12 @@ gdbm_ass_sub(gdbmobject *dp, PyObject *v, PyObject *w) } errno = 0; if (gdbm_store(dp->di_dbm, krec, drec, GDBM_REPLACE) < 0) { - if (errno != 0) + if (errno != 0) { PyErr_SetFromErrno(state->gdbm_error); - else - PyErr_SetString(state->gdbm_error, - gdbm_strerror(gdbm_errno)); + } + else { + set_gdbm_error(state, "gdbm_store() error"); + } return -1; } } @@ -534,10 +556,12 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls) check_gdbmobject_open(self, state->gdbm_error); errno = 0; if (gdbm_reorganize(self->di_dbm) < 0) { - if (errno != 0) + if (errno != 0) { PyErr_SetFromErrno(state->gdbm_error); - else - PyErr_SetString(state->gdbm_error, gdbm_strerror(gdbm_errno)); + } + else { + set_gdbm_error(state, "gdbm_reorganize() error"); + } return NULL; } Py_RETURN_NONE; From 714dcb745054c11505cc3050cd3de6f8ab659265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:53:51 +0100 Subject: [PATCH 06/43] Indicate which external functions may return non-UTF8 error strings. --- Modules/_hashopenssl.c | 1 + Modules/_sqlite/util.c | 2 +- Modules/main.c | 1 + Modules/pyexpat.c | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 2c9a9feecc79f0..082929be3c77b7 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -319,6 +319,7 @@ _setException(PyObject *exc, const char* altmsg, ...) va_end(vargs); ERR_clear_error(); + /* ERR_ERROR_STRING(3) ensures that the messages below are ASCII */ lib = ERR_lib_error_string(errcode); func = ERR_func_error_string(errcode); reason = ERR_reason_error_string(errcode); diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 9e8613ef67916e..d6d70b1f5450c8 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -134,7 +134,7 @@ _pysqlite_seterror(pysqlite_state *state, sqlite3 *db) /* Create and set the exception. */ int extended_errcode = sqlite3_extended_errcode(db); - const char *errmsg = sqlite3_errmsg(db); + const char *errmsg = sqlite3_errmsg(db); // UTf-8 raise_exception(exc_class, extended_errcode, errmsg); return extended_errcode; } diff --git a/Modules/main.c b/Modules/main.c index 15ea49a1bad19e..5031512be80661 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -374,6 +374,7 @@ pymain_run_file_obj(PyObject *program_name, PyObject *filename, if (fp == NULL) { // Ignore the OSError PyErr_Clear(); + // XXX: strerror() is locale dependent but not PySys_FormatStderr(). PySys_FormatStderr("%S: can't open file %R: [Errno %d] %s\n", program_name, filename, errno, strerror(errno)); return 2; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 9733bc34f7c80a..40c0afb2e06c5c 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1783,6 +1783,7 @@ add_error(PyObject *errors_module, PyObject *codes_dict, * elsewhere within this file. pyexpat's copy of the messages * only acts as a fallback in case of outdated runtime libexpat, * where it returns NULL. */ + // XXX: verify that XML_ErrorString() returns UTF-8 strings only const char *error_string = XML_ErrorString(error_code); if (error_string == NULL) { error_string = error_info_of[error_index].description; From 3d71f6bbe29e54053eec06f59cff8dd870a0b12b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 29 Nov 2024 13:31:39 +0100 Subject: [PATCH 07/43] fix build --- Doc/data/refcounts.dat | 4 ++++ Doc/data/stable_abi.dat | 1 + Lib/test/test_stable_abi_ctypes.py | 1 + Misc/stable_abi.toml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 6bfcc191b2270b..c566fc3a9db87c 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -722,6 +722,10 @@ PyErr_SetImportErrorSubclass:PyObject*:path:+1: PyErr_SetInterrupt:void::: +PyErr_SetLocaleString:int::: +PyErr_SetLocaleString:PyObject*:type:+1: +PyErr_SetLocaleString:const char*:message:: + PyErr_SetNone:void::: PyErr_SetNone:PyObject*:type:+1: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 6f9d27297e8f65..aaf647c93b0c35 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -171,6 +171,7 @@ func,PyErr_SetImportError,3.7,, func,PyErr_SetImportErrorSubclass,3.6,, func,PyErr_SetInterrupt,3.2,, func,PyErr_SetInterruptEx,3.10,, +func,PyErr_SetLocaleString,3.14,, func,PyErr_SetNone,3.2,, func,PyErr_SetObject,3.2,, func,PyErr_SetRaisedException,3.12,, diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index fa08dc6a25b0ea..f19e5872f526ef 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -206,6 +206,7 @@ def test_windows_feature_macros(self): "PyErr_SetImportErrorSubclass", "PyErr_SetInterrupt", "PyErr_SetInterruptEx", + "PyErr_SetLocaleString", "PyErr_SetNone", "PyErr_SetObject", "PyErr_SetRaisedException", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index f9e51f0683c965..d8bf3b0a8082bf 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2540,3 +2540,5 @@ added = '3.14' [function.PyType_Freeze] added = '3.14' +[function.PyErr_SetLocaleString] + added = '3.14' From 1a23bf51d0c3a56947c38b165f70920c88112afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 29 Nov 2024 13:31:48 +0100 Subject: [PATCH 08/43] update dll --- PC/python3dll.c | 1 + 1 file changed, 1 insertion(+) diff --git a/PC/python3dll.c b/PC/python3dll.c index 8657ddb9fa5155..a1f462bd3b68f0 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -234,6 +234,7 @@ EXPORT_FUNC(PyErr_SetImportError) EXPORT_FUNC(PyErr_SetImportErrorSubclass) EXPORT_FUNC(PyErr_SetInterrupt) EXPORT_FUNC(PyErr_SetInterruptEx) +EXPORT_FUNC(PyErr_SetLocaleString) EXPORT_FUNC(PyErr_SetNone) EXPORT_FUNC(PyErr_SetObject) EXPORT_FUNC(PyErr_SetRaisedException) From 16a98a290b2b633b193cb487345e1215b9ac3bb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 29 Nov 2024 13:44:32 +0100 Subject: [PATCH 09/43] add docs & news --- Doc/c-api/exceptions.rst | 13 +++++++++++++ Doc/whatsnew/3.14.rst | 5 +++++ .../2024-11-29-13-38-46.gh-issue-126742.jP5vYV.rst | 2 ++ 3 files changed, 20 insertions(+) create mode 100644 Misc/NEWS.d/next/C_API/2024-11-29-13-38-46.gh-issue-126742.jP5vYV.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index fc2336d120c259..6a2f3630bd0022 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -136,6 +136,19 @@ For convenience, some of these functions will always return a The second argument is an error message; it is decoded from ``'utf-8'``. +.. c:function:: int PyErr_SetLocaleString(PyObject *type, const char *message) + + Similar to :c:func:`PyErr_SetString` but decodes the *message* from the + current locale via :c:func:`PyUnicode_DecodeLocale` configured with the + :ref:`surrogateescape` error handler. + + Exceptions occurring in decoding take priority over the desired exception, + in which case, this returns -1. Otherwise this returns 0 if the localized + exception has been successfully set. + + .. versionadded:: next + + .. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) This function is similar to :c:func:`PyErr_SetString` but lets you specify an diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 869a47c1261293..da2981d8812528 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -995,6 +995,11 @@ New features * Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling deferred reference counting, as outlined in :pep:`703`. +* Add :c:func:`PyErr_SetLocaleString` to allow defining a locale-dependent + exception message. + (Contributed by Bénédikt Tran in :gh:`126742`.) + + Porting to Python 3.14 ---------------------- diff --git a/Misc/NEWS.d/next/C_API/2024-11-29-13-38-46.gh-issue-126742.jP5vYV.rst b/Misc/NEWS.d/next/C_API/2024-11-29-13-38-46.gh-issue-126742.jP5vYV.rst new file mode 100644 index 00000000000000..1dbbb632c61923 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-11-29-13-38-46.gh-issue-126742.jP5vYV.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyErr_SetLocaleString` to allow defining a locale-dependent +exception message. Patch by Bénédikt Tran. From 5a150d21bd7b5483ae2519615ea1b4ffe8cdaccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:04:25 +0100 Subject: [PATCH 10/43] update comment --- Modules/pyexpat.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 40c0afb2e06c5c..cf7714e7656205 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1782,8 +1782,12 @@ add_error(PyObject *errors_module, PyObject *codes_dict, * with the other uses of the XML_ErrorString function * elsewhere within this file. pyexpat's copy of the messages * only acts as a fallback in case of outdated runtime libexpat, - * where it returns NULL. */ - // XXX: verify that XML_ErrorString() returns UTF-8 strings only + * where it returns NULL. + * + * In addition, XML_ErrorString is assumed to return UTF-8 encoded + * strings (in conv_string_to_unicode, we decode them using 'strict' + * error handling). + */ const char *error_string = XML_ErrorString(error_code); if (error_string == NULL) { error_string = error_info_of[error_index].description; From 074bd4cf599e160a22f4e0fc67d2c7e8fa2a27f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:05:07 +0100 Subject: [PATCH 11/43] update TODO --- Modules/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/main.c b/Modules/main.c index 5031512be80661..3bf2241f2837a3 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -374,7 +374,7 @@ pymain_run_file_obj(PyObject *program_name, PyObject *filename, if (fp == NULL) { // Ignore the OSError PyErr_Clear(); - // XXX: strerror() is locale dependent but not PySys_FormatStderr(). + // TODO(picnixz): strerror() is locale dependent but not PySys_FormatStderr(). PySys_FormatStderr("%S: can't open file %R: [Errno %d] %s\n", program_name, filename, errno, strerror(errno)); return 2; From 7f3a9093dc68e2521871d1d20c6e2327eddcdfd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:07:11 +0100 Subject: [PATCH 12/43] update comment --- Modules/_sqlite/util.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index d6d70b1f5450c8..b0622e66928f47 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -134,7 +134,8 @@ _pysqlite_seterror(pysqlite_state *state, sqlite3 *db) /* Create and set the exception. */ int extended_errcode = sqlite3_extended_errcode(db); - const char *errmsg = sqlite3_errmsg(db); // UTf-8 + // sqlite3_errmsg() always returns an UTF-8 encoded message + const char *errmsg = sqlite3_errmsg(db); raise_exception(exc_class, extended_errcode, errmsg); return extended_errcode; } From 1a321f949d7b25d98b71ea909151f0b7d4ed3fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:51:35 +0100 Subject: [PATCH 13/43] Revert "add docs & news" This reverts commit 16a98a290b2b633b193cb487345e1215b9ac3bb8. --- Doc/c-api/exceptions.rst | 13 ------------- Doc/whatsnew/3.14.rst | 5 ----- .../2024-11-29-13-38-46.gh-issue-126742.jP5vYV.rst | 2 -- 3 files changed, 20 deletions(-) delete mode 100644 Misc/NEWS.d/next/C_API/2024-11-29-13-38-46.gh-issue-126742.jP5vYV.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 6a2f3630bd0022..fc2336d120c259 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -136,19 +136,6 @@ For convenience, some of these functions will always return a The second argument is an error message; it is decoded from ``'utf-8'``. -.. c:function:: int PyErr_SetLocaleString(PyObject *type, const char *message) - - Similar to :c:func:`PyErr_SetString` but decodes the *message* from the - current locale via :c:func:`PyUnicode_DecodeLocale` configured with the - :ref:`surrogateescape` error handler. - - Exceptions occurring in decoding take priority over the desired exception, - in which case, this returns -1. Otherwise this returns 0 if the localized - exception has been successfully set. - - .. versionadded:: next - - .. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) This function is similar to :c:func:`PyErr_SetString` but lets you specify an diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index da2981d8812528..869a47c1261293 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -995,11 +995,6 @@ New features * Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling deferred reference counting, as outlined in :pep:`703`. -* Add :c:func:`PyErr_SetLocaleString` to allow defining a locale-dependent - exception message. - (Contributed by Bénédikt Tran in :gh:`126742`.) - - Porting to Python 3.14 ---------------------- diff --git a/Misc/NEWS.d/next/C_API/2024-11-29-13-38-46.gh-issue-126742.jP5vYV.rst b/Misc/NEWS.d/next/C_API/2024-11-29-13-38-46.gh-issue-126742.jP5vYV.rst deleted file mode 100644 index 1dbbb632c61923..00000000000000 --- a/Misc/NEWS.d/next/C_API/2024-11-29-13-38-46.gh-issue-126742.jP5vYV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :c:func:`PyErr_SetLocaleString` to allow defining a locale-dependent -exception message. Patch by Bénédikt Tran. From f5813a9d060c9ced3aed9be77489366aeb5aed18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:58:58 +0100 Subject: [PATCH 14/43] keep one internal helper instead of a public/private API pair --- Doc/data/stable_abi.dat | 1 - Include/internal/pycore_pyerrors.h | 9 ++++----- Include/pyerrors.h | 18 ++++-------------- Lib/test/test_stable_abi_ctypes.py | 1 - Misc/stable_abi.toml | 2 -- PC/python3dll.c | 1 - Python/errors.c | 12 ++---------- 7 files changed, 10 insertions(+), 34 deletions(-) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index aaf647c93b0c35..6f9d27297e8f65 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -171,7 +171,6 @@ func,PyErr_SetImportError,3.7,, func,PyErr_SetImportErrorSubclass,3.6,, func,PyErr_SetInterrupt,3.2,, func,PyErr_SetInterruptEx,3.10,, -func,PyErr_SetLocaleString,3.14,, func,PyErr_SetNone,3.2,, func,PyErr_SetObject,3.2,, func,PyErr_SetRaisedException,3.12,, diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 7806a245cef4b2..929fc0b7ecd659 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -131,16 +131,15 @@ PyAPI_FUNC(void) _PyErr_SetString( const char *string); /* - * Similar to _PyErr_SetString but decodes 'string' from the current locale. + * Set an exception with the error message decoded from the current locale. * * Exceptions occurring in decoding take priority over the desired exception, * in which case, this returns -1. Otherwise this returns 0 if the localized * exception has been successfully set. + * + * Exported for shared extensions but not part of the stable ABI. */ -extern int _PyErr_SetLocaleString( - PyThreadState *tstate, - PyObject *exception, - const char *string); +PyAPI_FUNC(int) _PyErr_SetLocaleString(PyObject *exception, const char *string); PyAPI_FUNC(PyObject*) _PyErr_Format( PyThreadState *tstate, diff --git a/Include/pyerrors.h b/Include/pyerrors.h index d814261abc17fe..5d0028c116e2d8 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -8,20 +8,10 @@ extern "C" { PyAPI_FUNC(void) PyErr_SetNone(PyObject *); PyAPI_FUNC(void) PyErr_SetObject(PyObject *, PyObject *); -/* - * Set an exception with the error message decoded from UTF-8. - * - * Exceptions occurring in decoding take priority over the desired exception. - */ -PyAPI_FUNC(void) PyErr_SetString(PyObject *exception, const char *string); -/* - * Set an exception with the error message decoded from the current locale. - * - * Exceptions occurring in decoding take priority over the desired exception, - * in which case, this returns -1. Otherwise this returns 0 if the localized - * exception has been successfully set. - */ -PyAPI_FUNC(int) PyErr_SetLocaleString(PyObject *, const char *string); +PyAPI_FUNC(void) PyErr_SetString( + PyObject *exception, + const char *string /* decoded from utf-8 */ + ); PyAPI_FUNC(PyObject *) PyErr_Occurred(void); PyAPI_FUNC(void) PyErr_Clear(void); PyAPI_FUNC(void) PyErr_Fetch(PyObject **, PyObject **, PyObject **); diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index f19e5872f526ef..fa08dc6a25b0ea 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -206,7 +206,6 @@ def test_windows_feature_macros(self): "PyErr_SetImportErrorSubclass", "PyErr_SetInterrupt", "PyErr_SetInterruptEx", - "PyErr_SetLocaleString", "PyErr_SetNone", "PyErr_SetObject", "PyErr_SetRaisedException", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index d8bf3b0a8082bf..f9e51f0683c965 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2540,5 +2540,3 @@ added = '3.14' [function.PyType_Freeze] added = '3.14' -[function.PyErr_SetLocaleString] - added = '3.14' diff --git a/PC/python3dll.c b/PC/python3dll.c index a1f462bd3b68f0..8657ddb9fa5155 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -234,7 +234,6 @@ EXPORT_FUNC(PyErr_SetImportError) EXPORT_FUNC(PyErr_SetImportErrorSubclass) EXPORT_FUNC(PyErr_SetInterrupt) EXPORT_FUNC(PyErr_SetInterruptEx) -EXPORT_FUNC(PyErr_SetLocaleString) EXPORT_FUNC(PyErr_SetNone) EXPORT_FUNC(PyErr_SetObject) EXPORT_FUNC(PyErr_SetRaisedException) diff --git a/Python/errors.c b/Python/errors.c index ddfd7ebad73c7a..81207fc61a3e2d 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -302,10 +302,9 @@ PyErr_SetString(PyObject *exception, const char *string) } int -_PyErr_SetLocaleString(PyThreadState *tstate, - PyObject *exception, - const char *string) +_PyErr_SetLocaleString(PyObject *exception, const char *string) { + PyThreadState *tstate = _PyThreadState_GET(); PyObject *value = PyUnicode_DecodeLocale(string, "surrogateescape"); if (value != NULL) { _PyErr_SetObject(tstate, exception, value); @@ -315,13 +314,6 @@ _PyErr_SetLocaleString(PyThreadState *tstate, return -1; } -int -PyErr_SetLocaleString(PyObject *exception, const char *string) -{ - PyThreadState *tstate = _PyThreadState_GET(); - return _PyErr_SetLocaleString(tstate, exception, string); -} - PyObject* _Py_HOT_FUNCTION PyErr_Occurred(void) { From 84afad2d04bfcd0ade77bddc3f0876da67509053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:00:40 +0100 Subject: [PATCH 15/43] rename references to `PyErr_SetLocaleString` --- Doc/data/refcounts.dat | 6 +++--- Modules/_ctypes/_ctypes.c | 4 ++-- Modules/_ctypes/callproc.c | 6 +++--- Modules/_gdbmmodule.c | 5 +++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index c566fc3a9db87c..ea3ea439ac4abf 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -722,9 +722,9 @@ PyErr_SetImportErrorSubclass:PyObject*:path:+1: PyErr_SetInterrupt:void::: -PyErr_SetLocaleString:int::: -PyErr_SetLocaleString:PyObject*:type:+1: -PyErr_SetLocaleString:const char*:message:: +_PyErr_SetLocaleString:int::: +_PyErr_SetLocaleString:PyObject*:type:+1: +_PyErr_SetLocaleString:const char*:message:: PyErr_SetNone:void::: PyErr_SetNone:PyObject*:type:+1: diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 6c193578c65a72..ab987966ffc392 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -984,7 +984,7 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - if (PyErr_SetLocaleString(PyExc_ValueError, dlerr) == 0) { + if (_PyErr_SetLocaleString(PyExc_ValueError, dlerr) == 0) { return NULL; } // Ignore decoding errors and fall back to the generic error. @@ -3804,7 +3804,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - if (PyErr_SetLocaleString(PyExc_AttributeError, dlerr) == 0) { + if (_PyErr_SetLocaleString(PyExc_AttributeError, dlerr) == 0) { Py_DECREF(ftuple); return NULL; } diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 0d6d2e3b2fed6f..5427c6de76fe11 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1589,7 +1589,7 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) if (!handle) { const char *errmsg = dlerror(); if (errmsg) { - if (PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { + if (_PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { return NULL; } // Ignore decoding errors and fall back to the generic error. @@ -1610,7 +1610,7 @@ static PyObject *py_dl_close(PyObject *self, PyObject *args) if (dlclose(handle)) { const char *errmsg = dlerror(); if (errmsg) { - if (PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { + if (_PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { return NULL; } // Ignore decoding errors and fall back to the generic error. @@ -1653,7 +1653,7 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) #ifdef USE_DLERROR const char *errmsg = dlerror(); if (errmsg) { - if (PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { + if (_PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { return NULL; } // Ignore decoding errors and fall back to the generic error below. diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index 61e752110726dc..a26616209dd17d 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -8,10 +8,11 @@ #endif #include "Python.h" +#include "pycore_pyerrors.h" // _PyErr_SetLocaleString() #include "gdbm.h" #include -#include // free() +#include // free() #include #include @@ -44,7 +45,7 @@ set_gdbm_error(_gdbm_state *state, const char *generic_error) { const char *gdbm_errmsg = gdbm_strerror(gdbm_errno); if (gdbm_errmsg) { - if (PyErr_SetLocaleString(state->gdbm_error, gdbm_errmsg) == 0) { + if (_PyErr_SetLocaleString(state->gdbm_error, gdbm_errmsg) == 0) { return; } // Ignore decoding errors and fall back to the generic error. From 166a73647035bad8eaef297f8b479cc80b783fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:02:40 +0100 Subject: [PATCH 16/43] Simplify logic --- Python/errors.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/errors.c b/Python/errors.c index 81207fc61a3e2d..757b8bffb85081 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -304,10 +304,9 @@ PyErr_SetString(PyObject *exception, const char *string) int _PyErr_SetLocaleString(PyObject *exception, const char *string) { - PyThreadState *tstate = _PyThreadState_GET(); PyObject *value = PyUnicode_DecodeLocale(string, "surrogateescape"); if (value != NULL) { - _PyErr_SetObject(tstate, exception, value); + PyErr_SetObject(exception, value); Py_DECREF(value); return 0; } From 314c9f1901e0646ddc8fce6eb02b415b85172421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 2 Dec 2024 17:52:20 +0100 Subject: [PATCH 17/43] Remove entry from refcounts.dat. We will add internal functions later in one go instead one-by-one. --- Doc/data/refcounts.dat | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 11ef1fd1f30495..3f49c88c3cc028 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -724,10 +724,6 @@ PyErr_SetImportErrorSubclass:PyObject*:path:+1: PyErr_SetInterrupt:void::: -_PyErr_SetLocaleString:int::: -_PyErr_SetLocaleString:PyObject*:type:+1: -_PyErr_SetLocaleString:const char*:message:: - PyErr_SetNone:void::: PyErr_SetNone:PyObject*:type:+1: From 571de05c1733388675d86fc4948099e066de90e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 2 Dec 2024 17:52:59 +0100 Subject: [PATCH 18/43] update comment --- Include/internal/pycore_pyerrors.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 929fc0b7ecd659..7a924468546b26 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -137,7 +137,7 @@ PyAPI_FUNC(void) _PyErr_SetString( * in which case, this returns -1. Otherwise this returns 0 if the localized * exception has been successfully set. * - * Exported for shared extensions but not part of the stable ABI. + * Exported for '_ctypes' shared extensions. */ PyAPI_FUNC(int) _PyErr_SetLocaleString(PyObject *exception, const char *string); From 2633d186b6d4f066887bdfca76e1ad1bdc81bed8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 2 Dec 2024 18:18:46 +0100 Subject: [PATCH 19/43] Update Include/internal/pycore_pyerrors.h Co-authored-by: Victor Stinner --- Include/internal/pycore_pyerrors.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 7a924468546b26..234964e1a2bf09 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -131,7 +131,8 @@ PyAPI_FUNC(void) _PyErr_SetString( const char *string); /* - * Set an exception with the error message decoded from the current locale. + * Set an exception with the error message decoded from the current locale + * encoding (LC_CTYPE). * * Exceptions occurring in decoding take priority over the desired exception, * in which case, this returns -1. Otherwise this returns 0 if the localized From 45442c312f3ca2108b8cd38b4bfcaba63d89e508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 2 Dec 2024 18:20:04 +0100 Subject: [PATCH 20/43] fix grammar --- Include/internal/pycore_pyerrors.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 234964e1a2bf09..f386f4f7e3dc45 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -134,9 +134,9 @@ PyAPI_FUNC(void) _PyErr_SetString( * Set an exception with the error message decoded from the current locale * encoding (LC_CTYPE). * - * Exceptions occurring in decoding take priority over the desired exception, - * in which case, this returns -1. Otherwise this returns 0 if the localized - * exception has been successfully set. + * Exceptions occurring in decoding take priority over the desired exception. + * In those cases, this function returns -1. Otherwise this returns 0 if the + * localized exception has been successfully set. * * Exported for '_ctypes' shared extensions. */ From 6f9ee0b37b70509144d7d546e0a7811da62a4398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 8 Dec 2024 14:38:30 +0100 Subject: [PATCH 21/43] Unconditionally re-raise decoding errors --- Include/internal/pycore_pyerrors.h | 6 +++--- Modules/_ctypes/_ctypes.c | 20 +++++++------------- Modules/_ctypes/callproc.c | 21 ++++++--------------- Modules/_gdbmmodule.c | 15 ++++++--------- Python/errors.c | 4 +--- 5 files changed, 23 insertions(+), 43 deletions(-) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index f386f4f7e3dc45..6f2fdda9a9f12f 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -135,12 +135,12 @@ PyAPI_FUNC(void) _PyErr_SetString( * encoding (LC_CTYPE). * * Exceptions occurring in decoding take priority over the desired exception. - * In those cases, this function returns -1. Otherwise this returns 0 if the - * localized exception has been successfully set. * * Exported for '_ctypes' shared extensions. */ -PyAPI_FUNC(int) _PyErr_SetLocaleString(PyObject *exception, const char *string); +PyAPI_FUNC(void) _PyErr_SetLocaleString( + PyObject *exception, + const char *string); PyAPI_FUNC(PyObject*) _PyErr_Format( PyThreadState *tstate, diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index ab987966ffc392..e8472cd16646f7 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -984,11 +984,8 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - if (_PyErr_SetLocaleString(PyExc_ValueError, dlerr) == 0) { - return NULL; - } - // Ignore decoding errors and fall back to the generic error. - PyErr_Clear(); + _PyErr_SetLocaleString(PyExc_ValueError, dlerr); + return NULL; } #endif #undef USE_DLERROR @@ -3801,17 +3798,14 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) address = (PPROC)dlsym(handle, name); if (!address) { - #ifdef USE_DLERROR + #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - if (_PyErr_SetLocaleString(PyExc_AttributeError, dlerr) == 0) { - Py_DECREF(ftuple); - return NULL; - } - // Ignore decoding errors and fall back to the generic error. - PyErr_Clear(); + _PyErr_SetLocaleString(PyExc_AttributeError, dlerr); + Py_DECREF(ftuple); + return NULL; } - #endif + #endif PyErr_Format(PyExc_AttributeError, "function '%s' not found", name); Py_DECREF(ftuple); return NULL; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 5427c6de76fe11..92eedff5ec94f1 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1589,11 +1589,8 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) if (!handle) { const char *errmsg = dlerror(); if (errmsg) { - if (_PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { - return NULL; - } - // Ignore decoding errors and fall back to the generic error. - PyErr_Clear(); + _PyErr_SetLocaleString(PyExc_OSError, errmsg); + return NULL; } PyErr_SetString(PyExc_OSError, "dlopen() error"); return NULL; @@ -1610,11 +1607,8 @@ static PyObject *py_dl_close(PyObject *self, PyObject *args) if (dlclose(handle)) { const char *errmsg = dlerror(); if (errmsg) { - if (_PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { - return NULL; - } - // Ignore decoding errors and fall back to the generic error. - PyErr_Clear(); + _PyErr_SetLocaleString(PyExc_OSError, errmsg); + return NULL; } PyErr_SetString(PyExc_OSError, "dlclose() error"); return NULL; @@ -1653,11 +1647,8 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) #ifdef USE_DLERROR const char *errmsg = dlerror(); if (errmsg) { - if (_PyErr_SetLocaleString(PyExc_OSError, errmsg) == 0) { - return NULL; - } - // Ignore decoding errors and fall back to the generic error below. - PyErr_Clear(); + _PyErr_SetLocaleString(PyExc_OSError, errmsg); + return NULL; } #endif #undef USE_DLERROR diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index a26616209dd17d..ea4fe247987e9d 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -37,22 +37,19 @@ get_gdbm_state(PyObject *module) /* * Set the gdbm error obtained by gdbm_strerror(gdbm_errno). * - * If the error message cannot be decoded from the current locale - * or if none exists, a generic (UTF-8) error message is used. + * If no error message exists, a generic (UTF-8) error message + * is used instead. */ static void set_gdbm_error(_gdbm_state *state, const char *generic_error) { const char *gdbm_errmsg = gdbm_strerror(gdbm_errno); if (gdbm_errmsg) { - if (_PyErr_SetLocaleString(state->gdbm_error, gdbm_errmsg) == 0) { - return; - } - // Ignore decoding errors and fall back to the generic error. - PyErr_Clear(); + _PyErr_SetLocaleString(state->gdbm_error, gdbm_errmsg); + } + else { + PyErr_SetString(state->gdbm_error, generic_error); } - assert(!PyErr_Occurred()); - PyErr_SetString(state->gdbm_error, generic_error); } /*[clinic input] diff --git a/Python/errors.c b/Python/errors.c index 757b8bffb85081..2d362c1864ffff 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -301,16 +301,14 @@ PyErr_SetString(PyObject *exception, const char *string) _PyErr_SetString(tstate, exception, string); } -int +void _PyErr_SetLocaleString(PyObject *exception, const char *string) { PyObject *value = PyUnicode_DecodeLocale(string, "surrogateescape"); if (value != NULL) { PyErr_SetObject(exception, value); Py_DECREF(value); - return 0; } - return -1; } PyObject* _Py_HOT_FUNCTION From b02a715fb71b2e3b39b2ece459160554b33d0880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:33:09 +0100 Subject: [PATCH 22/43] add tests for `_PyErr_SetLocaleString` --- Lib/test/test_capi/test_exceptions.py | 17 +++++++++++++ Modules/_testcapi/clinic/exceptions.c.h | 33 ++++++++++++++++++++++++- Modules/_testcapi/exceptions.c | 23 +++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index b22ddd8ad858d4..31ef47cf34962f 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -256,6 +256,23 @@ def test_set_string(self): # CRASHES setstring(ZeroDivisionError, NULL) # CRASHES setstring(NULL, b'error') + def test_set_locale_string(self): + # test _PyErr_SetLocaleString() + setlocalestring = _testcapi.err_setlocalestring + with self.assertRaises(ZeroDivisionError) as e: + setlocalestring(ZeroDivisionError, b'error') + self.assertEqual(e.exception.args, ('error',)) + with self.assertRaises(ZeroDivisionError) as e: + setlocalestring(ZeroDivisionError, 'помилка'.encode()) + self.assertEqual(e.exception.args, ('помилка',)) + + with self.assertRaises(ZeroDivisionError): + setlocalestring(ZeroDivisionError, b'\xff') + self.assertRaises(SystemError, setlocalestring, list, b'error') + + # CRASHES setlocalestring(ZeroDivisionError, NULL) + # CRASHES setlocalestring(NULL, b'error') + def test_format(self): """Test PyErr_Format()""" import_helper.import_module('ctypes') diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index 7c18bac0f686e2..7232436b15d7de 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -246,6 +246,37 @@ _testcapi_err_setstring(PyObject *module, PyObject *const *args, Py_ssize_t narg return return_value; } +PyDoc_STRVAR(_testcapi_err_setlocalestring__doc__, +"err_setlocalestring($module, exc, value, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_ERR_SETLOCALESTRING_METHODDEF \ + {"err_setlocalestring", _PyCFunction_CAST(_testcapi_err_setlocalestring), METH_FASTCALL, _testcapi_err_setlocalestring__doc__}, + +static PyObject * +_testcapi_err_setlocalestring_impl(PyObject *module, PyObject *exc, + const char *value, + Py_ssize_t value_length); + +static PyObject * +_testcapi_err_setlocalestring(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *exc; + const char *value; + Py_ssize_t value_length; + + if (!_PyArg_ParseStack(args, nargs, "Oz#:err_setlocalestring", + &exc, &value, &value_length)) { + goto exit; + } + return_value = _testcapi_err_setlocalestring_impl(module, exc, value, value_length); + +exit: + return return_value; +} + PyDoc_STRVAR(_testcapi_err_setfromerrnowithfilename__doc__, "err_setfromerrnowithfilename($module, error, exc, value, /)\n" "--\n" @@ -457,4 +488,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=d917e9ec082e69ee input=a9049054013a1b77]*/ +/*[clinic end generated code: output=51b21bb9bec436bf input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index 316ef0e7ad7e55..a3d331da680edc 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -3,6 +3,9 @@ #include "parts.h" #include "util.h" + +#include "pycore_pyerrors.h" + #include "clinic/exceptions.c.h" @@ -154,6 +157,25 @@ _testcapi_err_setstring_impl(PyObject *module, PyObject *exc, return NULL; } +/*[clinic input] +_testcapi.err_setlocalestring + exc: object + value: str(zeroes=True, accept={robuffer, str, NoneType}) + / +[clinic start generated code]*/ + +static PyObject * +_testcapi_err_setlocalestring_impl(PyObject *module, PyObject *exc, + const char *value, + Py_ssize_t value_length) +/*[clinic end generated code: output=5c02a10d4eb573d7 input=628c26bdc2497a92]*/ +{ + NULLABLE(exc) + _PyErr_SetLocaleString(exc, value); + return NULL; +} + + /*[clinic input] _testcapi.err_setfromerrnowithfilename error: int @@ -396,6 +418,7 @@ static PyMethodDef test_methods[] = { _TESTCAPI_EXC_SET_OBJECT_METHODDEF _TESTCAPI_EXC_SET_OBJECT_FETCH_METHODDEF _TESTCAPI_ERR_SETSTRING_METHODDEF + _TESTCAPI_ERR_SETLOCALESTRING_METHODDEF _TESTCAPI_ERR_SETFROMERRNOWITHFILENAME_METHODDEF _TESTCAPI_RAISE_EXCEPTION_METHODDEF _TESTCAPI_RAISE_MEMORYERROR_METHODDEF From 47d50b885d3daaa167465391e297ac4b11ae5168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:59:53 +0100 Subject: [PATCH 23/43] add tests for `dlerror` and `gdbm_*` functions --- Lib/test/test_ctypes/test_dlerror.py | 18 +++++++++++++++++- Lib/test/test_dbm_gnu.py | 9 ++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 4441e30cd7a2a7..66b8c517f59299 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,7 +1,11 @@ +import _ctypes import os +import platform +import re import sys +import test.support import unittest -import platform + FOO_C = r""" #include @@ -119,5 +123,17 @@ def test_null_dlsym(self): self.assertEqual(os.read(pipe_r, 2), b'OK') +class TestCAPI(unittest.TestCase): + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), 'require ctypes.dlopen()') + @test.support.run_with_locales('LC_ALL', 'fr_FR.utf8', 'fr_FR.iso88591') + def test_localized_error(self): + with self.assertRaisesRegex( + OSError, + re.escape("foo.so: Ne peut ouvrir le fichier d'objet partagé"), + ): + _ctypes.dlopen('foo.so', 2) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index e20addf1f04f1b..69b9444342015f 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -1,9 +1,11 @@ from test import support from test.support import import_helper, cpython_only gdbm = import_helper.import_module("dbm.gnu") #skip if not supported +import re import unittest import os -from test.support.os_helper import TESTFN, TESTFN_NONASCII, unlink, FakePath +from test.support.os_helper import (TESTFN, TESTFN_NONASCII, + create_empty_file, unlink, FakePath) filename = TESTFN @@ -205,6 +207,11 @@ def test_clear(self): self.assertNotIn(k, db) self.assertEqual(len(db), 0) + @support.run_with_locales('LC_ALL', 'fr_FR.UTF-8', 'fr_FR.iso88591') + def test_localized_error(self): + expect = re.escape('Base de données vide') + empty = create_empty_file(filename) + self.assertRaisesRegex(gdbm.error, expect, gdbm.open, filename, 'r') if __name__ == '__main__': From 574184c5b1ada841f823af568e6e14071d0de1b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:14:12 +0100 Subject: [PATCH 24/43] fix tests --- Lib/test/test_ctypes/test_dlerror.py | 30 ++++++++++++++++++---------- Lib/test/test_dbm_gnu.py | 18 ++++++++++------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 66b8c517f59299..2e4b6b4fb3913f 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,4 +1,3 @@ -import _ctypes import os import platform import re @@ -123,16 +122,27 @@ def test_null_dlsym(self): self.assertEqual(os.read(pipe_r, 2), b'OK') -class TestCAPI(unittest.TestCase): - - @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), 'require ctypes.dlopen()') - @test.support.run_with_locales('LC_ALL', 'fr_FR.utf8', 'fr_FR.iso88591') +@unittest.skipUnless(sys.platform.startswith('linux'), + 'test requires _ctypes.dlopen()') +class TestLinuxLocalization(unittest.TestCase): + + @test.support.run_with_locale( + 'LC_ALL', + 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', + '', + ) def test_localized_error(self): - with self.assertRaisesRegex( - OSError, - re.escape("foo.so: Ne peut ouvrir le fichier d'objet partagé"), - ): - _ctypes.dlopen('foo.so', 2) + # An ImportError would be propagated and would be unexpected on Linux. + from _ctypes import dlopen + + missing_filename = b'missing\xff.so' + # Depending whether the locale is ISO-88591 or not, we may either + # encode '\xff' as '\udcff' or 'ÿ', but we are only interested in + # avoiding a UnicodeDecodeError when reporting the dlerror() error + # message which contains the localized filename. + filename_pattern = r'missing[ÿ|\udcff].so:.+' + with self.assertRaisesRegex(OSError, filename_pattern): + dlopen(missing_filename, 2) if __name__ == "__main__": diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index 69b9444342015f..3a7a56379e864b 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -1,11 +1,10 @@ from test import support from test.support import import_helper, cpython_only gdbm = import_helper.import_module("dbm.gnu") #skip if not supported -import re import unittest import os -from test.support.os_helper import (TESTFN, TESTFN_NONASCII, - create_empty_file, unlink, FakePath) +from test.support.os_helper import (TESTFN, TESTFN_NONASCII, FakePath, + create_empty_file, temp_dir, unlink) filename = TESTFN @@ -207,11 +206,16 @@ def test_clear(self): self.assertNotIn(k, db) self.assertEqual(len(db), 0) - @support.run_with_locales('LC_ALL', 'fr_FR.UTF-8', 'fr_FR.iso88591') + @support.run_with_locale( + 'LC_ALL', + 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', + '', + ) def test_localized_error(self): - expect = re.escape('Base de données vide') - empty = create_empty_file(filename) - self.assertRaisesRegex(gdbm.error, expect, gdbm.open, filename, 'r') + with temp_dir() as d: + filename = os.path.join(os.fsencode(d), b'\xff') + create_empty_file(filename) + self.assertRaises(gdbm.error, gdbm.open, filename, 'r') if __name__ == '__main__': From 4fecd766006f376483bc4f8b2aad4faaa775cbaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:28:10 +0100 Subject: [PATCH 25/43] remove C API internal tests --- Lib/test/test_capi/test_exceptions.py | 17 ------------- Modules/_testcapi/clinic/exceptions.c.h | 33 +------------------------ Modules/_testcapi/exceptions.c | 19 -------------- 3 files changed, 1 insertion(+), 68 deletions(-) diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index 31ef47cf34962f..b22ddd8ad858d4 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -256,23 +256,6 @@ def test_set_string(self): # CRASHES setstring(ZeroDivisionError, NULL) # CRASHES setstring(NULL, b'error') - def test_set_locale_string(self): - # test _PyErr_SetLocaleString() - setlocalestring = _testcapi.err_setlocalestring - with self.assertRaises(ZeroDivisionError) as e: - setlocalestring(ZeroDivisionError, b'error') - self.assertEqual(e.exception.args, ('error',)) - with self.assertRaises(ZeroDivisionError) as e: - setlocalestring(ZeroDivisionError, 'помилка'.encode()) - self.assertEqual(e.exception.args, ('помилка',)) - - with self.assertRaises(ZeroDivisionError): - setlocalestring(ZeroDivisionError, b'\xff') - self.assertRaises(SystemError, setlocalestring, list, b'error') - - # CRASHES setlocalestring(ZeroDivisionError, NULL) - # CRASHES setlocalestring(NULL, b'error') - def test_format(self): """Test PyErr_Format()""" import_helper.import_module('ctypes') diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index 7232436b15d7de..7c18bac0f686e2 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -246,37 +246,6 @@ _testcapi_err_setstring(PyObject *module, PyObject *const *args, Py_ssize_t narg return return_value; } -PyDoc_STRVAR(_testcapi_err_setlocalestring__doc__, -"err_setlocalestring($module, exc, value, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_ERR_SETLOCALESTRING_METHODDEF \ - {"err_setlocalestring", _PyCFunction_CAST(_testcapi_err_setlocalestring), METH_FASTCALL, _testcapi_err_setlocalestring__doc__}, - -static PyObject * -_testcapi_err_setlocalestring_impl(PyObject *module, PyObject *exc, - const char *value, - Py_ssize_t value_length); - -static PyObject * -_testcapi_err_setlocalestring(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *exc; - const char *value; - Py_ssize_t value_length; - - if (!_PyArg_ParseStack(args, nargs, "Oz#:err_setlocalestring", - &exc, &value, &value_length)) { - goto exit; - } - return_value = _testcapi_err_setlocalestring_impl(module, exc, value, value_length); - -exit: - return return_value; -} - PyDoc_STRVAR(_testcapi_err_setfromerrnowithfilename__doc__, "err_setfromerrnowithfilename($module, error, exc, value, /)\n" "--\n" @@ -488,4 +457,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=51b21bb9bec436bf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d917e9ec082e69ee input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index a3d331da680edc..b6c3593dc4a5d9 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -157,24 +157,6 @@ _testcapi_err_setstring_impl(PyObject *module, PyObject *exc, return NULL; } -/*[clinic input] -_testcapi.err_setlocalestring - exc: object - value: str(zeroes=True, accept={robuffer, str, NoneType}) - / -[clinic start generated code]*/ - -static PyObject * -_testcapi_err_setlocalestring_impl(PyObject *module, PyObject *exc, - const char *value, - Py_ssize_t value_length) -/*[clinic end generated code: output=5c02a10d4eb573d7 input=628c26bdc2497a92]*/ -{ - NULLABLE(exc) - _PyErr_SetLocaleString(exc, value); - return NULL; -} - /*[clinic input] _testcapi.err_setfromerrnowithfilename @@ -418,7 +400,6 @@ static PyMethodDef test_methods[] = { _TESTCAPI_EXC_SET_OBJECT_METHODDEF _TESTCAPI_EXC_SET_OBJECT_FETCH_METHODDEF _TESTCAPI_ERR_SETSTRING_METHODDEF - _TESTCAPI_ERR_SETLOCALESTRING_METHODDEF _TESTCAPI_ERR_SETFROMERRNOWITHFILENAME_METHODDEF _TESTCAPI_RAISE_EXCEPTION_METHODDEF _TESTCAPI_RAISE_MEMORYERROR_METHODDEF From 26f074c64b4a96c974313253ce296e26fdc5682e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 12 Dec 2024 12:15:31 +0100 Subject: [PATCH 26/43] cosmetic changes on imports --- Lib/test/test_dbm_gnu.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index 3a7a56379e864b..5f2a4bbb079d3d 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -1,11 +1,11 @@ -from test import support -from test.support import import_helper, cpython_only -gdbm = import_helper.import_module("dbm.gnu") #skip if not supported -import unittest import os +import unittest +from test import support +from test.support import cpython_only, import_helper from test.support.os_helper import (TESTFN, TESTFN_NONASCII, FakePath, create_empty_file, temp_dir, unlink) +gdbm = import_helper.import_module("dbm.gnu") # skip if not supported filename = TESTFN From cb4d5e49276ef88e32ecaca893219955d56e0e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:17:44 +0100 Subject: [PATCH 27/43] fix tests --- Lib/test/test_dbm_gnu.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index 5f2a4bbb079d3d..661b3da3eceb2d 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -213,8 +213,7 @@ def test_clear(self): ) def test_localized_error(self): with temp_dir() as d: - filename = os.path.join(os.fsencode(d), b'\xff') - create_empty_file(filename) + create_empty_file(os.path.join(d, TESTFN)) self.assertRaises(gdbm.error, gdbm.open, filename, 'r') From 6798f213b3f79d290dfb839e12b4517b04c4e81f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 14 Dec 2024 13:25:24 +0100 Subject: [PATCH 28/43] fix `dbm` tests --- Lib/test/test_dbm_gnu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index 661b3da3eceb2d..66268c42a300b5 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -209,11 +209,12 @@ def test_clear(self): @support.run_with_locale( 'LC_ALL', 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', + 'fr_FR.utf8', 'en_US.utf8', '', ) def test_localized_error(self): with temp_dir() as d: - create_empty_file(os.path.join(d, TESTFN)) + create_empty_file(os.path.join(d, 'test')) self.assertRaises(gdbm.error, gdbm.open, filename, 'r') From ce8ae02f588051d1d8043dc463f6839867a4fec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 14 Dec 2024 13:25:27 +0100 Subject: [PATCH 29/43] improve test coverage --- Lib/test/test_ctypes/test_dlerror.py | 121 ++++++++++++++++++--------- 1 file changed, 80 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 2e4b6b4fb3913f..3db2357e4be443 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,9 +1,12 @@ +import _ctypes + import os import platform -import re -import sys +import subprocess +import tempfile import test.support import unittest +from ctypes import CDLL, c_int FOO_C = r""" @@ -28,8 +31,16 @@ """ -@unittest.skipUnless(sys.platform.startswith('linux'), - 'Test only valid for Linux') +def has_gcc(): + return subprocess.call(["gcc", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) == 0 + + +@unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') +@unittest.skipUnless(hasattr(_ctypes, 'dlsym'), + 'test requires _ctypes.dlsym()') class TestNullDlsym(unittest.TestCase): """GH-126554: Ensure that we catch NULL dlsym return values @@ -53,21 +64,7 @@ class TestNullDlsym(unittest.TestCase): """ def test_null_dlsym(self): - import subprocess - import tempfile - - # To avoid ImportErrors on Windows, where _ctypes does not have - # dlopen and dlsym, - # import here, i.e., inside the test function. - # The skipUnless('linux') decorator ensures that we're on linux - # if we're executing these statements. - from ctypes import CDLL, c_int - from _ctypes import dlopen, dlsym - - retcode = subprocess.call(["gcc", "--version"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) - if retcode != 0: + if not has_gcc(): self.skipTest("gcc is missing") pipe_r, pipe_w = os.pipe() @@ -114,35 +111,77 @@ def test_null_dlsym(self): self.assertEqual(os.read(pipe_r, 2), b'OK') # Case #3: Test 'py_dl_sym' from Modules/_ctypes/callproc.c - L = dlopen(dstname) + L = _ctypes.dlopen(dstname) with self.assertRaisesRegex(OSError, "symbol 'foo' not found"): - dlsym(L, "foo") - - # Assert that the IFUNC was called - self.assertEqual(os.read(pipe_r, 2), b'OK') + _ctypes.dlsym(L, "foo") -@unittest.skipUnless(sys.platform.startswith('linux'), - 'test requires _ctypes.dlopen()') class TestLinuxLocalization(unittest.TestCase): - @test.support.run_with_locale( - 'LC_ALL', - 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', - '', - ) - def test_localized_error(self): - # An ImportError would be propagated and would be unexpected on Linux. - from _ctypes import dlopen - + @staticmethod + def configure_locales(func): + return test.support.run_with_locale( + 'LC_ALL', + 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', + 'fr_FR.utf8', 'en_US.utf8', + '', + )(func) + + @classmethod + def setUpClass(cls): + if not has_gcc(): + raise unittest.SkipTest("gcc is missing") + + def make_libfoo(self, outdir, source, so_libname): + srcname = os.path.join(outdir, 'source.c') + dstname = os.path.join(outdir, so_libname) + with open(srcname, 'w') as f: + f.write(source) + args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] + p = subprocess.run(args, capture_output=True) + p.check_returncode() + return dstname + + @configure_locales + def test_localized_error_from_dll(self): + with tempfile.TemporaryDirectory() as outdir: + dstname = self.make_libfoo(outdir, 'int x = 0;', 'test_in_dll.so') + dll = CDLL(dstname) + with self.assertRaisesRegex(AttributeError, r'test_in_dll\.so:.+'): + dll.foo + + @configure_locales + def test_localized_error_in_dll(self): + with tempfile.TemporaryDirectory() as outdir: + dstname = self.make_libfoo(outdir, 'int x = 0;', 'test_in_dll.so') + dll = CDLL(dstname) + with self.assertRaisesRegex(ValueError, r'test_in_dll\.so:.+'): + c_int.in_dll(dll, 'foo') + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') + @configure_locales + def test_localized_error_dlopen(self): missing_filename = b'missing\xff.so' - # Depending whether the locale is ISO-88591 or not, we may either - # encode '\xff' as '\udcff' or 'ÿ', but we are only interested in - # avoiding a UnicodeDecodeError when reporting the dlerror() error - # message which contains the localized filename. - filename_pattern = r'missing[ÿ|\udcff].so:.+' + # Depending whether the locale, we may encode '\xff' differently + # but we are only interested in avoiding a UnicodeDecodeError + # when reporting the dlerror() error message which contains + # the localized filename. + filename_pattern = r'missing.*?\.so:.+' with self.assertRaisesRegex(OSError, filename_pattern): - dlopen(missing_filename, 2) + _ctypes.dlopen(missing_filename, 2) + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') + @unittest.skipUnless(hasattr(_ctypes, 'dlsym'), + 'test requires _ctypes.dlsym()') + @configure_locales + def test_localized_error_dlsym(self): + with tempfile.TemporaryDirectory() as outdir: + dstname = self.make_libfoo(outdir, 'int x = 0;', 'test_dlsym.so') + dll = _ctypes.dlopen(dstname) + with self.assertRaisesRegex(OSError, r'test_dlsym\.so:.+'): + _ctypes.dlsym(dll, 'foo') if __name__ == "__main__": From 8d442a3ef89fe2f20b5bd1793d49dca25e194f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 14 Dec 2024 13:40:40 +0100 Subject: [PATCH 30/43] fix macOS tests --- Lib/test/test_ctypes/test_dlerror.py | 32 +++++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 3db2357e4be443..ecfb47f812b999 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,12 +1,13 @@ import _ctypes - import os import platform import subprocess +import sys import tempfile import test.support import unittest from ctypes import CDLL, c_int +from test.support.os_helper import create_empty_file FOO_C = r""" @@ -116,7 +117,7 @@ def test_null_dlsym(self): _ctypes.dlsym(L, "foo") -class TestLinuxLocalization(unittest.TestCase): +class TestLocalization(unittest.TestCase): @staticmethod def configure_locales(func): @@ -132,11 +133,10 @@ def setUpClass(cls): if not has_gcc(): raise unittest.SkipTest("gcc is missing") - def make_libfoo(self, outdir, source, so_libname): - srcname = os.path.join(outdir, 'source.c') + def make_libfoo(self, outdir, so_libname): + srcname = os.path.join(outdir, 'empty.c') dstname = os.path.join(outdir, so_libname) - with open(srcname, 'w') as f: - f.write(source) + create_empty_file(srcname) args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] p = subprocess.run(args, capture_output=True) p.check_returncode() @@ -145,17 +145,21 @@ def make_libfoo(self, outdir, source, so_libname): @configure_locales def test_localized_error_from_dll(self): with tempfile.TemporaryDirectory() as outdir: - dstname = self.make_libfoo(outdir, 'int x = 0;', 'test_in_dll.so') + dstname = self.make_libfoo(outdir, 'test_from_dll.so') dll = CDLL(dstname) - with self.assertRaisesRegex(AttributeError, r'test_in_dll\.so:.+'): + # on macOS, the filename is not reported by dlerror() + pat = '.+' if sys.platform == 'darwin' else r'test_from_dll\.so' + with self.assertRaisesRegex(AttributeError, pat): dll.foo @configure_locales def test_localized_error_in_dll(self): with tempfile.TemporaryDirectory() as outdir: - dstname = self.make_libfoo(outdir, 'int x = 0;', 'test_in_dll.so') + dstname = self.make_libfoo(outdir, 'test_in_dll.so') dll = CDLL(dstname) - with self.assertRaisesRegex(ValueError, r'test_in_dll\.so:.+'): + # on macOS, the filename is not reported by dlerror() + pat = '.+' if sys.platform == 'darwin' else r'test_in_dll\.so' + with self.assertRaisesRegex(ValueError, pat): c_int.in_dll(dll, 'foo') @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), @@ -167,7 +171,7 @@ def test_localized_error_dlopen(self): # but we are only interested in avoiding a UnicodeDecodeError # when reporting the dlerror() error message which contains # the localized filename. - filename_pattern = r'missing.*?\.so:.+' + filename_pattern = r'missing.*?\.so' with self.assertRaisesRegex(OSError, filename_pattern): _ctypes.dlopen(missing_filename, 2) @@ -178,9 +182,11 @@ def test_localized_error_dlopen(self): @configure_locales def test_localized_error_dlsym(self): with tempfile.TemporaryDirectory() as outdir: - dstname = self.make_libfoo(outdir, 'int x = 0;', 'test_dlsym.so') + dstname = self.make_libfoo(outdir, 'test_dlsym.so') dll = _ctypes.dlopen(dstname) - with self.assertRaisesRegex(OSError, r'test_dlsym\.so:.+'): + # on macOS, the filename is not reported by dlerror() + pat = '.+' if sys.platform == 'darwin' else r'test_dlsym\.so' + with self.assertRaisesRegex(OSError, pat): _ctypes.dlsym(dll, 'foo') From f06d6a58a2ba6a0fd4f1b996055e0e83296c7bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 14 Dec 2024 13:42:10 +0100 Subject: [PATCH 31/43] update names --- Lib/test/test_ctypes/test_dlerror.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index ecfb47f812b999..eba23fd2ea5159 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -133,7 +133,7 @@ def setUpClass(cls): if not has_gcc(): raise unittest.SkipTest("gcc is missing") - def make_libfoo(self, outdir, so_libname): + def make_empty_lib(self, outdir, so_libname): srcname = os.path.join(outdir, 'empty.c') dstname = os.path.join(outdir, so_libname) create_empty_file(srcname) @@ -145,7 +145,7 @@ def make_libfoo(self, outdir, so_libname): @configure_locales def test_localized_error_from_dll(self): with tempfile.TemporaryDirectory() as outdir: - dstname = self.make_libfoo(outdir, 'test_from_dll.so') + dstname = self.make_empty_lib(outdir, 'test_from_dll.so') dll = CDLL(dstname) # on macOS, the filename is not reported by dlerror() pat = '.+' if sys.platform == 'darwin' else r'test_from_dll\.so' @@ -155,7 +155,7 @@ def test_localized_error_from_dll(self): @configure_locales def test_localized_error_in_dll(self): with tempfile.TemporaryDirectory() as outdir: - dstname = self.make_libfoo(outdir, 'test_in_dll.so') + dstname = self.make_empty_lib(outdir, 'test_in_dll.so') dll = CDLL(dstname) # on macOS, the filename is not reported by dlerror() pat = '.+' if sys.platform == 'darwin' else r'test_in_dll\.so' @@ -182,7 +182,7 @@ def test_localized_error_dlopen(self): @configure_locales def test_localized_error_dlsym(self): with tempfile.TemporaryDirectory() as outdir: - dstname = self.make_libfoo(outdir, 'test_dlsym.so') + dstname = self.make_empty_lib(outdir, 'test_dlsym.so') dll = _ctypes.dlopen(dstname) # on macOS, the filename is not reported by dlerror() pat = '.+' if sys.platform == 'darwin' else r'test_dlsym\.so' From 1c05250c14cb1d68f0b97dc1e5b849a38ba71187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 14 Dec 2024 14:29:32 +0100 Subject: [PATCH 32/43] fix tests --- Lib/test/test_ctypes/test_dlerror.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index eba23fd2ea5159..f0cdd2f6b7689a 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -147,20 +147,22 @@ def test_localized_error_from_dll(self): with tempfile.TemporaryDirectory() as outdir: dstname = self.make_empty_lib(outdir, 'test_from_dll.so') dll = CDLL(dstname) - # on macOS, the filename is not reported by dlerror() - pat = '.+' if sys.platform == 'darwin' else r'test_from_dll\.so' - with self.assertRaisesRegex(AttributeError, pat): + with self.assertRaises(AttributeError) as cm: dll.foo + if sys.platform.startswith('linux'): + # On macOS or Windows, the filename is not reported by dlerror() + self.assertIn('test_from_dll.so', str(cm.exception)) @configure_locales def test_localized_error_in_dll(self): with tempfile.TemporaryDirectory() as outdir: dstname = self.make_empty_lib(outdir, 'test_in_dll.so') dll = CDLL(dstname) - # on macOS, the filename is not reported by dlerror() - pat = '.+' if sys.platform == 'darwin' else r'test_in_dll\.so' - with self.assertRaisesRegex(ValueError, pat): + with self.assertRaises(ValueError) as cm: c_int.in_dll(dll, 'foo') + if sys.platform.startswith('linux'): + # On macOS or Windows, the filename is not reported by dlerror() + self.assertIn('test_in_dll.so', str(cm.exception)) @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), 'test requires _ctypes.dlopen()') @@ -184,10 +186,11 @@ def test_localized_error_dlsym(self): with tempfile.TemporaryDirectory() as outdir: dstname = self.make_empty_lib(outdir, 'test_dlsym.so') dll = _ctypes.dlopen(dstname) - # on macOS, the filename is not reported by dlerror() - pat = '.+' if sys.platform == 'darwin' else r'test_dlsym\.so' - with self.assertRaisesRegex(OSError, pat): + with self.assertRaises(OSError) as cm: _ctypes.dlsym(dll, 'foo') + if sys.platform.startswith('linux'): + # On macOS or Windows, the filename is not reported by dlerror() + self.assertIn('test_in_dll.so', str(cm.exception)) if __name__ == "__main__": From 3d69de1c844aa85858aadf894f48a88bb93e890d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 14 Dec 2024 14:54:05 +0100 Subject: [PATCH 33/43] fix tests (again???) --- Lib/test/test_ctypes/test_dlerror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index f0cdd2f6b7689a..120c0f7e3e7d19 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -190,7 +190,7 @@ def test_localized_error_dlsym(self): _ctypes.dlsym(dll, 'foo') if sys.platform.startswith('linux'): # On macOS or Windows, the filename is not reported by dlerror() - self.assertIn('test_in_dll.so', str(cm.exception)) + self.assertIn('test_dlsym.so', str(cm.exception)) if __name__ == "__main__": From a07951161fcb8a6b31abb68e2b7c1ce4cef7fa87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 14 Dec 2024 14:56:27 +0100 Subject: [PATCH 34/43] fix tests (macOS) --- Lib/test/test_ctypes/test_dlerror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 120c0f7e3e7d19..eb3b7b099ae533 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -137,7 +137,7 @@ def make_empty_lib(self, outdir, so_libname): srcname = os.path.join(outdir, 'empty.c') dstname = os.path.join(outdir, so_libname) create_empty_file(srcname) - args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] + args = ['gcc', '-shared', '-o', dstname, srcname] p = subprocess.run(args, capture_output=True) p.check_returncode() return dstname From 8618de8e364150faaf2bea19e170fbf0aabda03c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 14 Dec 2024 14:57:35 +0100 Subject: [PATCH 35/43] fix tests (windows) --- Lib/test/test_ctypes/test_dlerror.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index eb3b7b099ae533..2274d6665a5809 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -7,7 +7,7 @@ import test.support import unittest from ctypes import CDLL, c_int -from test.support.os_helper import create_empty_file +from test.support.os_helper import create_empty_file, temp_dir FOO_C = r""" @@ -144,7 +144,7 @@ def make_empty_lib(self, outdir, so_libname): @configure_locales def test_localized_error_from_dll(self): - with tempfile.TemporaryDirectory() as outdir: + with temp_dir() as outdir: dstname = self.make_empty_lib(outdir, 'test_from_dll.so') dll = CDLL(dstname) with self.assertRaises(AttributeError) as cm: @@ -155,7 +155,7 @@ def test_localized_error_from_dll(self): @configure_locales def test_localized_error_in_dll(self): - with tempfile.TemporaryDirectory() as outdir: + with temp_dir() as outdir: dstname = self.make_empty_lib(outdir, 'test_in_dll.so') dll = CDLL(dstname) with self.assertRaises(ValueError) as cm: @@ -183,7 +183,7 @@ def test_localized_error_dlopen(self): 'test requires _ctypes.dlsym()') @configure_locales def test_localized_error_dlsym(self): - with tempfile.TemporaryDirectory() as outdir: + with temp_dir() as outdir: dstname = self.make_empty_lib(outdir, 'test_dlsym.so') dll = _ctypes.dlopen(dstname) with self.assertRaises(OSError) as cm: From 9bc4012193604fa3b210b369b639392ec3ab1f5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 14 Dec 2024 15:14:04 +0100 Subject: [PATCH 36/43] fix macOS tests --- Lib/test/test_ctypes/test_dlerror.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 2274d6665a5809..897b5aa4da1b9d 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -38,10 +38,8 @@ def has_gcc(): stderr=subprocess.DEVNULL) == 0 -@unittest.skipUnless(hasattr(_ctypes, 'dlopen'), - 'test requires _ctypes.dlopen()') -@unittest.skipUnless(hasattr(_ctypes, 'dlsym'), - 'test requires _ctypes.dlsym()') +@unittest.skipUnless(sys.platform.startswith('linux'), + 'test requires GNU IFUNC support') class TestNullDlsym(unittest.TestCase): """GH-126554: Ensure that we catch NULL dlsym return values From 1ea0f733d647fc3ea6aed8032ebc8d487498e6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 14 Dec 2024 15:16:51 +0100 Subject: [PATCH 37/43] fix windows tests --- Lib/test/test_ctypes/test_dlerror.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 897b5aa4da1b9d..f9c33f8bbc2aab 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -115,6 +115,7 @@ def test_null_dlsym(self): _ctypes.dlsym(L, "foo") +@unittest.skipUnless(os.name != 'nt', 'test requires dlerror() calls') class TestLocalization(unittest.TestCase): @staticmethod @@ -148,7 +149,7 @@ def test_localized_error_from_dll(self): with self.assertRaises(AttributeError) as cm: dll.foo if sys.platform.startswith('linux'): - # On macOS or Windows, the filename is not reported by dlerror() + # On macOS, the filename is not reported by dlerror(). self.assertIn('test_from_dll.so', str(cm.exception)) @configure_locales @@ -159,7 +160,7 @@ def test_localized_error_in_dll(self): with self.assertRaises(ValueError) as cm: c_int.in_dll(dll, 'foo') if sys.platform.startswith('linux'): - # On macOS or Windows, the filename is not reported by dlerror() + # On macOS, the filename is not reported by dlerror(). self.assertIn('test_in_dll.so', str(cm.exception)) @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), @@ -187,7 +188,7 @@ def test_localized_error_dlsym(self): with self.assertRaises(OSError) as cm: _ctypes.dlsym(dll, 'foo') if sys.platform.startswith('linux'): - # On macOS or Windows, the filename is not reported by dlerror() + # On macOS, the filename is not reported by dlerror(). self.assertIn('test_dlsym.so', str(cm.exception)) From 09d60f44609734c51825a39d9c5d80b3f49365f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 15 Dec 2024 11:03:12 +0100 Subject: [PATCH 38/43] revert a missing line --- Lib/test/test_ctypes/test_dlerror.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index f9c33f8bbc2aab..744beb51aaa040 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -114,6 +114,9 @@ def test_null_dlsym(self): with self.assertRaisesRegex(OSError, "symbol 'foo' not found"): _ctypes.dlsym(L, "foo") + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + @unittest.skipUnless(os.name != 'nt', 'test requires dlerror() calls') class TestLocalization(unittest.TestCase): From a5722b1710caf0121b77cdba64dd0c9a053477f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 15 Dec 2024 13:24:13 +0100 Subject: [PATCH 39/43] revert changes --- Modules/_testcapi/exceptions.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index b6c3593dc4a5d9..b884eb7e4360ee 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -4,8 +4,6 @@ #include "parts.h" #include "util.h" -#include "pycore_pyerrors.h" - #include "clinic/exceptions.c.h" @@ -157,7 +155,6 @@ _testcapi_err_setstring_impl(PyObject *module, PyObject *exc, return NULL; } - /*[clinic input] _testcapi.err_setfromerrnowithfilename error: int From 5fc2e17ae69d11be1d44323687649a761c77c46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:39:18 +0100 Subject: [PATCH 40/43] use libc.so instead --- Lib/test/test_ctypes/test_dlerror.py | 72 +++++++++++----------------- 1 file changed, 27 insertions(+), 45 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 744beb51aaa040..0751f83c453feb 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,13 +1,11 @@ import _ctypes import os import platform -import subprocess import sys -import tempfile import test.support import unittest from ctypes import CDLL, c_int -from test.support.os_helper import create_empty_file, temp_dir +from ctypes.util import find_library FOO_C = r""" @@ -32,12 +30,6 @@ """ -def has_gcc(): - return subprocess.call(["gcc", "--version"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) == 0 - - @unittest.skipUnless(sys.platform.startswith('linux'), 'test requires GNU IFUNC support') class TestNullDlsym(unittest.TestCase): @@ -63,7 +55,13 @@ class TestNullDlsym(unittest.TestCase): """ def test_null_dlsym(self): - if not has_gcc(): + import subprocess + import tempfile + + retcode = subprocess.call(["gcc", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + if retcode != 0: self.skipTest("gcc is missing") pipe_r, pipe_w = os.pipe() @@ -132,39 +130,25 @@ def configure_locales(func): @classmethod def setUpClass(cls): - if not has_gcc(): - raise unittest.SkipTest("gcc is missing") - - def make_empty_lib(self, outdir, so_libname): - srcname = os.path.join(outdir, 'empty.c') - dstname = os.path.join(outdir, so_libname) - create_empty_file(srcname) - args = ['gcc', '-shared', '-o', dstname, srcname] - p = subprocess.run(args, capture_output=True) - p.check_returncode() - return dstname + cls.libc_file = find_library("c") @configure_locales def test_localized_error_from_dll(self): - with temp_dir() as outdir: - dstname = self.make_empty_lib(outdir, 'test_from_dll.so') - dll = CDLL(dstname) - with self.assertRaises(AttributeError) as cm: - dll.foo - if sys.platform.startswith('linux'): - # On macOS, the filename is not reported by dlerror(). - self.assertIn('test_from_dll.so', str(cm.exception)) + dll = CDLL(self.libc_file) + with self.assertRaises(AttributeError) as cm: + dll.foo + if sys.platform.startswith('linux'): + # On macOS, the filename is not reported by dlerror(). + self.assertIn(self.libc_file, str(cm.exception)) @configure_locales def test_localized_error_in_dll(self): - with temp_dir() as outdir: - dstname = self.make_empty_lib(outdir, 'test_in_dll.so') - dll = CDLL(dstname) - with self.assertRaises(ValueError) as cm: - c_int.in_dll(dll, 'foo') - if sys.platform.startswith('linux'): - # On macOS, the filename is not reported by dlerror(). - self.assertIn('test_in_dll.so', str(cm.exception)) + dll = CDLL(self.libc_file) + with self.assertRaises(ValueError) as cm: + c_int.in_dll(dll, 'foo') + if sys.platform.startswith('linux'): + # On macOS, the filename is not reported by dlerror(). + self.assertIn(self.libc_file, str(cm.exception)) @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), 'test requires _ctypes.dlopen()') @@ -185,14 +169,12 @@ def test_localized_error_dlopen(self): 'test requires _ctypes.dlsym()') @configure_locales def test_localized_error_dlsym(self): - with temp_dir() as outdir: - dstname = self.make_empty_lib(outdir, 'test_dlsym.so') - dll = _ctypes.dlopen(dstname) - with self.assertRaises(OSError) as cm: - _ctypes.dlsym(dll, 'foo') - if sys.platform.startswith('linux'): - # On macOS, the filename is not reported by dlerror(). - self.assertIn('test_dlsym.so', str(cm.exception)) + dll = _ctypes.dlopen(self.libc_file) + with self.assertRaises(OSError) as cm: + _ctypes.dlsym(dll, 'foo') + if sys.platform.startswith('linux'): + # On macOS, the filename is not reported by dlerror(). + self.assertIn(self.libc_file, str(cm.exception)) if __name__ == "__main__": From 62db957e6857b0486661c3cc6bd02e79674e45e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:58:24 +0100 Subject: [PATCH 41/43] address Victor's review - replace possible unknown symbol by a better unknown symbol (hopefully) - improve semantic naming ('libc_file' -> 'libc_filename') - ensure PyPy compatibility --- Lib/test/test_ctypes/test_dlerror.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 0751f83c453feb..c3c34d43481d36 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -108,9 +108,11 @@ def test_null_dlsym(self): self.assertEqual(os.read(pipe_r, 2), b'OK') # Case #3: Test 'py_dl_sym' from Modules/_ctypes/callproc.c - L = _ctypes.dlopen(dstname) + dlopen = test.support.get_attribute(_ctypes, 'dlopen') + dlsym = test.support.get_attribute(_ctypes, 'dlsym') + L = dlopen(dstname) with self.assertRaisesRegex(OSError, "symbol 'foo' not found"): - _ctypes.dlsym(L, "foo") + dlsym(L, "foo") # Assert that the IFUNC was called self.assertEqual(os.read(pipe_r, 2), b'OK') @@ -130,25 +132,25 @@ def configure_locales(func): @classmethod def setUpClass(cls): - cls.libc_file = find_library("c") + cls.libc_filename = find_library("c") @configure_locales def test_localized_error_from_dll(self): - dll = CDLL(self.libc_file) + dll = CDLL(self.libc_filename) with self.assertRaises(AttributeError) as cm: - dll.foo + dll.this_name_does_not_exist if sys.platform.startswith('linux'): # On macOS, the filename is not reported by dlerror(). - self.assertIn(self.libc_file, str(cm.exception)) + self.assertIn(self.libc_filename, str(cm.exception)) @configure_locales def test_localized_error_in_dll(self): - dll = CDLL(self.libc_file) + dll = CDLL(self.libc_filename) with self.assertRaises(ValueError) as cm: - c_int.in_dll(dll, 'foo') + c_int.in_dll(dll, 'this_name_does_not_exist') if sys.platform.startswith('linux'): # On macOS, the filename is not reported by dlerror(). - self.assertIn(self.libc_file, str(cm.exception)) + self.assertIn(self.libc_filename, str(cm.exception)) @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), 'test requires _ctypes.dlopen()') @@ -169,12 +171,12 @@ def test_localized_error_dlopen(self): 'test requires _ctypes.dlsym()') @configure_locales def test_localized_error_dlsym(self): - dll = _ctypes.dlopen(self.libc_file) + dll = _ctypes.dlopen(self.libc_filename) with self.assertRaises(OSError) as cm: - _ctypes.dlsym(dll, 'foo') + _ctypes.dlsym(dll, 'this_name_does_not_exist') if sys.platform.startswith('linux'): # On macOS, the filename is not reported by dlerror(). - self.assertIn(self.libc_file, str(cm.exception)) + self.assertIn(self.libc_filename, str(cm.exception)) if __name__ == "__main__": From c665a2db5dddf64afc73f3293b1e7d070cc3f181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 16 Dec 2024 17:04:34 +0100 Subject: [PATCH 42/43] Fix tests (hopefully it will still work) --- Lib/test/test_ctypes/test_dlerror.py | 2 +- Lib/test/test_dbm_gnu.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index c3c34d43481d36..1dce4171248cc9 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -123,7 +123,7 @@ class TestLocalization(unittest.TestCase): @staticmethod def configure_locales(func): - return test.support.run_with_locale( + return test.support.run_with_locales( 'LC_ALL', 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', 'fr_FR.utf8', 'en_US.utf8', diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index 66268c42a300b5..807b8e70008834 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -206,7 +206,7 @@ def test_clear(self): self.assertNotIn(k, db) self.assertEqual(len(db), 0) - @support.run_with_locale( + @support.run_with_locales( 'LC_ALL', 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', 'fr_FR.utf8', 'en_US.utf8', From 765e16c5bce91aba3047f00ed6d207f61eca3ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 16 Dec 2024 17:20:48 +0100 Subject: [PATCH 43/43] Reverting last commit to use `run_with_locale` only `run_with_locales` is a decorator-only but would run all specified locales, skipping those that cannot be found. Instead, we use `run_with_locale` which is a context manager *and* a decorator to speed-up tests. --- Lib/test/test_ctypes/test_dlerror.py | 2 +- Lib/test/test_dbm_gnu.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 1dce4171248cc9..c3c34d43481d36 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -123,7 +123,7 @@ class TestLocalization(unittest.TestCase): @staticmethod def configure_locales(func): - return test.support.run_with_locales( + return test.support.run_with_locale( 'LC_ALL', 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', 'fr_FR.utf8', 'en_US.utf8', diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index 807b8e70008834..66268c42a300b5 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -206,7 +206,7 @@ def test_clear(self): self.assertNotIn(k, db) self.assertEqual(len(db), 0) - @support.run_with_locales( + @support.run_with_locale( 'LC_ALL', 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', 'fr_FR.utf8', 'en_US.utf8',