gh-94526: getpath_dirname() no longer encodes the path (GH-97645)

miss-islington · vstinner · web-flow · commit 6537bc9a49d3 · 2022-09-30T06:38:41.000-07:00
Fix the Python path configuration used to initialized sys.path at Python startup. Paths are no longer encoded to UTF-8/strict to avoid encoding errors if it contains surrogate characters (bytes paths are decoded with the surrogateescape error handler). getpath_basename() and getpath_dirname() functions no longer encode the path to UTF-8/strict, but work directly on Unicode strings. These functions now use PyUnicode_FindChar() and PyUnicode_Substring() on the Unicode path, rather than strrchr() on the encoded bytes string. (cherry picked from commit 9f2f1dd) Co-authored-by: Victor Stinner <vstinner@python.org>
diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-29-15-19-29.gh-issue-94526.wq5m6T.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-29-15-19-29.gh-issue-94526.wq5m6T.rst
@@ -0,0 +1,4 @@
+Fix the Python path configuration used to initialized :data:`sys.path` at
+Python startup. Paths are no longer encoded to UTF-8/strict to avoid encoding
+errors if it contains surrogate characters (bytes paths are decoded with the
+surrogateescape error handler). Patch by Victor Stinner.
diff --git a/Modules/getpath.c b/Modules/getpath.c
@@ -82,27 +82,32 @@ getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args)
 static PyObject *
 getpath_basename(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    const char *path;
-    if (!PyArg_ParseTuple(args, "s", &path)) {
+    PyObject *path;
+    if (!PyArg_ParseTuple(args, "U", &path)) {
         return NULL;
     }
-    const char *name = strrchr(path, SEP);
-    return PyUnicode_FromString(name ? name + 1 : path);
+    Py_ssize_t end = PyUnicode_GET_LENGTH(path);
+    Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1);
+    if (pos < 0) {
+        return Py_NewRef(path);
+    }
+    return PyUnicode_Substring(path, pos + 1, end);
 }
 
 
 static PyObject *
 getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args)
 {
-    const char *path;
-    if (!PyArg_ParseTuple(args, "s", &path)) {
+    PyObject *path;
+    if (!PyArg_ParseTuple(args, "U", &path)) {
         return NULL;
     }
-    const char *name = strrchr(path, SEP);
-    if (!name) {
+    Py_ssize_t end = PyUnicode_GET_LENGTH(path);
+    Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1);
+    if (pos < 0) {
         return PyUnicode_FromStringAndSize(NULL, 0);
     }
-    return PyUnicode_FromStringAndSize(path, (name - path));
+    return PyUnicode_Substring(path, 0, pos);
 }
 
 

Original file line number	Diff line number	Diff line change
`@@ -82,27 +82,32 @@ getpath_abspath(PyObject Py_UNUSED(self), PyObject args)`
`82`	`82`	`static PyObject *`
`83`	`83`	`getpath_basename(PyObject Py_UNUSED(self), PyObject args)`
`84`	`84`	`{`
`85`		`- const char *path;`
`86`		`- if (!PyArg_ParseTuple(args, "s", &path)) {`
	`85`	`+ PyObject *path;`
	`86`	`+ if (!PyArg_ParseTuple(args, "U", &path)) {`
`87`	`87`	`return NULL;`
`88`	`88`	`}`
`89`		`- const char *name = strrchr(path, SEP);`
`90`		`- return PyUnicode_FromString(name ? name + 1 : path);`
	`89`	`+ Py_ssize_t end = PyUnicode_GET_LENGTH(path);`
	`90`	`+ Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1);`
	`91`	`+ if (pos < 0) {`
	`92`	`+ return Py_NewRef(path);`
	`93`	`+ }`
	`94`	`+ return PyUnicode_Substring(path, pos + 1, end);`
`91`	`95`	`}`
`92`	`96`
`93`	`97`
`94`	`98`	`static PyObject *`
`95`	`99`	`getpath_dirname(PyObject Py_UNUSED(self), PyObject args)`
`96`	`100`	`{`
`97`		`- const char *path;`
`98`		`- if (!PyArg_ParseTuple(args, "s", &path)) {`
	`101`	`+ PyObject *path;`
	`102`	`+ if (!PyArg_ParseTuple(args, "U", &path)) {`
`99`	`103`	`return NULL;`
`100`	`104`	`}`
`101`		`- const char *name = strrchr(path, SEP);`
`102`		`- if (!name) {`
	`105`	`+ Py_ssize_t end = PyUnicode_GET_LENGTH(path);`
	`106`	`+ Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1);`
	`107`	`+ if (pos < 0) {`
`103`	`108`	`return PyUnicode_FromStringAndSize(NULL, 0);`
`104`	`109`	`}`
`105`		`- return PyUnicode_FromStringAndSize(path, (name - path));`
	`110`	`+ return PyUnicode_Substring(path, 0, pos);`
`106`	`111`	`}`
`107`	`112`
`108`	`113`