Skip to content

Commit 48dfccf

Browse files
serhiy-storchakaserhiy-storchaka
committed
gh-87193: Support bytes objects with refcount > 1 in _PyBytes_Resize()
Create a new bytes object and destroy the old one if it has refcount > 1.
1 parent 63d6f26 commit 48dfccf

File tree

11 files changed

+123
-30
lines changed

11 files changed

+123
-30
lines changed

‎Doc/c-api/bytes.rst

+4-4
Original file line numberDiff line numberDiff line change
@@ -191,10 +191,10 @@ called with a non-bytes parameter.
191191
192192
.. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize)
193193
194-
A way to resize a bytes object even though it is "immutable". Only use this
195-
to build up a brand new bytes object; don't use this if the bytes may already
196-
be known in other parts of the code. It is an error to call this function if
197-
the refcount on the input bytes object is not one. Pass the address of an
194+
Resize a bytes object. *newsize* will be the new length of the bytes object.
195+
You can think of it as creating a new bytes object and destroying the old
196+
one, only more efficiently.
197+
Pass the address of an
198198
existing bytes object as an lvalue (it may be written into), and the new size
199199
desired. On success, *\*bytes* holds the resized bytes object and ``0`` is
200200
returned; the address in *\*bytes* may differ from its input value. If the

‎Lib/test/test_capi/test_bytes.py

+30
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from test.support import import_helper
33

44
_testlimitedcapi = import_helper.import_module('_testlimitedcapi')
5+
_testcapi = import_helper.import_module('_testcapi')
56
from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX
67

78
NULL = None
@@ -217,6 +218,35 @@ def test_decodeescape(self):
217218
# CRASHES decodeescape(b'abc', NULL, -1)
218219
# CRASHES decodeescape(NULL, NULL, 1)
219220

221+
def test_resize(self):
222+
"""Test _PyBytes_Resize()"""
223+
resize = _testcapi.bytes_resize
224+
225+
for new in True, False:
226+
self.assertEqual(resize(b'abc', 0, new), b'')
227+
self.assertEqual(resize(b'abc', 1, new), b'a')
228+
self.assertEqual(resize(b'abc', 2, new), b'ab')
229+
self.assertEqual(resize(b'abc', 3, new), b'abc')
230+
b = resize(b'abc', 4, new)
231+
self.assertEqual(len(b), 4)
232+
self.assertEqual(b[:3], b'abc')
233+
234+
self.assertEqual(resize(b'a', 0, new), b'')
235+
self.assertEqual(resize(b'a', 1, new), b'a')
236+
b = resize(b'a', 2, new)
237+
self.assertEqual(len(b), 2)
238+
self.assertEqual(b[:1], b'a')
239+
240+
self.assertEqual(resize(b'', 0, new), b'')
241+
self.assertEqual(len(resize(b'', 1, new)), 1)
242+
self.assertEqual(len(resize(b'', 2, new)), 2)
243+
244+
self.assertRaises(SystemError, resize, b'abc', -1, False)
245+
self.assertRaises(SystemError, resize, bytearray(b'abc'), 3, False)
246+
247+
# CRASHES resize(NULL, 0, False)
248+
# CRASHES resize(NULL, 3, False)
249+
220250

221251
if __name__ == "__main__":
222252
unittest.main()

‎Misc/NEWS.d/next/C API/2024-03-22-19-29-24.gh-issue-87193.u7O-jY.rst

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
:c:func:`_PyBytes_Resize` can now be called for bytes objects with reference
2+
count > 1, including 1-byte bytes objects. It creates a new bytes object and
3+
destroys the old one if it has reference count > 1.

‎Modules/Setup.stdlib.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@
162162
@MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
163163
@MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
164164
@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c
165-
@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c
165+
@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c
166166
@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c
167167
@MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c
168168
@MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c

‎Modules/_testcapi/bytes.c

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#include "parts.h"
2+
#include "util.h"
3+
4+
5+
/* Test _PyBytes_Resize() */
6+
static PyObject *
7+
bytes_resize(PyObject *Py_UNUSED(module), PyObject *args)
8+
{
9+
PyObject *obj;
10+
Py_ssize_t newsize;
11+
int new;
12+
13+
if (!PyArg_ParseTuple(args, "Onp", &obj, &newsize, &new))
14+
return NULL;
15+
16+
NULLABLE(obj);
17+
if (new) {
18+
assert(obj != NULL);
19+
assert(PyBytes_CheckExact(obj));
20+
PyObject *newobj = PyBytes_FromStringAndSize(NULL, PyBytes_Size(obj));
21+
if (newobj == NULL) {
22+
return NULL;
23+
}
24+
memcpy(PyBytes_AsString(newobj), PyBytes_AsString(obj), PyBytes_Size(obj));
25+
obj = newobj;
26+
}
27+
else {
28+
Py_XINCREF(obj);
29+
}
30+
if (_PyBytes_Resize(&obj, newsize) < 0) {
31+
assert(obj == NULL);
32+
}
33+
else {
34+
assert(obj != NULL);
35+
}
36+
return obj;
37+
}
38+
39+
40+
static PyMethodDef test_methods[] = {
41+
{"bytes_resize", bytes_resize, METH_VARARGS},
42+
{NULL},
43+
};
44+
45+
int
46+
_PyTestCapi_Init_Bytes(PyObject *m)
47+
{
48+
if (PyModule_AddFunctions(m, test_methods) < 0) {
49+
return -1;
50+
}
51+
52+
return 0;
53+
}

‎Modules/_testcapi/parts.h

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
int _PyTestCapi_Init_Vectorcall(PyObject *module);
3232
int _PyTestCapi_Init_Heaptype(PyObject *module);
3333
int _PyTestCapi_Init_Abstract(PyObject *module);
34+
int _PyTestCapi_Init_Bytes(PyObject *module);
3435
int _PyTestCapi_Init_Unicode(PyObject *module);
3536
int _PyTestCapi_Init_GetArgs(PyObject *module);
3637
int _PyTestCapi_Init_DateTime(PyObject *module);

‎Modules/_testcapimodule.c

+3
Original file line numberDiff line numberDiff line change
@@ -3971,6 +3971,9 @@ PyInit__testcapi(void)
39713971
if (_PyTestCapi_Init_Abstract(m) < 0) {
39723972
return NULL;
39733973
}
3974+
if (_PyTestCapi_Init_Bytes(m) < 0) {
3975+
return NULL;
3976+
}
39743977
if (_PyTestCapi_Init_Unicode(m) < 0) {
39753978
return NULL;
39763979
}

‎Objects/bytesobject.c

+23-18
Original file line numberDiff line numberDiff line change
@@ -3025,11 +3025,9 @@ PyBytes_ConcatAndDel(PyObject **pv, PyObject *w)
30253025

30263026

30273027
/* The following function breaks the notion that bytes are immutable:
3028-
it changes the size of a bytes object. We get away with this only if there
3029-
is only one module referencing the object. You can also think of it
3028+
it changes the size of a bytes object. You can think of it
30303029
as creating a new bytes object and destroying the old one, only
3031-
more efficiently. In any case, don't use this if the bytes object may
3032-
already be known to some other part of the code...
3030+
more efficiently.
30333031
Note that if there's not enough memory to resize the bytes object, the
30343032
original bytes object at *pv is deallocated, *pv is set to NULL, an "out of
30353033
memory" exception is set, and -1 is returned. Else (on success) 0 is
@@ -3045,28 +3043,40 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize)
30453043
PyBytesObject *sv;
30463044
v = *pv;
30473045
if (!PyBytes_Check(v) || newsize < 0) {
3048-
goto error;
3046+
*pv = 0;
3047+
Py_DECREF(v);
3048+
PyErr_BadInternalCall();
3049+
return -1;
30493050
}
3050-
if (Py_SIZE(v) == newsize) {
3051+
Py_ssize_t oldsize = PyBytes_GET_SIZE(v);
3052+
if (oldsize == newsize) {
30513053
/* return early if newsize equals to v->ob_size */
30523054
return 0;
30533055
}
3054-
if (Py_SIZE(v) == 0) {
3055-
if (newsize == 0) {
3056-
return 0;
3057-
}
3056+
if (oldsize == 0) {
30583057
*pv = _PyBytes_FromSize(newsize, 0);
30593058
Py_DECREF(v);
30603059
return (*pv == NULL) ? -1 : 0;
30613060
}
3062-
if (Py_REFCNT(v) != 1) {
3063-
goto error;
3064-
}
30653061
if (newsize == 0) {
30663062
*pv = bytes_get_empty();
30673063
Py_DECREF(v);
30683064
return 0;
30693065
}
3066+
if (Py_REFCNT(v) != 1) {
3067+
if (oldsize < newsize) {
3068+
*pv = _PyBytes_FromSize(newsize, 0);
3069+
if (*pv) {
3070+
memcpy(PyBytes_AS_STRING(*pv), PyBytes_AS_STRING(v), oldsize);
3071+
}
3072+
}
3073+
else {
3074+
*pv = PyBytes_FromStringAndSize(PyBytes_AS_STRING(v), newsize);
3075+
}
3076+
Py_DECREF(v);
3077+
return (*pv == NULL) ? -1 : 0;
3078+
}
3079+
30703080
#ifdef Py_TRACE_REFS
30713081
_Py_ForgetReference(v);
30723082
#endif
@@ -3089,11 +3099,6 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS
30893099
sv->ob_shash = -1; /* invalidate cached hash value */
30903100
_Py_COMP_DIAG_POP
30913101
return 0;
3092-
error:
3093-
*pv = 0;
3094-
Py_DECREF(v);
3095-
PyErr_BadInternalCall();
3096-
return -1;
30973102
}
30983103

30993104

‎Objects/fileobject.c

+1-7
Original file line numberDiff line numberDiff line change
@@ -80,13 +80,7 @@ PyFile_GetLine(PyObject *f, int n)
8080
"EOF when reading a line");
8181
}
8282
else if (s[len-1] == '\n') {
83-
if (Py_REFCNT(result) == 1)
84-
_PyBytes_Resize(&result, len-1);
85-
else {
86-
PyObject *v;
87-
v = PyBytes_FromStringAndSize(s, len-1);
88-
Py_SETREF(result, v);
89-
}
83+
(void) _PyBytes_Resize(&result, len-1);
9084
}
9185
}
9286
if (n < 0 && result != NULL && PyUnicode_Check(result)) {

‎PCbuild/_testcapi.vcxproj

+1
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
<ClCompile Include="..\Modules\_testcapi\vectorcall.c" />
9999
<ClCompile Include="..\Modules\_testcapi\heaptype.c" />
100100
<ClCompile Include="..\Modules\_testcapi\abstract.c" />
101+
<ClCompile Include="..\Modules\_testcapi\bytes.c" />
101102
<ClCompile Include="..\Modules\_testcapi\unicode.c" />
102103
<ClCompile Include="..\Modules\_testcapi\dict.c" />
103104
<ClCompile Include="..\Modules\_testcapi\set.c" />

‎PCbuild/_testcapi.vcxproj.filters

+3
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@
3030
<ClCompile Include="..\Modules\_testcapi\abstract.c">
3131
<Filter>Source Files</Filter>
3232
</ClCompile>
33+
<ClCompile Include="..\Modules\_testcapi\bytes.c">
34+
<Filter>Source Files</Filter>
35+
</ClCompile>
3336
<ClCompile Include="..\Modules\_testcapi\unicode.c">
3437
<Filter>Source Files</Filter>
3538
</ClCompile>

0 commit comments

Comments
 (0)