Skip to content

Commit c76da79

Browse files
markshannonmarkshannon
authored
bpo-42739: Don't use sentinels to mark end of line table. (GH-25657)
* Add length parameter to PyLineTable_InitAddressRange and doen't use sentinel values at end of table. Makes the line number table more robust. * Update PyCodeAddressRange to match PEP 626.
1 parent 53dd6c9 commit c76da79

File tree

13 files changed

+4966
-4984
lines changed

13 files changed

+4966
-4984
lines changed

‎Include/cpython/code.h

+8-3
Original file line numberDiff line numberDiff line change
@@ -135,12 +135,17 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno);
135135
PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);
136136

137137
/* for internal use only */
138+
struct _opaque {
139+
int computed_line;
140+
char *lo_next;
141+
char *limit;
142+
};
143+
138144
typedef struct _line_offsets {
139145
int ar_start;
140146
int ar_end;
141147
int ar_line;
142-
int ar_computed_line;
143-
char *lo_next;
148+
struct _opaque opaque;
144149
} PyCodeAddressRange;
145150

146151
/* Update *bounds to describe the first and one-past-the-last instructions in the
@@ -170,7 +175,7 @@ PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index,
170175
int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds);
171176

172177
/** Out of process API for initializing the line number table. */
173-
void PyLineTable_InitAddressRange(char *linetable, int firstlineno, PyCodeAddressRange *range);
178+
void PyLineTable_InitAddressRange(char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range);
174179

175180
/** API for traversing the line number table. */
176181
int PyLineTable_NextAddressRange(PyCodeAddressRange *range);

‎Lib/ctypes/test/test_values.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,9 @@ class struct_frozen(Structure):
8080
continue
8181
items.append((entry.name.decode("ascii"), entry.size))
8282

83-
expected = [("__hello__", 139),
84-
("__phello__", -139),
85-
("__phello__.spam", 139),
83+
expected = [("__hello__", 137),
84+
("__phello__", -137),
85+
("__phello__.spam", 137),
8686
]
8787
self.assertEqual(items, expected, "PyImport_FrozenModules example "
8888
"in Doc/library/ctypes.rst may be out of date")

‎Lib/importlib/_bootstrap_external.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,7 @@ def _write_atomic(path, data, mode=0o666):
350350
# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets).
351351
# Python 3.10b1 3436 (Add GEN_START bytecode #43683)
352352
# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!)
353+
# Python 3.10b1 3438 Safer line number table handling.
353354

354355
#
355356
# MAGIC must change whenever the bytecode emitted by the compiler may no
@@ -359,7 +360,7 @@ def _write_atomic(path, data, mode=0o666):
359360
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
360361
# in PC/launcher.c must also be updated.
361362

362-
MAGIC_NUMBER = (3437).to_bytes(2, 'little') + b'\r\n'
363+
MAGIC_NUMBER = (3438).to_bytes(2, 'little') + b'\r\n'
363364
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
364365

365366
_PYCACHE = '__pycache__'

‎Lib/test/test_code.py

+6
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,12 @@ def func2():
264264
new_code = code.replace(**{attr: value})
265265
self.assertEqual(getattr(new_code, attr), value)
266266

267+
def test_empty_linetable(self):
268+
def func():
269+
pass
270+
new_code = code = func.__code__.replace(co_linetable=b'')
271+
self.assertEqual(list(new_code.co_lines()), [])
272+
267273

268274
def isinterned(s):
269275
return s is sys.intern(('_' + s + '_')[1:-1])

‎Lib/test/test_dis.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ def bug42562():
172172

173173

174174
# Set line number for 'pass' to None
175-
bug42562.__code__ = bug42562.__code__.replace(co_linetable=b'\x04\x80\xff\x80')
175+
bug42562.__code__ = bug42562.__code__.replace(co_linetable=b'\x04\x80')
176176

177177

178178
dis_bug42562 = """\

‎Misc/NEWS.d/next/Core and Builtins/2021-04-27-10-59-10.bpo-42739.PrVkKM.rst

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
The internal representation of line number tables is changed to not use
2+
sentinels, and an explicit length parameter is added to the out of process
3+
API function ``PyLineTable_InitAddressRange``. This makes the handling of
4+
line number tables more robust in some circumstances.

‎Objects/codeobject.c

+21-20
Original file line numberDiff line numberDiff line change
@@ -456,15 +456,15 @@ code_getlnotab(PyCodeObject *code, void *closure)
456456
}
457457
_PyCode_InitAddressRange(code, &bounds);
458458
while (PyLineTable_NextAddressRange(&bounds)) {
459-
if (bounds.ar_computed_line != line) {
459+
if (bounds.opaque.computed_line != line) {
460460
int bdelta = bounds.ar_start - code_offset;
461-
int ldelta = bounds.ar_computed_line - line;
461+
int ldelta = bounds.opaque.computed_line - line;
462462
if (!emit_delta(&bytes, bdelta, ldelta, &table_offset)) {
463463
Py_DECREF(bytes);
464464
return NULL;
465465
}
466466
code_offset = bounds.ar_start;
467-
line = bounds.ar_computed_line;
467+
line = bounds.opaque.computed_line;
468468
}
469469
}
470470
_PyBytes_Resize(&bytes, table_offset);
@@ -1120,44 +1120,43 @@ code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args))
11201120
static void
11211121
retreat(PyCodeAddressRange *bounds)
11221122
{
1123-
int ldelta = ((signed char *)bounds->lo_next)[-1];
1123+
int ldelta = ((signed char *)bounds->opaque.lo_next)[-1];
11241124
if (ldelta == -128) {
11251125
ldelta = 0;
11261126
}
1127-
bounds->ar_computed_line -= ldelta;
1128-
bounds->lo_next -= 2;
1127+
bounds->opaque.computed_line -= ldelta;
1128+
bounds->opaque.lo_next -= 2;
11291129
bounds->ar_end = bounds->ar_start;
1130-
bounds->ar_start -= ((unsigned char *)bounds->lo_next)[-2];
1131-
ldelta = ((signed char *)bounds->lo_next)[-1];
1130+
bounds->ar_start -= ((unsigned char *)bounds->opaque.lo_next)[-2];
1131+
ldelta = ((signed char *)bounds->opaque.lo_next)[-1];
11321132
if (ldelta == -128) {
11331133
bounds->ar_line = -1;
11341134
}
11351135
else {
1136-
bounds->ar_line = bounds->ar_computed_line;
1136+
bounds->ar_line = bounds->opaque.computed_line;
11371137
}
11381138
}
11391139

11401140
static void
11411141
advance(PyCodeAddressRange *bounds)
11421142
{
11431143
bounds->ar_start = bounds->ar_end;
1144-
int delta = ((unsigned char *)bounds->lo_next)[0];
1145-
assert (delta < 255);
1144+
int delta = ((unsigned char *)bounds->opaque.lo_next)[0];
11461145
bounds->ar_end += delta;
1147-
int ldelta = ((signed char *)bounds->lo_next)[1];
1148-
bounds->lo_next += 2;
1146+
int ldelta = ((signed char *)bounds->opaque.lo_next)[1];
1147+
bounds->opaque.lo_next += 2;
11491148
if (ldelta == -128) {
11501149
bounds->ar_line = -1;
11511150
}
11521151
else {
1153-
bounds->ar_computed_line += ldelta;
1154-
bounds->ar_line = bounds->ar_computed_line;
1152+
bounds->opaque.computed_line += ldelta;
1153+
bounds->ar_line = bounds->opaque.computed_line;
11551154
}
11561155
}
11571156

11581157
static inline int
11591158
at_end(PyCodeAddressRange *bounds) {
1160-
return ((unsigned char *)bounds->lo_next)[0] == 255;
1159+
return bounds->opaque.lo_next >= bounds->opaque.limit;
11611160
}
11621161

11631162
int
@@ -1256,20 +1255,22 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq)
12561255
}
12571256

12581257
void
1259-
PyLineTable_InitAddressRange(char *linetable, int firstlineno, PyCodeAddressRange *range)
1258+
PyLineTable_InitAddressRange(char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range)
12601259
{
1261-
range->lo_next = linetable;
1260+
range->opaque.lo_next = linetable;
1261+
range->opaque.limit = range->opaque.lo_next + length;
12621262
range->ar_start = -1;
12631263
range->ar_end = 0;
1264-
range->ar_computed_line = firstlineno;
1264+
range->opaque.computed_line = firstlineno;
12651265
range->ar_line = -1;
12661266
}
12671267

12681268
int
12691269
_PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds)
12701270
{
12711271
char *linetable = PyBytes_AS_STRING(co->co_linetable);
1272-
PyLineTable_InitAddressRange(linetable, co->co_firstlineno, bounds);
1272+
Py_ssize_t length = PyBytes_GET_SIZE(co->co_linetable);
1273+
PyLineTable_InitAddressRange(linetable, length, co->co_firstlineno, bounds);
12731274
return bounds->ar_line;
12741275
}
12751276

‎Objects/lnotab_notes.txt

-4
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ Note that the end - start value is always positive.
3939
Finally, in order to fit into a single byte we need to convert start deltas to the range 0 <= delta <= 254,
4040
and line deltas to the range -127 <= delta <= 127.
4141
A line delta of -128 is used to indicate no line number.
42-
A start delta of 255 is used as a sentinel to mark the end of the table.
4342
Also note that a delta of zero indicates that there are no bytecodes in the given range,
4443
which means we can use an invalid line number for that range.
4544

@@ -54,7 +53,6 @@ Final form:
5453
16 +1
5554
0 +127 (line 135, but the range is empty as no bytecodes are at line 135)
5655
4 +73
57-
255 (end mark) ---
5856

5957
Iterating over the table.
6058
-------------------------
@@ -68,8 +66,6 @@ def co_lines(code):
6866
end = 0
6967
table_iter = iter(code.internal_line_table):
7068
for sdelta, ldelta in table_iter:
71-
if sdelta == 255:
72-
break
7369
if ldelta == 0: # No change to line number, just accumulate changes to end
7470
end += odelta
7571
continue

‎Python/compile.c

-4
Original file line numberDiff line numberDiff line change
@@ -6959,10 +6959,6 @@ assemble(struct compiler *c, int addNone)
69596959
if (!assemble_line_range(&a)) {
69606960
return 0;
69616961
}
6962-
/* Emit sentinel at end of line number table */
6963-
if (!assemble_emit_linetable_pair(&a, 255, -128)) {
6964-
goto error;
6965-
}
69666962

69676963
if (_PyBytes_Resize(&a.a_lnotab, a.a_lnotab_off) < 0) {
69686964
goto error;

‎Python/frozen_hello.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,5 @@ const unsigned char _Py_M__hello[] = {
88
5,112,114,105,110,116,169,0,114,1,0,0,0,114,1,0,
99
0,0,122,14,60,102,114,111,122,101,110,32,104,101,108,108,
1010
111,62,218,8,60,109,111,100,117,108,101,62,1,0,0,0,
11-
115,6,0,0,0,4,0,12,1,255,128,
11+
115,4,0,0,0,4,0,12,1,
1212
};

0 commit comments

Comments
 (0)