Skip to content

Commit a9a74da

Browse files
JelleZijlstraJelleZijlstra
authored
gh-119311: Fix name mangling with PEP 695 generic classes (#119464)
Fixes #119311. Fixes #119395.
1 parent 3e8b609 commit a9a74da

File tree

7 files changed

+150
-14
lines changed

7 files changed

+150
-14
lines changed

‎Include/internal/pycore_symtable.h

+2
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ typedef struct _symtable_entry {
8181
PyObject *ste_varnames; /* list of function parameters */
8282
PyObject *ste_children; /* list of child blocks */
8383
PyObject *ste_directives;/* locations of global and nonlocal statements */
84+
PyObject *ste_mangled_names; /* set of names for which mangling should be applied */
8485
_Py_block_ty ste_type;
8586
int ste_nested; /* true if block is nested */
8687
unsigned ste_free : 1; /* true if block has free variables */
@@ -128,6 +129,7 @@ extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *);
128129

129130
extern void _PySymtable_Free(struct symtable *);
130131

132+
extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name);
131133
extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);
132134

133135
/* Flags for def-use information */

‎Lib/importlib/_bootstrap_external.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,7 @@ def _write_atomic(path, data, mode=0o666):
473473
# Python 3.13a5 3569 (Specialize CONTAINS_OP)
474474
# Python 3.13a6 3570 (Add __firstlineno__ class attribute)
475475
# Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT)
476+
# Python 3.14a1 3601 (Fix miscompilation of private names in generic classes)
476477

477478
# Python 3.15 will start with 3700
478479

@@ -489,7 +490,7 @@ def _write_atomic(path, data, mode=0o666):
489490
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
490491
# in PC/launcher.c must also be updated.
491492

492-
MAGIC_NUMBER = (3600).to_bytes(2, 'little') + b'\r\n'
493+
MAGIC_NUMBER = (3601).to_bytes(2, 'little') + b'\r\n'
493494

494495
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
495496

‎Lib/test/test_type_params.py

+94
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,100 @@ def meth[__U](self, arg: __T, arg2: __U):
823823

824824
self.assertEqual(Foo.Alias.__value__, (T, V))
825825

826+
def test_no_leaky_mangling_in_module(self):
827+
ns = run_code("""
828+
__before = "before"
829+
class X[T]: pass
830+
__after = "after"
831+
""")
832+
self.assertEqual(ns["__before"], "before")
833+
self.assertEqual(ns["__after"], "after")
834+
835+
def test_no_leaky_mangling_in_function(self):
836+
ns = run_code("""
837+
def f():
838+
class X[T]: pass
839+
_X_foo = 2
840+
__foo = 1
841+
assert locals()['__foo'] == 1
842+
return __foo
843+
""")
844+
self.assertEqual(ns["f"](), 1)
845+
846+
def test_no_leaky_mangling_in_class(self):
847+
ns = run_code("""
848+
class Outer:
849+
__before = "before"
850+
class Inner[T]:
851+
__x = "inner"
852+
__after = "after"
853+
""")
854+
Outer = ns["Outer"]
855+
self.assertEqual(Outer._Outer__before, "before")
856+
self.assertEqual(Outer.Inner._Inner__x, "inner")
857+
self.assertEqual(Outer._Outer__after, "after")
858+
859+
def test_no_mangling_in_bases(self):
860+
ns = run_code("""
861+
class __Base:
862+
def __init_subclass__(self, **kwargs):
863+
self.kwargs = kwargs
864+
865+
class Derived[T](__Base, __kwarg=1):
866+
pass
867+
""")
868+
Derived = ns["Derived"]
869+
self.assertEqual(Derived.__bases__, (ns["__Base"], Generic))
870+
self.assertEqual(Derived.kwargs, {"__kwarg": 1})
871+
872+
def test_no_mangling_in_nested_scopes(self):
873+
ns = run_code("""
874+
from test.test_type_params import make_base
875+
876+
class __X:
877+
pass
878+
879+
class Y[T: __X](
880+
make_base(lambda: __X),
881+
# doubly nested scope
882+
make_base(lambda: (lambda: __X)),
883+
# list comprehension
884+
make_base([__X for _ in (1,)]),
885+
# genexp
886+
make_base(__X for _ in (1,)),
887+
):
888+
pass
889+
""")
890+
Y = ns["Y"]
891+
T, = Y.__type_params__
892+
self.assertIs(T.__bound__, ns["__X"])
893+
base0 = Y.__bases__[0]
894+
self.assertIs(base0.__arg__(), ns["__X"])
895+
base1 = Y.__bases__[1]
896+
self.assertIs(base1.__arg__()(), ns["__X"])
897+
base2 = Y.__bases__[2]
898+
self.assertEqual(base2.__arg__, [ns["__X"]])
899+
base3 = Y.__bases__[3]
900+
self.assertEqual(list(base3.__arg__), [ns["__X"]])
901+
902+
def test_type_params_are_mangled(self):
903+
ns = run_code("""
904+
from test.test_type_params import make_base
905+
906+
class Foo[__T, __U: __T](make_base(__T), make_base(lambda: __T)):
907+
param = __T
908+
""")
909+
Foo = ns["Foo"]
910+
T, U = Foo.__type_params__
911+
self.assertEqual(T.__name__, "__T")
912+
self.assertEqual(U.__name__, "__U")
913+
self.assertIs(U.__bound__, T)
914+
self.assertIs(Foo.param, T)
915+
916+
base1, base2, *_ = Foo.__bases__
917+
self.assertIs(base1.__arg__, T)
918+
self.assertIs(base2.__arg__(), T)
919+
826920

827921
class TypeParamsComplexCallsTest(unittest.TestCase):
828922
def test_defaults(self):

‎Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix bug where names appearing after a generic class are mangled as if they
2+
are in the generic class.

‎Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix bug where names are unexpectedly mangled in the bases of generic
2+
classes.

‎Python/compile.c

+6-6
Original file line numberDiff line numberDiff line change
@@ -1001,7 +1001,7 @@ static int
10011001
compiler_addop_name(struct compiler_unit *u, location loc,
10021002
int opcode, PyObject *dict, PyObject *o)
10031003
{
1004-
PyObject *mangled = _Py_Mangle(u->u_private, o);
1004+
PyObject *mangled = _Py_MaybeMangle(u->u_private, u->u_ste, o);
10051005
if (!mangled) {
10061006
return ERROR;
10071007
}
@@ -1873,7 +1873,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc,
18731873
arg_ty arg = asdl_seq_GET(kwonlyargs, i);
18741874
expr_ty default_ = asdl_seq_GET(kw_defaults, i);
18751875
if (default_) {
1876-
PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg);
1876+
PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, arg->arg);
18771877
if (!mangled) {
18781878
goto error;
18791879
}
@@ -1930,7 +1930,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id,
19301930
if (!annotation) {
19311931
return SUCCESS;
19321932
}
1933-
PyObject *mangled = _Py_Mangle(c->u->u_private, id);
1933+
PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, id);
19341934
if (!mangled) {
19351935
return ERROR;
19361936
}
@@ -2625,7 +2625,6 @@ compiler_class(struct compiler *c, stmt_ty s)
26252625
asdl_type_param_seq *type_params = s->v.ClassDef.type_params;
26262626
int is_generic = asdl_seq_LEN(type_params) > 0;
26272627
if (is_generic) {
2628-
Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name));
26292628
PyObject *type_params_name = PyUnicode_FromFormat("<generic parameters of %U>",
26302629
s->v.ClassDef.name);
26312630
if (!type_params_name) {
@@ -2637,6 +2636,7 @@ compiler_class(struct compiler *c, stmt_ty s)
26372636
return ERROR;
26382637
}
26392638
Py_DECREF(type_params_name);
2639+
Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name));
26402640
RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params));
26412641
_Py_DECLARE_STR(type_params, ".type_params");
26422642
RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store));
@@ -4203,7 +4203,7 @@ compiler_nameop(struct compiler *c, location loc,
42034203
return ERROR;
42044204
}
42054205

4206-
mangled = _Py_Mangle(c->u->u_private, name);
4206+
mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, name);
42074207
if (!mangled) {
42084208
return ERROR;
42094209
}
@@ -6512,7 +6512,7 @@ compiler_annassign(struct compiler *c, stmt_ty s)
65126512
VISIT(c, expr, s->v.AnnAssign.annotation);
65136513
}
65146514
ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);
6515-
mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id);
6515+
mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);
65166516
ADDOP_LOAD_CONST_NEW(c, loc, mangled);
65176517
ADDOP(c, loc, STORE_SUBSCR);
65186518
}

‎Python/symtable.c

+42-7
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
103103
ste->ste_children = NULL;
104104

105105
ste->ste_directives = NULL;
106+
ste->ste_mangled_names = NULL;
106107

107108
ste->ste_type = block;
108109
ste->ste_nested = 0;
@@ -166,6 +167,7 @@ ste_dealloc(PySTEntryObject *ste)
166167
Py_XDECREF(ste->ste_varnames);
167168
Py_XDECREF(ste->ste_children);
168169
Py_XDECREF(ste->ste_directives);
170+
Py_XDECREF(ste->ste_mangled_names);
169171
PyObject_Free(ste);
170172
}
171173

@@ -1338,6 +1340,11 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
13381340
if (prev) {
13391341
ste->ste_comp_iter_expr = prev->ste_comp_iter_expr;
13401342
}
1343+
/* No need to inherit ste_mangled_names in classes, where all names
1344+
* are mangled. */
1345+
if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) {
1346+
ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names);
1347+
}
13411348
/* The entry is owned by the stack. Borrow it for st_cur. */
13421349
Py_DECREF(ste);
13431350
st->st_cur = ste;
@@ -1363,7 +1370,7 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
13631370
static long
13641371
symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name)
13651372
{
1366-
PyObject *mangled = _Py_Mangle(st->st_private, name);
1373+
PyObject *mangled = _Py_MaybeMangle(st->st_private, ste, name);
13671374
if (!mangled)
13681375
return 0;
13691376
long ret = _PyST_GetSymbol(ste, mangled);
@@ -1384,8 +1391,7 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s
13841391
PyObject *o;
13851392
PyObject *dict;
13861393
long val;
1387-
PyObject *mangled = _Py_Mangle(st->st_private, name);
1388-
1394+
PyObject *mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name);
13891395

13901396
if (!mangled)
13911397
return 0;
@@ -1474,6 +1480,11 @@ static int
14741480
symtable_add_def(struct symtable *st, PyObject *name, int flag,
14751481
int lineno, int col_offset, int end_lineno, int end_col_offset)
14761482
{
1483+
if ((flag & DEF_TYPE_PARAM) && st->st_cur->ste_mangled_names != NULL) {
1484+
if(PySet_Add(st->st_cur->ste_mangled_names, name) < 0) {
1485+
return 0;
1486+
}
1487+
}
14771488
return symtable_add_def_helper(st, name, flag, st->st_cur,
14781489
lineno, col_offset, end_lineno, end_col_offset);
14791490
}
@@ -1508,7 +1519,6 @@ symtable_enter_type_param_block(struct symtable *st, identifier name,
15081519
lineno, col_offset, end_lineno, end_col_offset)) {
15091520
return 0;
15101521
}
1511-
st->st_private = name;
15121522
// This is used for setting the generic base
15131523
_Py_DECLARE_STR(generic_base, ".generic_base");
15141524
if (!symtable_add_def(st, &_Py_STR(generic_base), DEF_LOCAL,
@@ -1597,7 +1607,7 @@ symtable_record_directive(struct symtable *st, identifier name, int lineno,
15971607
if (!st->st_cur->ste_directives)
15981608
return 0;
15991609
}
1600-
mangled = _Py_Mangle(st->st_private, name);
1610+
mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name);
16011611
if (!mangled)
16021612
return 0;
16031613
data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset);
@@ -1673,13 +1683,19 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
16731683
VISIT_QUIT(st, 0);
16741684
if (s->v.ClassDef.decorator_list)
16751685
VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
1686+
tmp = st->st_private;
16761687
if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
16771688
if (!symtable_enter_type_param_block(st, s->v.ClassDef.name,
16781689
(void *)s->v.ClassDef.type_params,
16791690
false, false, s->kind,
16801691
LOCATION(s))) {
16811692
VISIT_QUIT(st, 0);
16821693
}
1694+
st->st_private = s->v.ClassDef.name;
1695+
st->st_cur->ste_mangled_names = PySet_New(NULL);
1696+
if (!st->st_cur->ste_mangled_names) {
1697+
VISIT_QUIT(st, 0);
1698+
}
16831699
VISIT_SEQ(st, type_param, s->v.ClassDef.type_params);
16841700
}
16851701
VISIT_SEQ(st, expr, s->v.ClassDef.bases);
@@ -1688,7 +1704,6 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
16881704
(void *)s, s->lineno, s->col_offset,
16891705
s->end_lineno, s->end_col_offset))
16901706
VISIT_QUIT(st, 0);
1691-
tmp = st->st_private;
16921707
st->st_private = s->v.ClassDef.name;
16931708
if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
16941709
if (!symtable_add_def(st, &_Py_ID(__type_params__),
@@ -1702,13 +1717,13 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
17021717
}
17031718
}
17041719
VISIT_SEQ(st, stmt, s->v.ClassDef.body);
1705-
st->st_private = tmp;
17061720
if (!symtable_exit_block(st))
17071721
VISIT_QUIT(st, 0);
17081722
if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
17091723
if (!symtable_exit_block(st))
17101724
VISIT_QUIT(st, 0);
17111725
}
1726+
st->st_private = tmp;
17121727
break;
17131728
}
17141729
case TypeAlias_kind: {
@@ -2776,6 +2791,26 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
27762791
return st;
27772792
}
27782793

2794+
PyObject *
2795+
_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name)
2796+
{
2797+
/* Special case for type parameter blocks around generic classes:
2798+
* we want to mangle type parameter names (so a type param with a private
2799+
* name can be used inside the class body), but we don't want to mangle
2800+
* any other names that appear within the type parameter scope.
2801+
*/
2802+
if (ste->ste_mangled_names != NULL) {
2803+
int result = PySet_Contains(ste->ste_mangled_names, name);
2804+
if (result < 0) {
2805+
return NULL;
2806+
}
2807+
if (result == 0) {
2808+
return Py_NewRef(name);
2809+
}
2810+
}
2811+
return _Py_Mangle(privateobj, name);
2812+
}
2813+
27792814
PyObject *
27802815
_Py_Mangle(PyObject *privateobj, PyObject *ident)
27812816
{

0 commit comments

Comments
 (0)