Skip to content

gh-119311: Fix name mangling with PEP 695 generic classes #119464

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
May 28, 2024
Merged

gh-119311: Fix name mangling with PEP 695 generic classes #119464

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a43da60
gh-119311: Fix name mangling with PEP 695 generic classes
JelleZijlstra May 23, 2024
0bfe77c
Another test
JelleZijlstra May 23, 2024
1f27659
magic
JelleZijlstra May 23, 2024
fffc4ea
Remove unnecessary line
JelleZijlstra May 23, 2024
b119fe2
Merge branch 'main' into pep695-mangle2
JelleZijlstra May 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Include/internal/pycore_symtable.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ typedef struct _symtable_entry {
PyObject *ste_varnames; /* list of function parameters */
PyObject *ste_children; /* list of child blocks */
PyObject *ste_directives;/* locations of global and nonlocal statements */
PyObject *ste_mangled_names; /* set of names for which mangling should be applied */
_Py_block_ty ste_type;
int ste_nested; /* true if block is nested */
unsigned ste_free : 1; /* true if block has free variables */
Expand Down Expand Up @@ -128,6 +129,7 @@ extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *);

extern void _PySymtable_Free(struct symtable *);

extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name);
extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);

/* Flags for def-use information */
Expand Down
3 changes: 2 additions & 1 deletion Lib/importlib/_bootstrap_external.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,7 @@ def _write_atomic(path, data, mode=0o666):
# Python 3.13a5 3569 (Specialize CONTAINS_OP)
# Python 3.13a6 3570 (Add __firstlineno__ class attribute)
# Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT)
# Python 3.14a1 3601 (Fix miscompilation of private names in generic classes)

# Python 3.15 will start with 3700

Expand All @@ -489,7 +490,7 @@ def _write_atomic(path, data, mode=0o666):
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
# in PC/launcher.c must also be updated.

MAGIC_NUMBER = (3600).to_bytes(2, 'little') + b'\r\n'
MAGIC_NUMBER = (3601).to_bytes(2, 'little') + b'\r\n'

_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c

Expand Down
94 changes: 94 additions & 0 deletions Lib/test/test_type_params.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -823,6 +823,100 @@ def meth[__U](self, arg: __T, arg2: __U):

self.assertEqual(Foo.Alias.__value__, (T, V))

def test_no_leaky_mangling_in_module(self):
ns = run_code("""
__before = "before"
class X[T]: pass
__after = "after"
""")
self.assertEqual(ns["__before"], "before")
self.assertEqual(ns["__after"], "after")

def test_no_leaky_mangling_in_function(self):
ns = run_code("""
def f():
class X[T]: pass
_X_foo = 2
__foo = 1
assert locals()['__foo'] == 1
return __foo
""")
self.assertEqual(ns["f"](), 1)

def test_no_leaky_mangling_in_class(self):
ns = run_code("""
class Outer:
__before = "before"
class Inner[T]:
__x = "inner"
__after = "after"
""")
Outer = ns["Outer"]
self.assertEqual(Outer._Outer__before, "before")
self.assertEqual(Outer.Inner._Inner__x, "inner")
self.assertEqual(Outer._Outer__after, "after")

def test_no_mangling_in_bases(self):
ns = run_code("""
class __Base:
def __init_subclass__(self, **kwargs):
self.kwargs = kwargs

class Derived[T](__Base, __kwarg=1):
pass
""")
Derived = ns["Derived"]
self.assertEqual(Derived.__bases__, (ns["__Base"], Generic))
self.assertEqual(Derived.kwargs, {"__kwarg": 1})

def test_no_mangling_in_nested_scopes(self):
ns = run_code("""
from test.test_type_params import make_base

class __X:
pass

class Y[T: __X](
make_base(lambda: __X),
# doubly nested scope
make_base(lambda: (lambda: __X)),
# list comprehension
make_base([__X for _ in (1,)]),
# genexp
make_base(__X for _ in (1,)),
):
pass
""")
Y = ns["Y"]
T, = Y.__type_params__
self.assertIs(T.__bound__, ns["__X"])
base0 = Y.__bases__[0]
self.assertIs(base0.__arg__(), ns["__X"])
base1 = Y.__bases__[1]
self.assertIs(base1.__arg__()(), ns["__X"])
base2 = Y.__bases__[2]
self.assertEqual(base2.__arg__, [ns["__X"]])
base3 = Y.__bases__[3]
self.assertEqual(list(base3.__arg__), [ns["__X"]])

def test_type_params_are_mangled(self):
ns = run_code("""
from test.test_type_params import make_base

class Foo[__T, __U: __T](make_base(__T), make_base(lambda: __T)):
param = __T
""")
Foo = ns["Foo"]
T, U = Foo.__type_params__
self.assertEqual(T.__name__, "__T")
self.assertEqual(U.__name__, "__U")
self.assertIs(U.__bound__, T)
self.assertIs(Foo.param, T)

base1, base2, *_ = Foo.__bases__
self.assertIs(base1.__arg__, T)
self.assertIs(base2.__arg__(), T)


class TypeParamsComplexCallsTest(unittest.TestCase):
def test_defaults(self):
Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix bug where names appearing after a generic class are mangled as if they
are in the generic class.
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix bug where names are unexpectedly mangled in the bases of generic
classes.
12 changes: 6 additions & 6 deletions Python/compile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1001,7 +1001,7 @@ static int
compiler_addop_name(struct compiler_unit *u, location loc,
int opcode, PyObject *dict, PyObject *o)
{
PyObject *mangled = _Py_Mangle(u->u_private, o);
PyObject *mangled = _Py_MaybeMangle(u->u_private, u->u_ste, o);
if (!mangled) {
return ERROR;
}
Expand Down Expand Up @@ -1873,7 +1873,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc,
arg_ty arg = asdl_seq_GET(kwonlyargs, i);
expr_ty default_ = asdl_seq_GET(kw_defaults, i);
if (default_) {
PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg);
PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, arg->arg);
if (!mangled) {
goto error;
}
Expand Down Expand Up @@ -1930,7 +1930,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id,
if (!annotation) {
return SUCCESS;
}
PyObject *mangled = _Py_Mangle(c->u->u_private, id);
PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, id);
if (!mangled) {
return ERROR;
}
Expand Down Expand Up @@ -2625,7 +2625,6 @@ compiler_class(struct compiler *c, stmt_ty s)
asdl_type_param_seq *type_params = s->v.ClassDef.type_params;
int is_generic = asdl_seq_LEN(type_params) > 0;
if (is_generic) {
Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name));
PyObject *type_params_name = PyUnicode_FromFormat("<generic parameters of %U>",
s->v.ClassDef.name);
if (!type_params_name) {
Expand All @@ -2637,6 +2636,7 @@ compiler_class(struct compiler *c, stmt_ty s)
return ERROR;
}
Py_DECREF(type_params_name);
Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name));
RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params));
_Py_DECLARE_STR(type_params, ".type_params");
RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store));
Expand Down Expand Up @@ -4203,7 +4203,7 @@ compiler_nameop(struct compiler *c, location loc,
return ERROR;
}

mangled = _Py_Mangle(c->u->u_private, name);
mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, name);
if (!mangled) {
return ERROR;
}
Expand Down Expand Up @@ -6512,7 +6512,7 @@ compiler_annassign(struct compiler *c, stmt_ty s)
VISIT(c, expr, s->v.AnnAssign.annotation);
}
ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);
mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id);
mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);
ADDOP_LOAD_CONST_NEW(c, loc, mangled);
ADDOP(c, loc, STORE_SUBSCR);
}
Expand Down
49 changes: 42 additions & 7 deletions Python/symtable.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
ste->ste_children = NULL;

ste->ste_directives = NULL;
ste->ste_mangled_names = NULL;

ste->ste_type = block;
ste->ste_nested = 0;
Expand Down Expand Up @@ -166,6 +167,7 @@ ste_dealloc(PySTEntryObject *ste)
Py_XDECREF(ste->ste_varnames);
Py_XDECREF(ste->ste_children);
Py_XDECREF(ste->ste_directives);
Py_XDECREF(ste->ste_mangled_names);
PyObject_Free(ste);
}

Expand Down Expand Up @@ -1338,6 +1340,11 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
if (prev) {
ste->ste_comp_iter_expr = prev->ste_comp_iter_expr;
}
/* No need to inherit ste_mangled_names in classes, where all names
* are mangled. */
if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) {
ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names);
}
/* The entry is owned by the stack. Borrow it for st_cur. */
Py_DECREF(ste);
st->st_cur = ste;
Expand All @@ -1363,7 +1370,7 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
static long
symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name)
{
PyObject *mangled = _Py_Mangle(st->st_private, name);
PyObject *mangled = _Py_MaybeMangle(st->st_private, ste, name);
if (!mangled)
return 0;
long ret = _PyST_GetSymbol(ste, mangled);
Expand All @@ -1384,8 +1391,7 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s
PyObject *o;
PyObject *dict;
long val;
PyObject *mangled = _Py_Mangle(st->st_private, name);

PyObject *mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name);

if (!mangled)
return 0;
Expand Down Expand Up @@ -1474,6 +1480,11 @@ static int
symtable_add_def(struct symtable *st, PyObject *name, int flag,
int lineno, int col_offset, int end_lineno, int end_col_offset)
{
if ((flag & DEF_TYPE_PARAM) && st->st_cur->ste_mangled_names != NULL) {
if(PySet_Add(st->st_cur->ste_mangled_names, name) < 0) {
return 0;
}
}
return symtable_add_def_helper(st, name, flag, st->st_cur,
lineno, col_offset, end_lineno, end_col_offset);
}
Expand Down Expand Up @@ -1508,7 +1519,6 @@ symtable_enter_type_param_block(struct symtable *st, identifier name,
lineno, col_offset, end_lineno, end_col_offset)) {
return 0;
}
st->st_private = name;
// This is used for setting the generic base
_Py_DECLARE_STR(generic_base, ".generic_base");
if (!symtable_add_def(st, &_Py_STR(generic_base), DEF_LOCAL,
Expand Down Expand Up @@ -1597,7 +1607,7 @@ symtable_record_directive(struct symtable *st, identifier name, int lineno,
if (!st->st_cur->ste_directives)
return 0;
}
mangled = _Py_Mangle(st->st_private, name);
mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name);
if (!mangled)
return 0;
data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset);
Expand Down Expand Up @@ -1673,13 +1683,19 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
VISIT_QUIT(st, 0);
if (s->v.ClassDef.decorator_list)
VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
tmp = st->st_private;
if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
if (!symtable_enter_type_param_block(st, s->v.ClassDef.name,
(void *)s->v.ClassDef.type_params,
false, false, s->kind,
LOCATION(s))) {
VISIT_QUIT(st, 0);
}
st->st_private = s->v.ClassDef.name;
st->st_cur->ste_mangled_names = PySet_New(NULL);
if (!st->st_cur->ste_mangled_names) {
VISIT_QUIT(st, 0);
}
VISIT_SEQ(st, type_param, s->v.ClassDef.type_params);
}
VISIT_SEQ(st, expr, s->v.ClassDef.bases);
Expand All @@ -1688,7 +1704,6 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
(void *)s, s->lineno, s->col_offset,
s->end_lineno, s->end_col_offset))
VISIT_QUIT(st, 0);
tmp = st->st_private;
st->st_private = s->v.ClassDef.name;
if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
if (!symtable_add_def(st, &_Py_ID(__type_params__),
Expand All @@ -1702,13 +1717,13 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
}
}
VISIT_SEQ(st, stmt, s->v.ClassDef.body);
st->st_private = tmp;
if (!symtable_exit_block(st))
VISIT_QUIT(st, 0);
if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
if (!symtable_exit_block(st))
VISIT_QUIT(st, 0);
}
st->st_private = tmp;
break;
}
case TypeAlias_kind: {
Expand Down Expand Up @@ -2776,6 +2791,26 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
return st;
}

PyObject *
_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name)
{
/* Special case for type parameter blocks around generic classes:
* we want to mangle type parameter names (so a type param with a private
* name can be used inside the class body), but we don't want to mangle
* any other names that appear within the type parameter scope.
*/
if (ste->ste_mangled_names != NULL) {
int result = PySet_Contains(ste->ste_mangled_names, name);
if (result < 0) {
return NULL;
}
if (result == 0) {
return Py_NewRef(name);
}
}
return _Py_Mangle(privateobj, name);
}

PyObject *
_Py_Mangle(PyObject *privateobj, PyObject *ident)
{
Expand Down
Loading