Skip to content

Commit 7301979

Browse files
uriyyoserhiy-storchakamethane
authored
bpo-42202: Store func annotations as a tuple (GH-23316)
Reduce memory footprint and improve performance of loading modules having many func annotations. >>> sys.getsizeof({"a":"int","b":"int","return":"int"}) 232 >>> sys.getsizeof(("a","int","b","int","return","int")) 88 The tuple is converted into dict on the fly when `func.__annotations__` is accessed first. Co-authored-by: Serhiy Storchaka <storchaka@gmail.com> Co-authored-by: Inada Naoki <songofacandy@gmail.com>
1 parent 85c8492 commit 7301979

File tree

8 files changed

+174
-155
lines changed

8 files changed

+174
-155
lines changed

‎Doc/library/dis.rst

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1144,11 +1144,13 @@ All of the following opcodes use their arguments.
11441144
* ``0x01`` a tuple of default values for positional-only and
11451145
positional-or-keyword parameters in positional order
11461146
* ``0x02`` a dictionary of keyword-only parameters' default values
1147-
* ``0x04`` an annotation dictionary
1147+
* ``0x04`` a tuple of strings containing parameters' annotations
11481148
* ``0x08`` a tuple containing cells for free variables, making a closure
11491149
* the code associated with the function (at TOS1)
11501150
* the :term:`qualified name` of the function (at TOS)
11511151

1152+
.. versionchanged:: 3.10
1153+
Flag value ``0x04`` is a tuple of strings instead of dictionary
11521154

11531155
.. opcode:: BUILD_SLICE (argc)
11541156

‎Doc/whatsnew/3.10.rst

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,11 @@ Optimizations
388388
for more details. (Contributed by Victor Stinner and Pablo Galindo in
389389
:issue:`38980`)
390390

391+
* Function parameters and their annotations are no longer computed at runtime,
392+
but rather at compilation time. They are stored as a tuple of strings at the
393+
bytecode level. It is now around 100% faster to create a function with parameter
394+
annotations. (Contributed by Yurii Karabas and Inada Naoki in :issue:`42202`)
395+
391396
Deprecated
392397
==========
393398

@@ -461,6 +466,12 @@ Changes in the Python API
461466
have been renamed to *exc*.
462467
(Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.)
463468

469+
CPython bytecode changes
470+
========================
471+
472+
* The ``MAKE_FUNCTION`` instruction accepts tuple of strings as annotations
473+
instead of dictionary.
474+
(Contributed by Yurii Karabas and Inada Naoki in :issue:`42202`)
464475

465476
Build Changes
466477
=============

‎Lib/importlib/_bootstrap_external.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,7 @@ def _write_atomic(path, data, mode=0o666):
311311
# Python 3.9a2 3425 (simplify bytecodes for **value unpacking)
312312
# Python 3.10a1 3430 (Make 'annotations' future by default)
313313
# Python 3.10a1 3431 (New line number table format -- PEP 626)
314+
# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202)
314315

315316
#
316317
# MAGIC must change whenever the bytecode emitted by the compiler may no
@@ -320,7 +321,7 @@ def _write_atomic(path, data, mode=0o666):
320321
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
321322
# in PC/launcher.c must also be updated.
322323

323-
MAGIC_NUMBER = (3431).to_bytes(2, 'little') + b'\r\n'
324+
MAGIC_NUMBER = (3432).to_bytes(2, 'little') + b'\r\n'
324325
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
325326

326327
_PYCACHE = '__pycache__'

‎Misc/NEWS.d/next/Core and Builtins/2020-11-16-18-13-07.bpo-42202.ZxenYD.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Change function parameters annotations internal representation to tuple
2+
of strings. Patch provided by Yurii Karabas.

‎Objects/funcobject.c

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,25 @@ func_get_annotations(PyFunctionObject *op, void *Py_UNUSED(ignored))
424424
if (op->func_annotations == NULL)
425425
return NULL;
426426
}
427+
if (PyTuple_CheckExact(op->func_annotations)) {
428+
PyObject *ann_tuple = op->func_annotations;
429+
PyObject *ann_dict = PyDict_New();
430+
if (ann_dict == NULL) {
431+
return NULL;
432+
}
433+
434+
assert(PyTuple_GET_SIZE(ann_tuple) % 2 == 0);
435+
436+
for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(ann_tuple); i += 2) {
437+
int err = PyDict_SetItem(ann_dict,
438+
PyTuple_GET_ITEM(ann_tuple, i),
439+
PyTuple_GET_ITEM(ann_tuple, i + 1));
440+
441+
if (err < 0)
442+
return NULL;
443+
}
444+
Py_SETREF(op->func_annotations, ann_dict);
445+
}
427446
Py_INCREF(op->func_annotations);
428447
return op->func_annotations;
429448
}

‎Python/ceval.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3880,7 +3880,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
38803880
func ->func_closure = POP();
38813881
}
38823882
if (oparg & 0x04) {
3883-
assert(PyDict_CheckExact(TOP()));
3883+
assert(PyTuple_CheckExact(TOP()));
38843884
func->func_annotations = POP();
38853885
}
38863886
if (oparg & 0x02) {

‎Python/compile.c

Lines changed: 27 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2027,26 +2027,24 @@ compiler_visit_annexpr(struct compiler *c, expr_ty annotation)
20272027

20282028
static int
20292029
compiler_visit_argannotation(struct compiler *c, identifier id,
2030-
expr_ty annotation, PyObject *names)
2030+
expr_ty annotation, Py_ssize_t *annotations_len)
20312031
{
20322032
if (annotation) {
2033-
PyObject *mangled;
2034-
VISIT(c, annexpr, annotation);
2035-
mangled = _Py_Mangle(c->u->u_private, id);
2033+
PyObject *mangled = _Py_Mangle(c->u->u_private, id);
20362034
if (!mangled)
20372035
return 0;
2038-
if (PyList_Append(names, mangled) < 0) {
2039-
Py_DECREF(mangled);
2040-
return 0;
2041-
}
2036+
2037+
ADDOP_LOAD_CONST(c, mangled);
20422038
Py_DECREF(mangled);
2039+
VISIT(c, annexpr, annotation);
2040+
*annotations_len += 2;
20432041
}
20442042
return 1;
20452043
}
20462044

20472045
static int
20482046
compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args,
2049-
PyObject *names)
2047+
Py_ssize_t *annotations_len)
20502048
{
20512049
int i;
20522050
for (i = 0; i < asdl_seq_LEN(args); i++) {
@@ -2055,7 +2053,7 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args,
20552053
c,
20562054
arg->arg,
20572055
arg->annotation,
2058-
names))
2056+
annotations_len))
20592057
return 0;
20602058
}
20612059
return 1;
@@ -2065,58 +2063,44 @@ static int
20652063
compiler_visit_annotations(struct compiler *c, arguments_ty args,
20662064
expr_ty returns)
20672065
{
2068-
/* Push arg annotation dict.
2066+
/* Push arg annotation names and values.
20692067
The expressions are evaluated out-of-order wrt the source code.
20702068
2071-
Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed.
2069+
Return 0 on error, -1 if no annotations pushed, 1 if a annotations is pushed.
20722070
*/
20732071
static identifier return_str;
2074-
PyObject *names;
2075-
Py_ssize_t len;
2076-
names = PyList_New(0);
2077-
if (!names)
2078-
return 0;
2072+
Py_ssize_t annotations_len = 0;
20792073

2080-
if (!compiler_visit_argannotations(c, args->args, names))
2081-
goto error;
2082-
if (!compiler_visit_argannotations(c, args->posonlyargs, names))
2083-
goto error;
2074+
if (!compiler_visit_argannotations(c, args->args, &annotations_len))
2075+
return 0;
2076+
if (!compiler_visit_argannotations(c, args->posonlyargs, &annotations_len))
2077+
return 0;
20842078
if (args->vararg && args->vararg->annotation &&
20852079
!compiler_visit_argannotation(c, args->vararg->arg,
2086-
args->vararg->annotation, names))
2087-
goto error;
2088-
if (!compiler_visit_argannotations(c, args->kwonlyargs, names))
2089-
goto error;
2080+
args->vararg->annotation, &annotations_len))
2081+
return 0;
2082+
if (!compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len))
2083+
return 0;
20902084
if (args->kwarg && args->kwarg->annotation &&
20912085
!compiler_visit_argannotation(c, args->kwarg->arg,
2092-
args->kwarg->annotation, names))
2093-
goto error;
2086+
args->kwarg->annotation, &annotations_len))
2087+
return 0;
20942088

20952089
if (!return_str) {
20962090
return_str = PyUnicode_InternFromString("return");
20972091
if (!return_str)
2098-
goto error;
2092+
return 0;
20992093
}
2100-
if (!compiler_visit_argannotation(c, return_str, returns, names)) {
2101-
goto error;
2094+
if (!compiler_visit_argannotation(c, return_str, returns, &annotations_len)) {
2095+
return 0;
21022096
}
21032097

2104-
len = PyList_GET_SIZE(names);
2105-
if (len) {
2106-
PyObject *keytuple = PyList_AsTuple(names);
2107-
Py_DECREF(names);
2108-
ADDOP_LOAD_CONST_NEW(c, keytuple);
2109-
ADDOP_I(c, BUILD_CONST_KEY_MAP, len);
2098+
if (annotations_len) {
2099+
ADDOP_I(c, BUILD_TUPLE, annotations_len);
21102100
return 1;
21112101
}
2112-
else {
2113-
Py_DECREF(names);
2114-
return -1;
2115-
}
21162102

2117-
error:
2118-
Py_DECREF(names);
2119-
return 0;
2103+
return -1;
21202104
}
21212105

21222106
static int

0 commit comments

Comments
 (0)