From fde5f1863f8a48aa7f0266b499a96a0a6ec78214 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 2 Nov 2023 12:14:45 +0000 Subject: [PATCH 1/4] Allow arbitrary annotations on instructions and micro-ops. Use one to declare specializing micro-ops --- Include/internal/pycore_opcode_metadata.h | 10 ++++++- Lib/test/test_generated_cases.py | 16 ++++-------- Python/abstract_interp_cases.c.h | 4 --- Python/bytecodes.c | 30 ++++++++++++--------- Python/executor_cases.c.h | 18 ------------- Tools/cases_generator/analysis.py | 14 +++------- Tools/cases_generator/generate_cases.py | 5 +++- Tools/cases_generator/instructions.py | 4 +++ Tools/cases_generator/lexer.py | 21 ++++++++++++--- Tools/cases_generator/parsing.py | 32 +++++++++++------------ 10 files changed, 74 insertions(+), 80 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index ed38ed0f39f94c..6783da746dc96f 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1771,6 +1771,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [END_SEND] = { .nuops = 1, .uops = { { END_SEND, 0, 0 } } }, [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { UNARY_NEGATIVE, 0, 0 } } }, [UNARY_NOT] = { .nuops = 1, .uops = { { UNARY_NOT, 0, 0 } } }, + [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { TO_BOOL_BOOL, 0, 0 } } }, [TO_BOOL_INT] = { .nuops = 1, .uops = { { TO_BOOL_INT, 0, 0 } } }, [TO_BOOL_LIST] = { .nuops = 1, .uops = { { TO_BOOL_LIST, 0, 0 } } }, @@ -1785,6 +1786,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, + [BINARY_SUBSCR] = { .nuops = 1, .uops = { { _BINARY_SUBSCR, 0, 0 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { BINARY_SLICE, 0, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } }, [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_LIST_INT, 0, 0 } } }, @@ -1793,6 +1795,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { SET_ADD, 0, 0 } } }, + [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, 0, 0 } } }, [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { STORE_SUBSCR_LIST_INT, 0, 0 } } }, [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { STORE_SUBSCR_DICT, 0, 0 } } }, [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, @@ -1808,17 +1811,19 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { LOAD_BUILD_CLASS, 0, 0 } } }, [STORE_NAME] = { .nuops = 1, .uops = { { STORE_NAME, 0, 0 } } }, [DELETE_NAME] = { .nuops = 1, .uops = { { DELETE_NAME, 0, 0 } } }, - [UNPACK_SEQUENCE] = { .nuops = 2, .uops = { { _SPECIALIZE_UNPACK_SEQUENCE, 1, 0 }, { _UNPACK_SEQUENCE, 0, 0 } } }, + [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, 0, 0 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_LIST, 0, 0 } } }, [UNPACK_EX] = { .nuops = 1, .uops = { { UNPACK_EX, 0, 0 } } }, + [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } }, [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } }, [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } }, [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } }, [LOAD_LOCALS] = { .nuops = 1, .uops = { { LOAD_LOCALS, 0, 0 } } }, [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { LOAD_NAME, 0, 0 } } }, + [LOAD_GLOBAL] = { .nuops = 1, .uops = { { _LOAD_GLOBAL, 0, 0 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } }, @@ -1842,6 +1847,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, + [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, @@ -1849,6 +1855,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, + [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, 0, 0 } } }, [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } }, [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } }, [COMPARE_OP_STR] = { .nuops = 1, .uops = { { COMPARE_OP_STR, 0, 0 } } }, @@ -1894,6 +1901,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { FORMAT_SIMPLE, 0, 0 } } }, [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { FORMAT_WITH_SPEC, 0, 0 } } }, [COPY] = { .nuops = 1, .uops = { { COPY, 0, 0 } } }, + [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, 0, 0 } } }, [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } }, }; #endif // NEED_OPCODE_METADATA diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index ea0c11621a2ca9..a545d872a8d25d 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -666,12 +666,9 @@ def test_macro_push_push(self): """ self.run_cases_test(input, output) - def test_override_inst(self): + def test_annotated_inst(self): input = """ - inst(OP, (--)) { - spam(); - } - override inst(OP, (--)) { + annotation inst(OP, (--)) { ham(); } """ @@ -686,22 +683,19 @@ def test_override_inst(self): """ self.run_cases_test(input, output) - def test_override_op(self): + def test_annotated_op(self): input = """ - op(OP, (--)) { + annotation op(OP, (--)) { spam(); } macro(M) = OP; - override op(OP, (--)) { - ham(); - } """ output = """ TARGET(M) { frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(M); - ham(); + spam(); DISPATCH(); } """ diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 0e58ed1b0be0f3..44ad9deb5ad8d9 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -242,10 +242,6 @@ break; } - case _SPECIALIZE_UNPACK_SEQUENCE: { - break; - } - case _UNPACK_SEQUENCE: { STACK_SHRINK(1); STACK_GROW(oparg); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f487e95854b7fa..e031de684801dd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -50,6 +50,10 @@ #define family(name, ...) static int family_##name #define pseudo(name) static int pseudo_##name +/* Annotations */ +#define guard +#define specializing + // Dummy variables for stack effects. static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub; static PyObject *container, *start, *stop, *v, *lhs, *rhs, *res2; @@ -312,7 +316,7 @@ dummy_func( TO_BOOL_STR, }; - op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) { + specializing op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -537,7 +541,7 @@ dummy_func( BINARY_SUBSCR_TUPLE_INT, }; - op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { + specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -689,7 +693,7 @@ dummy_func( STORE_SUBSCR_LIST_INT, }; - op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { + specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -974,7 +978,7 @@ dummy_func( SEND_GEN, }; - op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { + specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -1208,7 +1212,7 @@ dummy_func( UNPACK_SEQUENCE_LIST, }; - op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) { + specializing op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; @@ -1277,7 +1281,7 @@ dummy_func( STORE_ATTR_WITH_HINT, }; - op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { + specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -1404,7 +1408,7 @@ dummy_func( LOAD_GLOBAL_BUILTIN, }; - op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) { + specializing op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -1744,7 +1748,7 @@ dummy_func( LOAD_SUPER_ATTR_METHOD, }; - op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) { + specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION int load_method = oparg & 1; @@ -1860,7 +1864,7 @@ dummy_func( LOAD_ATTR_NONDESCRIPTOR_NO_DICT, }; - op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { + specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -2182,7 +2186,7 @@ dummy_func( COMPARE_OP_STR, }; - op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) { + specializing op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -2506,7 +2510,7 @@ dummy_func( FOR_ITER_GEN, }; - op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { + specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -3001,7 +3005,7 @@ dummy_func( CALL_ALLOC_AND_ENTER_INIT, }; - op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -3866,7 +3870,7 @@ dummy_func( top = Py_NewRef(bottom); } - op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { + specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { TIER_ONE_ONLY #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index eb56c34b432783..474aaced20fe62 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -871,24 +871,6 @@ break; } - case _SPECIALIZE_UNPACK_SEQUENCE: { - PyObject *seq; - seq = stack_pointer[-1]; - uint16_t counter = (uint16_t)next_uop[-1].operand; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { - next_instr = this_instr; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - (void)seq; - (void)counter; - break; - } - case _UNPACK_SEQUENCE: { PyObject *seq; seq = stack_pointer[-1]; diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index bd08e9de481e09..cf23fbf4f59b73 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -139,20 +139,12 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: match thing: case parsing.InstDef(name=name): macro: parsing.Macro | None = None - if thing.kind == "inst" and not thing.override: + if thing.kind == "inst": macro = parsing.Macro(name, [parsing.OpName(name)]) if name in self.instrs: - if not thing.override: - raise psr.make_syntax_error( - f"Duplicate definition of '{name}' @ {thing.context} " - f"previous definition @ {self.instrs[name].inst.context}", - thing_first_token, - ) - self.everything[instrs_idx[name]] = thing - if name not in self.instrs and thing.override: raise psr.make_syntax_error( - f"Definition of '{name}' @ {thing.context} is supposed to be " - "an override but no previous definition exists.", + f"Duplicate definition of '{name}' @ {thing.context} " + f"previous definition @ {self.instrs[name].inst.context}", thing_first_token, ) self.instrs[name] = Instruction(thing) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 54bd51885d9785..edfd89d2de7260 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -651,7 +651,10 @@ def write_macro_expansions( expansions: list[tuple[str, int, int]] = [] # [(name, size, offset), ...] for part in parts: if isinstance(part, Component): - # All component instructions must be viable uops + # Skip specializations + if part.instr.annotation == "specializing": + continue + # All other component instructions must be viable uops if not part.instr.is_viable_uop(): # This note just reminds us about macros that cannot # be expanded to Tier 2 uops. It is not an error. diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index f3e4f76697853b..b3989ac486a29c 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -46,6 +46,7 @@ class Instruction: # Parts of the underlying instruction definition inst: parsing.InstDef name: str + annotation: str | None block: parsing.Block block_text: list[str] # Block.text, less curlies, less PREDICT() calls block_line: int # First line of block in original code @@ -70,6 +71,7 @@ class Instruction: def __init__(self, inst: parsing.InstDef): self.inst = inst self.name = inst.name + self.annotation = inst.annotation self.block = inst.block self.block_text, self.check_eval_breaker, self.block_line = extract_block_text( self.block @@ -118,6 +120,8 @@ def is_viable_uop(self) -> bool: if self.name == "_EXIT_TRACE": return True # This has 'return frame' but it's okay + if self.name == "_SAVE_RETURN_OFFSET": + return True # Adjusts next_instr, but only in tier 1 code if self.always_exits: dprint(f"Skipping {self.name} because it always exits: {self.always_exits}") return False diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index a60f6c11a4c460..d52d1653332139 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -80,11 +80,12 @@ def choice(*opts: str) -> str: # Macros macro = r"# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)" -MACRO = "MACRO" +CMACRO = "CMACRO" id_re = r"[a-zA-Z_][0-9a-zA-Z_]*" IDENTIFIER = "IDENTIFIER" + suffix = r"([uU]?[lL]?[lL]?)" octal = r"0[0-7]+" + suffix hex = r"0[xX][0-9a-fA-F]+" @@ -173,8 +174,6 @@ def choice(*opts: str) -> str: kwds.append(INT) LONG = "LONG" kwds.append(LONG) -OVERRIDE = "OVERRIDE" -kwds.append(OVERRIDE) REGISTER = "REGISTER" kwds.append(REGISTER) OFFSETOF = "OFFSETOF" @@ -207,8 +206,20 @@ def choice(*opts: str) -> str: kwds.append(VOLATILE) WHILE = "WHILE" kwds.append(WHILE) +#An instruction in the DSL +INST = "INST" +kwds.append(INST) +#A micro-op in the DSL +OP = "OP" +kwds.append(OP) +#A macro in the DSL +MACRO = "MACRO" +kwds.append(MACRO) keywords = {name.lower(): name for name in kwds} +ANNOTATION = "ANNOTATION" +annotations = {"specializing", "guard"} + __all__ = [] __all__.extend(kwds) @@ -270,6 +281,8 @@ def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[T text = m.group(0) if text in keywords: kind = keywords[text] + elif text in annotations: + kind = ANNOTATION elif letter.match(text): kind = IDENTIFIER elif text == "...": @@ -289,7 +302,7 @@ def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[T elif text[0] == "'": kind = CHARACTER elif text[0] == "#": - kind = MACRO + kind = CMACRO elif text[0] == "/" and text[1] in "/*": kind = COMMENT else: diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 25be5ca3e0da5d..a54fcb3abd88df 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -105,8 +105,7 @@ class OpName(Node): @dataclass class InstHeader(Node): - override: bool - register: bool + annotation : str | None kind: Literal["inst", "op"] name: str inputs: list[InputEffect] @@ -115,8 +114,7 @@ class InstHeader(Node): @dataclass class InstDef(Node): - override: bool - register: bool + annotation : str | None kind: Literal["inst", "op"] name: str inputs: list[InputEffect] @@ -146,14 +144,14 @@ class Pseudo(Node): class Parser(PLexer): @contextual def definition(self) -> InstDef | Macro | Pseudo | Family | None: - if inst := self.inst_def(): - return inst if macro := self.macro_def(): return macro if family := self.family_def(): return family if pseudo := self.pseudo_def(): return pseudo + if inst := self.inst_def(): + return inst return None @contextual @@ -161,8 +159,7 @@ def inst_def(self) -> InstDef | None: if hdr := self.inst_header(): if block := self.block(): return InstDef( - hdr.override, - hdr.register, + hdr.annotation, hdr.kind, hdr.name, hdr.inputs, @@ -174,13 +171,14 @@ def inst_def(self) -> InstDef | None: @contextual def inst_header(self) -> InstHeader | None: - # [override] inst(NAME) - # | [override] [register] inst(NAME, (inputs -- outputs)) - # | [override] [register] op(NAME, (inputs -- outputs)) - # TODO: Make INST a keyword in the lexer. - override = bool(self.expect(lx.OVERRIDE)) - register = bool(self.expect(lx.REGISTER)) - if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text in ("inst", "op"): + # [annotation] inst(NAME, (inputs -- outputs)) + # | [annotation] op(NAME, (inputs -- outputs)) + anno = self.expect(lx.ANNOTATION) + annotation = None if anno is None else anno.text + tkn = self.expect(lx.INST) + if not tkn: + tkn = self.expect(lx.OP) + if tkn: kind = cast(Literal["inst", "op"], tkn.text) if self.expect(lx.LPAREN) and (tkn := self.expect(lx.IDENTIFIER)): name = tkn.text @@ -188,7 +186,7 @@ def inst_header(self) -> InstHeader | None: inp, outp = self.io_effect() if self.expect(lx.RPAREN): if (tkn := self.peek()) and tkn.kind == lx.LBRACE: - return InstHeader(override, register, kind, name, inp, outp) + return InstHeader(annotation, kind, name, inp, outp) return None def io_effect(self) -> tuple[list[InputEffect], list[OutputEffect]]: @@ -312,7 +310,7 @@ def op(self) -> OpName | None: @contextual def macro_def(self) -> Macro | None: - if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "macro": + if tkn := self.expect(lx.MACRO): if self.expect(lx.LPAREN): if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.RPAREN): From 140a6817ef826e03f16e4ce15c7f6a796599fb99 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 3 Nov 2023 23:12:16 +0000 Subject: [PATCH 2/4] Restore override behavior --- Lib/test/test_generated_cases.py | 45 +++++++++++++++++++++++++++++-- Python/bytecodes.c | 1 + Tools/cases_generator/analysis.py | 14 +++++++--- Tools/cases_generator/lexer.py | 8 +++--- 4 files changed, 59 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index a545d872a8d25d..e423563a962f7d 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -666,9 +666,50 @@ def test_macro_push_push(self): """ self.run_cases_test(input, output) + def test_override_inst(self): + input = """ + inst(OP, (--)) { + spam(); + } + override inst(OP, (--)) { + ham(); + } + """ + output = """ + TARGET(OP) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP); + ham(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_override_op(self): + input = """ + op(OP, (--)) { + spam(); + } + macro(M) = OP; + override op(OP, (--)) { + ham(); + } + """ + output = """ + TARGET(M) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(M); + ham(); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + def test_annotated_inst(self): input = """ - annotation inst(OP, (--)) { + guard inst(OP, (--)) { ham(); } """ @@ -685,7 +726,7 @@ def test_annotated_inst(self): def test_annotated_op(self): input = """ - annotation op(OP, (--)) { + guard op(OP, (--)) { spam(); } macro(M) = OP; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e031de684801dd..8b7640d9141f71 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -52,6 +52,7 @@ /* Annotations */ #define guard +#define override #define specializing // Dummy variables for stack effects. diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index cf23fbf4f59b73..0c0b15f7665027 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -139,12 +139,20 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: match thing: case parsing.InstDef(name=name): macro: parsing.Macro | None = None - if thing.kind == "inst": + if thing.kind == "inst" and not thing.annotation == "override": macro = parsing.Macro(name, [parsing.OpName(name)]) if name in self.instrs: + if not thing.annotation == "override": + raise psr.make_syntax_error( + f"Duplicate definition of '{name}' @ {thing.context} " + f"previous definition @ {self.instrs[name].inst.context}", + thing_first_token, + ) + self.everything[instrs_idx[name]] = thing + if name not in self.instrs and thing.annotation == "override": raise psr.make_syntax_error( - f"Duplicate definition of '{name}' @ {thing.context} " - f"previous definition @ {self.instrs[name].inst.context}", + f"Definition of '{name}' @ {thing.context} is supposed to be " + "an override but no previous definition exists.", thing_first_token, ) self.instrs[name] = Instruction(thing) diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index d52d1653332139..a5f133493d4c4f 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -206,19 +206,19 @@ def choice(*opts: str) -> str: kwds.append(VOLATILE) WHILE = "WHILE" kwds.append(WHILE) -#An instruction in the DSL +# An instruction in the DSL INST = "INST" kwds.append(INST) -#A micro-op in the DSL +# A micro-op in the DSL OP = "OP" kwds.append(OP) -#A macro in the DSL +# A macro in the DSL MACRO = "MACRO" kwds.append(MACRO) keywords = {name.lower(): name for name in kwds} ANNOTATION = "ANNOTATION" -annotations = {"specializing", "guard"} +annotations = {"specializing", "guard", "override", "register"} __all__ = [] __all__.extend(kwds) From febc85cade1c0003d3a58486e2ad256ffb7f4448 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 3 Nov 2023 23:32:06 +0000 Subject: [PATCH 3/4] Allow multiple annotations --- Lib/test/test_generated_cases.py | 8 ++++++++ Tools/cases_generator/analysis.py | 6 +++--- Tools/cases_generator/generate_cases.py | 2 +- Tools/cases_generator/instructions.py | 4 ++-- Tools/cases_generator/lexer.py | 2 -- Tools/cases_generator/parsing.py | 13 +++++++------ 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index e423563a962f7d..585be642f0427d 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -742,6 +742,14 @@ def test_annotated_op(self): """ self.run_cases_test(input, output) + input = """ + guard register specializing op(OP, (--)) { + spam(); + } + macro(M) = OP; + """ + self.run_cases_test(input, output) + if __name__ == "__main__": unittest.main() diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 0c0b15f7665027..ee93b881d884bf 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -139,17 +139,17 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: match thing: case parsing.InstDef(name=name): macro: parsing.Macro | None = None - if thing.kind == "inst" and not thing.annotation == "override": + if thing.kind == "inst" and "override" not in thing.annotations: macro = parsing.Macro(name, [parsing.OpName(name)]) if name in self.instrs: - if not thing.annotation == "override": + if "override" not in thing.annotations: raise psr.make_syntax_error( f"Duplicate definition of '{name}' @ {thing.context} " f"previous definition @ {self.instrs[name].inst.context}", thing_first_token, ) self.everything[instrs_idx[name]] = thing - if name not in self.instrs and thing.annotation == "override": + if name not in self.instrs and "override" in thing.annotations: raise psr.make_syntax_error( f"Definition of '{name}' @ {thing.context} is supposed to be " "an override but no previous definition exists.", diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index edfd89d2de7260..a3313b4929afc4 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -652,7 +652,7 @@ def write_macro_expansions( for part in parts: if isinstance(part, Component): # Skip specializations - if part.instr.annotation == "specializing": + if "specializing" in part.instr.annotations: continue # All other component instructions must be viable uops if not part.instr.is_viable_uop(): diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index b3989ac486a29c..181d8badf7bf22 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -46,7 +46,7 @@ class Instruction: # Parts of the underlying instruction definition inst: parsing.InstDef name: str - annotation: str | None + annotations: list[str] block: parsing.Block block_text: list[str] # Block.text, less curlies, less PREDICT() calls block_line: int # First line of block in original code @@ -71,7 +71,7 @@ class Instruction: def __init__(self, inst: parsing.InstDef): self.inst = inst self.name = inst.name - self.annotation = inst.annotation + self.annotations = inst.annotations self.block = inst.block self.block_text, self.check_eval_breaker, self.block_line = extract_block_text( self.block diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index a5f133493d4c4f..4c5403dc1d1fa8 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -174,8 +174,6 @@ def choice(*opts: str) -> str: kwds.append(INT) LONG = "LONG" kwds.append(LONG) -REGISTER = "REGISTER" -kwds.append(REGISTER) OFFSETOF = "OFFSETOF" kwds.append(OFFSETOF) RESTRICT = "RESTRICT" diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index a54fcb3abd88df..5b56ed1cec4b22 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -105,7 +105,7 @@ class OpName(Node): @dataclass class InstHeader(Node): - annotation : str | None + annotations : list[str] kind: Literal["inst", "op"] name: str inputs: list[InputEffect] @@ -114,7 +114,7 @@ class InstHeader(Node): @dataclass class InstDef(Node): - annotation : str | None + annotations : list[str] kind: Literal["inst", "op"] name: str inputs: list[InputEffect] @@ -159,7 +159,7 @@ def inst_def(self) -> InstDef | None: if hdr := self.inst_header(): if block := self.block(): return InstDef( - hdr.annotation, + hdr.annotations, hdr.kind, hdr.name, hdr.inputs, @@ -173,8 +173,9 @@ def inst_def(self) -> InstDef | None: def inst_header(self) -> InstHeader | None: # [annotation] inst(NAME, (inputs -- outputs)) # | [annotation] op(NAME, (inputs -- outputs)) - anno = self.expect(lx.ANNOTATION) - annotation = None if anno is None else anno.text + annotations = [] + while anno := self.expect(lx.ANNOTATION): + annotations.append(anno.text) tkn = self.expect(lx.INST) if not tkn: tkn = self.expect(lx.OP) @@ -186,7 +187,7 @@ def inst_header(self) -> InstHeader | None: inp, outp = self.io_effect() if self.expect(lx.RPAREN): if (tkn := self.peek()) and tkn.kind == lx.LBRACE: - return InstHeader(annotation, kind, name, inp, outp) + return InstHeader(annotations, kind, name, inp, outp) return None def io_effect(self) -> tuple[list[InputEffect], list[OutputEffect]]: From 5d216dac71871e67d8ab226569bfe079c467b412 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 7 Nov 2023 09:13:52 +0000 Subject: [PATCH 4/4] Update Tools/cases_generator/parsing.py Co-authored-by: Guido van Rossum --- Tools/cases_generator/parsing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 5b56ed1cec4b22..49459be68ae5e8 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -171,8 +171,8 @@ def inst_def(self) -> InstDef | None: @contextual def inst_header(self) -> InstHeader | None: - # [annotation] inst(NAME, (inputs -- outputs)) - # | [annotation] op(NAME, (inputs -- outputs)) + # annotation* inst(NAME, (inputs -- outputs)) + # | annotation* op(NAME, (inputs -- outputs)) annotations = [] while anno := self.expect(lx.ANNOTATION): annotations.append(anno.text)