From 65535ee181cc6787363d9832dbe10ada7dd05856 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 29 Apr 2025 02:28:14 +0100 Subject: [PATCH 1/7] Use Ruff for all formatting in pre-commit --- .pre-commit-config.yaml | 17 +++++------- Tools/build/check_warnings.py | 1 - Tools/jit/_llvm.py | 4 +-- Tools/jit/_stencils.py | 52 +++++++++++++++++++---------------- Tools/jit/_targets.py | 12 ++------ 5 files changed, 40 insertions(+), 46 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 048cff1d32c7af..9ba3030c2a4441 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.6 + rev: v0.11.7 hooks: - id: ruff name: Run Ruff (lint) on Doc/ @@ -22,16 +22,13 @@ repos: name: Run Ruff (format) on Doc/ args: [--check] files: ^Doc/ - - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 - hooks: - - id: black - name: Run Black on Tools/build/check_warnings.py + - id: ruff-format + name: Run Ruff (format) on Tools/build/check_warnings.py + args: [--check, --config=Tools/build/.ruff.toml] files: ^Tools/build/check_warnings.py - args: [--line-length=79] - - id: black - name: Run Black on Tools/jit/ + - id: ruff-format + name: Run Ruff (format) on Tools/jit/ + args: [--check] files: ^Tools/jit/ - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 7a8721087b6c66..621f431beb9a3f 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -151,7 +151,6 @@ def get_unexpected_warnings( """ unexpected_warnings = {} for file in files_with_warnings.keys(): - rule = is_file_ignored(file, ignore_rules) if rule: diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 925b56ac669aa0..903eaa16fbc0c2 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -21,9 +21,7 @@ def _async_cache(f: _C[_P, _R]) -> _C[_P, _R]: lock = asyncio.Lock() @functools.wraps(f) - async def wrapper( - *args: _P.args, **kwargs: _P.kwargs # pylint: disable = no-member - ) -> _R: + async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: async with lock: if args not in cache: cache[args] = await f(*args, **kwargs) diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 22d3014351df90..8ae05528b1e6e5 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -230,35 +230,41 @@ def remove_jump(self) -> None: """Remove a zero-length continuation jump, if it exists.""" hole = max(self.holes, key=lambda hole: hole.offset) match hole: - case Hole( - offset=offset, - kind="IMAGE_REL_AMD64_REL32", - value=HoleValue.GOT, - symbol="_JIT_CONTINUE", - addend=-4, - ) as hole: + case ( + Hole( + offset=offset, + kind="IMAGE_REL_AMD64_REL32", + value=HoleValue.GOT, + symbol="_JIT_CONTINUE", + addend=-4, + ) as hole + ): # jmp qword ptr [rip] jump = b"\x48\xff\x25\x00\x00\x00\x00" offset -= 3 - case Hole( - offset=offset, - kind="IMAGE_REL_I386_REL32" | "R_X86_64_PLT32" | "X86_64_RELOC_BRANCH", - value=HoleValue.CONTINUE, - symbol=None, - addend=addend, - ) as hole if ( - _signed(addend) == -4 - ): + case ( + Hole( + offset=offset, + kind="IMAGE_REL_I386_REL32" + | "R_X86_64_PLT32" + | "X86_64_RELOC_BRANCH", + value=HoleValue.CONTINUE, + symbol=None, + addend=addend, + ) as hole + ) if _signed(addend) == -4: # jmp 5 jump = b"\xe9\x00\x00\x00\x00" offset -= 1 - case Hole( - offset=offset, - kind="R_AARCH64_JUMP26", - value=HoleValue.CONTINUE, - symbol=None, - addend=0, - ) as hole: + case ( + Hole( + offset=offset, + kind="R_AARCH64_JUMP26", + value=HoleValue.CONTINUE, + symbol=None, + addend=0, + ) as hole + ): # b #4 jump = b"\x00\x00\x00\x14" case _: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index f7ea159884c5a4..5d2aece1d59ddc 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -224,9 +224,7 @@ def build( jit_stencils_new.unlink(missing_ok=True) -class _COFF( - _Target[_schema.COFFSection, _schema.COFFRelocation] -): # pylint: disable = too-few-public-methods +class _COFF(_Target[_schema.COFFSection, _schema.COFFRelocation]): def _handle_section( self, section: _schema.COFFSection, group: _stencils.StencilGroup ) -> None: @@ -312,9 +310,7 @@ def _handle_relocation( return _stencils.Hole(offset, kind, value, symbol, addend) -class _ELF( - _Target[_schema.ELFSection, _schema.ELFRelocation] -): # pylint: disable = too-few-public-methods +class _ELF(_Target[_schema.ELFSection, _schema.ELFRelocation]): def _handle_section( self, section: _schema.ELFSection, group: _stencils.StencilGroup ) -> None: @@ -402,9 +398,7 @@ def _handle_relocation( return _stencils.Hole(offset, kind, value, symbol, addend) -class _MachO( - _Target[_schema.MachOSection, _schema.MachORelocation] -): # pylint: disable = too-few-public-methods +class _MachO(_Target[_schema.MachOSection, _schema.MachORelocation]): def _handle_section( self, section: _schema.MachOSection, group: _stencils.StencilGroup ) -> None: From ba8e4bff0e134e9883b727e97c63cbccec8d4ccf Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 1 May 2025 10:15:01 +0100 Subject: [PATCH 2/7] Add Tools/jit/.ruff.toml --- .pre-commit-config.yaml | 2 +- Tools/jit/.ruff.toml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 Tools/jit/.ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ba3030c2a4441..1b180313b0a19f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: files: ^Tools/build/check_warnings.py - id: ruff-format name: Run Ruff (format) on Tools/jit/ - args: [--check] + args: [--check, --config=Tools/jit/.ruff.toml] files: ^Tools/jit/ - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/Tools/jit/.ruff.toml b/Tools/jit/.ruff.toml new file mode 100644 index 00000000000000..1db9901657e462 --- /dev/null +++ b/Tools/jit/.ruff.toml @@ -0,0 +1,7 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +line-length = 88 + +[format] +preview = true +docstring-code-format = true From 42ce0146d730001283ec193167d822f265afe18a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Thu, 1 May 2025 10:21:26 +0100 Subject: [PATCH 3/7] PEP 8 (keep to 80 chars) --- Tools/jit/.ruff.toml | 2 -- Tools/jit/_llvm.py | 20 +++++++++---- Tools/jit/_schema.py | 12 ++++++-- Tools/jit/_stencils.py | 28 +++++++++++++---- Tools/jit/_targets.py | 68 ++++++++++++++++++++++++++++++++---------- Tools/jit/_writer.py | 8 +++-- Tools/jit/build.py | 15 ++++++++-- 7 files changed, 117 insertions(+), 36 deletions(-) diff --git a/Tools/jit/.ruff.toml b/Tools/jit/.ruff.toml index 1db9901657e462..4b949f636c45b3 100644 --- a/Tools/jit/.ruff.toml +++ b/Tools/jit/.ruff.toml @@ -1,7 +1,5 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings -line-length = 88 - [format] preview = true docstring-code-format = true diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 903eaa16fbc0c2..bead010bda3267 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -9,7 +9,9 @@ import typing _LLVM_VERSION = 19 -_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") +_LLVM_VERSION_PATTERN = re.compile( + rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+" +) _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") @@ -33,7 +35,9 @@ async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: _CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) -async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None: +async def _run( + tool: str, args: typing.Iterable[str], echo: bool = False +) -> str | None: command = [tool, *args] async with _CORES: if echo: @@ -46,7 +50,9 @@ async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str return None out, _ = await process.communicate() if process.returncode: - raise RuntimeError(f"{tool} exited with return code {process.returncode}") + raise RuntimeError( + f"{tool} exited with return code {process.returncode}" + ) return out.decode() @@ -58,7 +64,9 @@ async def _check_tool_version(name: str, *, echo: bool = False) -> bool: @_async_cache async def _get_brew_llvm_prefix(*, echo: bool = False) -> str | None: - output = await _run("brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo) + output = await _run( + "brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo + ) return output and output.removesuffix("\n") @@ -90,7 +98,9 @@ async def maybe_run( return path and await _run(path, args, echo=echo) -async def run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str: +async def run( + tool: str, args: typing.Iterable[str], echo: bool = False +) -> str: """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" output = await maybe_run(tool, args, echo=echo) if output is None: diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 228fc389584dd7..72e0425ece6d2a 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -97,7 +97,9 @@ class COFFSection(typing.TypedDict): class ELFSection(typing.TypedDict): """An ELF object file section.""" - Flags: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] + Flags: dict[ + typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] + ] Index: int Info: int Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] @@ -110,11 +112,15 @@ class MachOSection(typing.TypedDict): """A Mach-O object file section.""" Address: int - Attributes: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] + Attributes: dict[ + typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] + ] Index: int Name: dict[typing.Literal["Value"], str] Relocations: typing.NotRequired[ list[dict[typing.Literal["Relocation"], MachORelocation]] ] SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]] - Symbols: typing.NotRequired[list[dict[typing.Literal["Symbol"], _MachOSymbol]]] + Symbols: typing.NotRequired[ + list[dict[typing.Literal["Symbol"], _MachOSymbol]] + ] diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index dc178c78a824a6..7ba7cd0809cd74 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -199,14 +199,18 @@ class Stencil: body: bytearray = dataclasses.field(default_factory=bytearray, init=False) holes: list[Hole] = dataclasses.field(default_factory=list, init=False) - disassembly: list[str] = dataclasses.field(default_factory=list, init=False) + disassembly: list[str] = dataclasses.field( + default_factory=list, init=False + ) def pad(self, alignment: int) -> None: """Pad the stencil to the given alignment.""" offset = len(self.body) padding = -offset % alignment if padding: - self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") + self.disassembly.append( + f"{offset:x}: {' '.join(['00'] * padding)}" + ) self.body.extend([0] * padding) def add_nops(self, nop: bytes, alignment: int) -> None: @@ -291,13 +295,21 @@ class StencilGroup: _trampolines: set[int] = dataclasses.field(default_factory=set, init=False) def process_relocations( - self, known_symbols: dict[str, int], *, alignment: int = 1, nop: bytes = b"" + self, + known_symbols: dict[str, int], + *, + alignment: int = 1, + nop: bytes = b"", ) -> None: """Fix up all GOT and internal relocations for this stencil group.""" for hole in self.code.holes.copy(): if ( hole.kind - in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"} + in { + "R_AARCH64_CALL26", + "R_AARCH64_JUMP26", + "ARM64_RELOC_BRANCH26", + } and hole.value is HoleValue.ZERO and hole.symbol not in self.symbols ): @@ -320,7 +332,9 @@ def process_relocations( if hole.value is HoleValue.GOT: assert hole.symbol is not None hole.value = HoleValue.DATA - hole.addend += self._global_offset_table_lookup(hole.symbol) + hole.addend += self._global_offset_table_lookup( + hole.symbol + ) hole.symbol = None elif hole.symbol in self.symbols: hole.value, addend = self.symbols[hole.symbol] @@ -338,7 +352,9 @@ def process_relocations( self.data.holes.sort(key=lambda hole: hole.offset) def _global_offset_table_lookup(self, symbol: str) -> int: - return len(self.data.body) + self._got.setdefault(symbol, 8 * len(self._got)) + return len(self.data.body) + self._got.setdefault( + symbol, 8 * len(self._got) + ) def _emit_global_offset_table(self) -> None: got = len(self.data.body) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index cbfb0f2120d34f..8ef2416e91b59e 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -17,7 +17,9 @@ import _writer if sys.version_info < (3, 11): - raise RuntimeError("Building the JIT compiler requires Python 3.11 or newer!") + raise RuntimeError( + "Building the JIT compiler requires Python 3.11 or newer!" + ) TOOLS_JIT_BUILD = pathlib.Path(__file__).resolve() TOOLS_JIT = TOOLS_JIT_BUILD.parent @@ -27,9 +29,14 @@ TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" ASYNCIO_RUNNER = asyncio.Runner() -_S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) +_S = typing.TypeVar( + "_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection +) _R = typing.TypeVar( - "_R", _schema.COFFRelocation, _schema.ELFRelocation, _schema.MachORelocation + "_R", + _schema.COFFRelocation, + _schema.ELFRelocation, + _schema.MachORelocation, ) @@ -95,7 +102,9 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: # ...and also COFF: output = output[output.index("[", 1, None) :] output = output[: output.rindex("]", None, -1) + 1] - sections: list[dict[typing.Literal["Section"], _S]] = json.loads(output) + sections: list[dict[typing.Literal["Section"], _S]] = json.loads( + output + ) for wrapped_section in sections: self._handle_section(wrapped_section["Section"], group) assert group.symbols["_JIT_ENTRY"] == (_stencils.HoleValue.CODE, 0) @@ -104,7 +113,9 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: group.data.disassembly.append(line) return group - def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: + def _handle_section( + self, section: _S, group: _stencils.StencilGroup + ) -> None: raise NotImplementedError(type(self)) def _handle_relocation( @@ -160,7 +171,9 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() cases_and_opnames = sorted( re.findall( - r"\n {8}(case (\w+): \{\n.*?\n {8}\})", generated_cases, flags=re.DOTALL + r"\n {8}(case (\w+): \{\n.*?\n {8}\})", + generated_cases, + flags=re.DOTALL, ) ) tasks = [] @@ -200,7 +213,9 @@ def build( """Build jit_stencils.h in the given directory.""" if not self.stable: warning = f"JIT support for {self.triple} is still experimental!" - request = "Please report any issues you encounter.".center(len(warning)) + request = "Please report any issues you encounter.".center( + len(warning) + ) outline = "=" * len(warning) print("\n".join(["", outline, warning, request, outline, ""])) digest = f"// {self._compute_digest(out)}\n" @@ -264,7 +279,9 @@ def _handle_section( hole = self._handle_relocation(base, relocation, stencil.body) stencil.holes.append(hole) - def _unwrap_dllimport(self, name: str) -> tuple[_stencils.HoleValue, str | None]: + def _unwrap_dllimport( + self, name: str + ) -> tuple[_stencils.HoleValue, str | None]: if name.startswith("__imp_"): name = name.removeprefix("__imp_") name = name.removeprefix(self.prefix) @@ -291,13 +308,17 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind + "Name": "IMAGE_REL_AMD64_REL32" + | "IMAGE_REL_I386_REL32" as kind }, }: offset += base value, symbol = self._unwrap_dllimport(s) addend = ( - int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + int.from_bytes( + raw[offset : offset + 4], "little", signed=True + ) + - 4 ) case { "Offset": offset, @@ -429,7 +450,12 @@ def _handle_section( base = section["Address"] - start_address group.symbols[section["Index"]] = value, base stencil.body.extend( - [0] * (section["Address"] - len(group.code.body) - len(group.data.body)) + [0] + * ( + section["Address"] + - len(group.code.body) + - len(group.data.body) + ) ) stencil.body.extend(section["SectionData"]["Bytes"]) assert "Symbols" in section @@ -468,13 +494,19 @@ def _handle_relocation( case { "Offset": offset, "Symbol": {"Name": s}, - "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, + "Type": { + "Name": "X86_64_RELOC_GOT" + | "X86_64_RELOC_GOT_LOAD" as kind + }, }: offset += base s = s.removeprefix(self.prefix) value, symbol = _stencils.HoleValue.GOT, s addend = ( - int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + int.from_bytes( + raw[offset : offset + 4], "little", signed=True + ) + - 4 ) case { "Offset": offset, @@ -483,13 +515,19 @@ def _handle_relocation( } | { "Offset": offset, "Symbol": {"Name": s}, - "Type": {"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind}, + "Type": { + "Name": "X86_64_RELOC_BRANCH" + | "X86_64_RELOC_SIGNED" as kind + }, }: offset += base s = s.removeprefix(self.prefix) value, symbol = _stencils.symbol_to_value(s) addend = ( - int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 + int.from_bytes( + raw[offset : offset + 4], "little", signed=True + ) + - 4 ) case { "Offset": offset, diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 090b52660f009c..689f6b3f7743df 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -40,7 +40,9 @@ def _dump_footer( yield "};" -def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]: +def _dump_stencil( + opname: str, group: _stencils.StencilGroup +) -> typing.Iterator[str]: yield "void" yield f"emit_{opname}(" yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," @@ -62,7 +64,9 @@ def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator yield f" memcpy({part}, {part}_body, sizeof({part}_body));" skip = False stencil.holes.sort(key=lambda hole: hole.offset) - for hole, pair in itertools.zip_longest(stencil.holes, stencil.holes[1:]): + for hole, pair in itertools.zip_longest( + stencil.holes, stencil.holes[1:] + ): if skip: skip = False continue diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 4d1e484b6838eb..5a67b71e8b8112 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -17,13 +17,22 @@ help="a PEP 11 target triple to compile for", ) parser.add_argument( - "-d", "--debug", action="store_true", help="compile for a debug build of Python" + "-d", + "--debug", + action="store_true", + help="compile for a debug build of Python", ) parser.add_argument( - "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt" + "-f", + "--force", + action="store_true", + help="force the entire JIT to be rebuilt", ) parser.add_argument( - "-v", "--verbose", action="store_true", help="echo commands as they are run" + "-v", + "--verbose", + action="store_true", + help="echo commands as they are run", ) args = parser.parse_args() for target in args.target: From e36f747c9cbac431da1c7b2f33c0e4f92c8a5c87 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 2 May 2025 19:29:03 +0100 Subject: [PATCH 4/7] Revert "PEP 8 (keep to 80 chars)" This reverts commit 42ce0146d730001283ec193167d822f265afe18a. --- Tools/jit/.ruff.toml | 2 ++ Tools/jit/_llvm.py | 20 ++++--------- Tools/jit/_schema.py | 12 ++------ Tools/jit/_stencils.py | 28 ++++------------- Tools/jit/_targets.py | 68 ++++++++++-------------------------------- Tools/jit/_writer.py | 8 ++--- Tools/jit/build.py | 15 ++-------- 7 files changed, 36 insertions(+), 117 deletions(-) diff --git a/Tools/jit/.ruff.toml b/Tools/jit/.ruff.toml index 4b949f636c45b3..1db9901657e462 100644 --- a/Tools/jit/.ruff.toml +++ b/Tools/jit/.ruff.toml @@ -1,5 +1,7 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings +line-length = 88 + [format] preview = true docstring-code-format = true diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index bead010bda3267..903eaa16fbc0c2 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -9,9 +9,7 @@ import typing _LLVM_VERSION = 19 -_LLVM_VERSION_PATTERN = re.compile( - rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+" -) +_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") @@ -35,9 +33,7 @@ async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: _CORES = asyncio.BoundedSemaphore(os.cpu_count() or 1) -async def _run( - tool: str, args: typing.Iterable[str], echo: bool = False -) -> str | None: +async def _run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str | None: command = [tool, *args] async with _CORES: if echo: @@ -50,9 +46,7 @@ async def _run( return None out, _ = await process.communicate() if process.returncode: - raise RuntimeError( - f"{tool} exited with return code {process.returncode}" - ) + raise RuntimeError(f"{tool} exited with return code {process.returncode}") return out.decode() @@ -64,9 +58,7 @@ async def _check_tool_version(name: str, *, echo: bool = False) -> bool: @_async_cache async def _get_brew_llvm_prefix(*, echo: bool = False) -> str | None: - output = await _run( - "brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo - ) + output = await _run("brew", ["--prefix", f"llvm@{_LLVM_VERSION}"], echo=echo) return output and output.removesuffix("\n") @@ -98,9 +90,7 @@ async def maybe_run( return path and await _run(path, args, echo=echo) -async def run( - tool: str, args: typing.Iterable[str], echo: bool = False -) -> str: +async def run(tool: str, args: typing.Iterable[str], echo: bool = False) -> str: """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError.""" output = await maybe_run(tool, args, echo=echo) if output is None: diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 72e0425ece6d2a..228fc389584dd7 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -97,9 +97,7 @@ class COFFSection(typing.TypedDict): class ELFSection(typing.TypedDict): """An ELF object file section.""" - Flags: dict[ - typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] - ] + Flags: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] Index: int Info: int Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] @@ -112,15 +110,11 @@ class MachOSection(typing.TypedDict): """A Mach-O object file section.""" Address: int - Attributes: dict[ - typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] - ] + Attributes: dict[typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]]] Index: int Name: dict[typing.Literal["Value"], str] Relocations: typing.NotRequired[ list[dict[typing.Literal["Relocation"], MachORelocation]] ] SectionData: typing.NotRequired[dict[typing.Literal["Bytes"], list[int]]] - Symbols: typing.NotRequired[ - list[dict[typing.Literal["Symbol"], _MachOSymbol]] - ] + Symbols: typing.NotRequired[list[dict[typing.Literal["Symbol"], _MachOSymbol]]] diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 7ba7cd0809cd74..dc178c78a824a6 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -199,18 +199,14 @@ class Stencil: body: bytearray = dataclasses.field(default_factory=bytearray, init=False) holes: list[Hole] = dataclasses.field(default_factory=list, init=False) - disassembly: list[str] = dataclasses.field( - default_factory=list, init=False - ) + disassembly: list[str] = dataclasses.field(default_factory=list, init=False) def pad(self, alignment: int) -> None: """Pad the stencil to the given alignment.""" offset = len(self.body) padding = -offset % alignment if padding: - self.disassembly.append( - f"{offset:x}: {' '.join(['00'] * padding)}" - ) + self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") self.body.extend([0] * padding) def add_nops(self, nop: bytes, alignment: int) -> None: @@ -295,21 +291,13 @@ class StencilGroup: _trampolines: set[int] = dataclasses.field(default_factory=set, init=False) def process_relocations( - self, - known_symbols: dict[str, int], - *, - alignment: int = 1, - nop: bytes = b"", + self, known_symbols: dict[str, int], *, alignment: int = 1, nop: bytes = b"" ) -> None: """Fix up all GOT and internal relocations for this stencil group.""" for hole in self.code.holes.copy(): if ( hole.kind - in { - "R_AARCH64_CALL26", - "R_AARCH64_JUMP26", - "ARM64_RELOC_BRANCH26", - } + in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"} and hole.value is HoleValue.ZERO and hole.symbol not in self.symbols ): @@ -332,9 +320,7 @@ def process_relocations( if hole.value is HoleValue.GOT: assert hole.symbol is not None hole.value = HoleValue.DATA - hole.addend += self._global_offset_table_lookup( - hole.symbol - ) + hole.addend += self._global_offset_table_lookup(hole.symbol) hole.symbol = None elif hole.symbol in self.symbols: hole.value, addend = self.symbols[hole.symbol] @@ -352,9 +338,7 @@ def process_relocations( self.data.holes.sort(key=lambda hole: hole.offset) def _global_offset_table_lookup(self, symbol: str) -> int: - return len(self.data.body) + self._got.setdefault( - symbol, 8 * len(self._got) - ) + return len(self.data.body) + self._got.setdefault(symbol, 8 * len(self._got)) def _emit_global_offset_table(self) -> None: got = len(self.data.body) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 8ef2416e91b59e..cbfb0f2120d34f 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -17,9 +17,7 @@ import _writer if sys.version_info < (3, 11): - raise RuntimeError( - "Building the JIT compiler requires Python 3.11 or newer!" - ) + raise RuntimeError("Building the JIT compiler requires Python 3.11 or newer!") TOOLS_JIT_BUILD = pathlib.Path(__file__).resolve() TOOLS_JIT = TOOLS_JIT_BUILD.parent @@ -29,14 +27,9 @@ TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c" ASYNCIO_RUNNER = asyncio.Runner() -_S = typing.TypeVar( - "_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection -) +_S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection) _R = typing.TypeVar( - "_R", - _schema.COFFRelocation, - _schema.ELFRelocation, - _schema.MachORelocation, + "_R", _schema.COFFRelocation, _schema.ELFRelocation, _schema.MachORelocation ) @@ -102,9 +95,7 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: # ...and also COFF: output = output[output.index("[", 1, None) :] output = output[: output.rindex("]", None, -1) + 1] - sections: list[dict[typing.Literal["Section"], _S]] = json.loads( - output - ) + sections: list[dict[typing.Literal["Section"], _S]] = json.loads(output) for wrapped_section in sections: self._handle_section(wrapped_section["Section"], group) assert group.symbols["_JIT_ENTRY"] == (_stencils.HoleValue.CODE, 0) @@ -113,9 +104,7 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: group.data.disassembly.append(line) return group - def _handle_section( - self, section: _S, group: _stencils.StencilGroup - ) -> None: + def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: raise NotImplementedError(type(self)) def _handle_relocation( @@ -171,9 +160,7 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: generated_cases = PYTHON_EXECUTOR_CASES_C_H.read_text() cases_and_opnames = sorted( re.findall( - r"\n {8}(case (\w+): \{\n.*?\n {8}\})", - generated_cases, - flags=re.DOTALL, + r"\n {8}(case (\w+): \{\n.*?\n {8}\})", generated_cases, flags=re.DOTALL ) ) tasks = [] @@ -213,9 +200,7 @@ def build( """Build jit_stencils.h in the given directory.""" if not self.stable: warning = f"JIT support for {self.triple} is still experimental!" - request = "Please report any issues you encounter.".center( - len(warning) - ) + request = "Please report any issues you encounter.".center(len(warning)) outline = "=" * len(warning) print("\n".join(["", outline, warning, request, outline, ""])) digest = f"// {self._compute_digest(out)}\n" @@ -279,9 +264,7 @@ def _handle_section( hole = self._handle_relocation(base, relocation, stencil.body) stencil.holes.append(hole) - def _unwrap_dllimport( - self, name: str - ) -> tuple[_stencils.HoleValue, str | None]: + def _unwrap_dllimport(self, name: str) -> tuple[_stencils.HoleValue, str | None]: if name.startswith("__imp_"): name = name.removeprefix("__imp_") name = name.removeprefix(self.prefix) @@ -308,17 +291,13 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Name": "IMAGE_REL_AMD64_REL32" - | "IMAGE_REL_I386_REL32" as kind + "Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind }, }: offset += base value, symbol = self._unwrap_dllimport(s) addend = ( - int.from_bytes( - raw[offset : offset + 4], "little", signed=True - ) - - 4 + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 ) case { "Offset": offset, @@ -450,12 +429,7 @@ def _handle_section( base = section["Address"] - start_address group.symbols[section["Index"]] = value, base stencil.body.extend( - [0] - * ( - section["Address"] - - len(group.code.body) - - len(group.data.body) - ) + [0] * (section["Address"] - len(group.code.body) - len(group.data.body)) ) stencil.body.extend(section["SectionData"]["Bytes"]) assert "Symbols" in section @@ -494,19 +468,13 @@ def _handle_relocation( case { "Offset": offset, "Symbol": {"Name": s}, - "Type": { - "Name": "X86_64_RELOC_GOT" - | "X86_64_RELOC_GOT_LOAD" as kind - }, + "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, }: offset += base s = s.removeprefix(self.prefix) value, symbol = _stencils.HoleValue.GOT, s addend = ( - int.from_bytes( - raw[offset : offset + 4], "little", signed=True - ) - - 4 + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 ) case { "Offset": offset, @@ -515,19 +483,13 @@ def _handle_relocation( } | { "Offset": offset, "Symbol": {"Name": s}, - "Type": { - "Name": "X86_64_RELOC_BRANCH" - | "X86_64_RELOC_SIGNED" as kind - }, + "Type": {"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind}, }: offset += base s = s.removeprefix(self.prefix) value, symbol = _stencils.symbol_to_value(s) addend = ( - int.from_bytes( - raw[offset : offset + 4], "little", signed=True - ) - - 4 + int.from_bytes(raw[offset : offset + 4], "little", signed=True) - 4 ) case { "Offset": offset, diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 689f6b3f7743df..090b52660f009c 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -40,9 +40,7 @@ def _dump_footer( yield "};" -def _dump_stencil( - opname: str, group: _stencils.StencilGroup -) -> typing.Iterator[str]: +def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]: yield "void" yield f"emit_{opname}(" yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," @@ -64,9 +62,7 @@ def _dump_stencil( yield f" memcpy({part}, {part}_body, sizeof({part}_body));" skip = False stencil.holes.sort(key=lambda hole: hole.offset) - for hole, pair in itertools.zip_longest( - stencil.holes, stencil.holes[1:] - ): + for hole, pair in itertools.zip_longest(stencil.holes, stencil.holes[1:]): if skip: skip = False continue diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 5a67b71e8b8112..4d1e484b6838eb 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -17,22 +17,13 @@ help="a PEP 11 target triple to compile for", ) parser.add_argument( - "-d", - "--debug", - action="store_true", - help="compile for a debug build of Python", + "-d", "--debug", action="store_true", help="compile for a debug build of Python" ) parser.add_argument( - "-f", - "--force", - action="store_true", - help="force the entire JIT to be rebuilt", + "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt" ) parser.add_argument( - "-v", - "--verbose", - action="store_true", - help="echo commands as they are run", + "-v", "--verbose", action="store_true", help="echo commands as they are run" ) args = parser.parse_args() for target in args.target: From ef24b969ad6e155cb21dd7621d6b27dc0596a7f3 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 2 May 2025 19:31:46 +0100 Subject: [PATCH 5/7] Restore pylint comments --- Tools/jit/_llvm.py | 2 +- Tools/jit/_targets.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 903eaa16fbc0c2..cd437db089f371 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -21,7 +21,7 @@ def _async_cache(f: _C[_P, _R]) -> _C[_P, _R]: lock = asyncio.Lock() @functools.wraps(f) - async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: # pylint: disable = no-member async with lock: if args not in cache: cache[args] = await f(*args, **kwargs) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index cbfb0f2120d34f..b41f58afb0649b 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -231,7 +231,7 @@ def build( jit_stencils_new.unlink(missing_ok=True) -class _COFF(_Target[_schema.COFFSection, _schema.COFFRelocation]): +class _COFF(_Target[_schema.COFFSection, _schema.COFFRelocation]): # pylint: disable = too-few-public-methods def _handle_section( self, section: _schema.COFFSection, group: _stencils.StencilGroup ) -> None: @@ -317,7 +317,7 @@ def _handle_relocation( return _stencils.Hole(offset, kind, value, symbol, addend) -class _ELF(_Target[_schema.ELFSection, _schema.ELFRelocation]): +class _ELF(_Target[_schema.ELFSection, _schema.ELFRelocation]): # pylint: disable = too-few-public-methods def _handle_section( self, section: _schema.ELFSection, group: _stencils.StencilGroup ) -> None: @@ -405,7 +405,7 @@ def _handle_relocation( return _stencils.Hole(offset, kind, value, symbol, addend) -class _MachO(_Target[_schema.MachOSection, _schema.MachORelocation]): +class _MachO(_Target[_schema.MachOSection, _schema.MachORelocation]): # pylint: disable = too-few-public-methods def _handle_section( self, section: _schema.MachOSection, group: _stencils.StencilGroup ) -> None: From 40f9b0d2703126947aca46be433e25b04fdded57 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 2 May 2025 20:34:28 +0100 Subject: [PATCH 6/7] Revert non-JIT changes --- .pre-commit-config.yaml | 14 +++++++++----- Tools/build/check_warnings.py | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b180313b0a19f..3c982c388a08d4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.7 + rev: v0.11.8 hooks: - id: ruff name: Run Ruff (lint) on Doc/ @@ -22,15 +22,19 @@ repos: name: Run Ruff (format) on Doc/ args: [--check] files: ^Doc/ - - id: ruff-format - name: Run Ruff (format) on Tools/build/check_warnings.py - args: [--check, --config=Tools/build/.ruff.toml] - files: ^Tools/build/check_warnings.py - id: ruff-format name: Run Ruff (format) on Tools/jit/ args: [--check, --config=Tools/jit/.ruff.toml] files: ^Tools/jit/ + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.1.0 + hooks: + - id: black + name: Run Black on Tools/build/check_warnings.py + files: ^Tools/build/check_warnings.py + args: [--line-length=79] + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 621f431beb9a3f..7a8721087b6c66 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -151,6 +151,7 @@ def get_unexpected_warnings( """ unexpected_warnings = {} for file in files_with_warnings.keys(): + rule = is_file_ignored(file, ignore_rules) if rule: From d7e99f0f4b1dad6025f00d0fb6ee9e625674e014 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 2 May 2025 20:35:42 +0100 Subject: [PATCH 7/7] Add comment to longer line length --- Tools/jit/.ruff.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/jit/.ruff.toml b/Tools/jit/.ruff.toml index 1db9901657e462..6c4b0e77664384 100644 --- a/Tools/jit/.ruff.toml +++ b/Tools/jit/.ruff.toml @@ -1,6 +1,6 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings -line-length = 88 +line-length = 88 # slightly longer than PEP 8, for readability [format] preview = true