From 6b8750c22696e04afcaf2758a6d1023ff603cced Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 8 Aug 2025 10:58:30 +0300 Subject: [PATCH 1/3] gh-133403: Check `generate_stdlib_module_names` and `check_extension_modules` with mypy --- .github/workflows/mypy.yml | 2 + Tools/build/check_extension_modules.py | 77 ++++++++++++--------- Tools/build/generate_stdlib_module_names.py | 23 +++--- Tools/build/mypy.ini | 2 + 4 files changed, 63 insertions(+), 41 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 95133c1338b682..dc636715032969 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -13,9 +13,11 @@ on: - "Lib/test/libregrtest/**" - "Lib/tomllib/**" - "Misc/mypy/**" + - "Tools/build/check_extension_modules.py" - "Tools/build/compute-changes.py" - "Tools/build/deepfreeze.py" - "Tools/build/generate_sbom.py" + - "Tools/build/generate_stdlib_module_names.py" - "Tools/build/generate-build-details.py" - "Tools/build/verify_ensurepip_wheels.py" - "Tools/build/update_file.py" diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 9815bcfe27d995..39ac7df17628df 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -17,6 +17,9 @@ See --help for more information """ + +from __future__ import annotations + import _imp import argparse import collections @@ -29,13 +32,14 @@ import sysconfig import warnings from collections.abc import Iterable -from importlib._bootstrap import _load as bootstrap_load +from importlib._bootstrap import _load as bootstrap_load # type: ignore[attr-defined] from importlib.machinery import ( BuiltinImporter, ExtensionFileLoader, ModuleSpec, ) from importlib.util import spec_from_file_location, spec_from_loader +from typing import NamedTuple SRC_DIR = pathlib.Path(__file__).parent.parent.parent @@ -112,6 +116,7 @@ ) +@enum.unique class ModuleState(enum.Enum): # Makefile state "yes" BUILTIN = "builtin" @@ -123,11 +128,13 @@ class ModuleState(enum.Enum): # disabled by Setup / makesetup rule DISABLED_SETUP = "disabled_setup" - def __bool__(self): + def __bool__(self) -> bool: return self.value in {"builtin", "shared"} -ModuleInfo = collections.namedtuple("ModuleInfo", "name state") +class ModuleInfo(NamedTuple): + name: str + state: ModuleState class ModuleChecker: @@ -135,9 +142,9 @@ class ModuleChecker: setup_files = ( # see end of configure.ac - "Modules/Setup.local", - "Modules/Setup.stdlib", - "Modules/Setup.bootstrap", + pathlib.Path("Modules/Setup.local"), + pathlib.Path("Modules/Setup.stdlib"), + pathlib.Path("Modules/Setup.bootstrap"), SRC_DIR / "Modules/Setup", ) @@ -149,15 +156,15 @@ def __init__(self, cross_compiling: bool = False, strict: bool = False): self.builddir = self.get_builddir() self.modules = self.get_modules() - self.builtin_ok = [] - self.shared_ok = [] - self.failed_on_import = [] - self.missing = [] - self.disabled_configure = [] - self.disabled_setup = [] - self.notavailable = [] + self.builtin_ok: list[ModuleInfo] = [] + self.shared_ok: list[ModuleInfo] = [] + self.failed_on_import: list[ModuleInfo] = [] + self.missing: list[ModuleInfo] = [] + self.disabled_configure: list[ModuleInfo] = [] + self.disabled_setup: list[ModuleInfo] = [] + self.notavailable: list[ModuleInfo] = [] - def check(self): + def check(self) -> None: if not hasattr(_imp, 'create_dynamic'): logger.warning( ('Dynamic extensions not supported ' @@ -189,10 +196,10 @@ def check(self): assert modinfo.state == ModuleState.SHARED self.shared_ok.append(modinfo) - def summary(self, *, verbose: bool = False): + def summary(self, *, verbose: bool = False) -> None: longest = max([len(e.name) for e in self.modules], default=0) - def print_three_column(modinfos: list[ModuleInfo]): + def print_three_column(modinfos: list[ModuleInfo]) -> None: names = [modinfo.name for modinfo in modinfos] names.sort(key=str.lower) # guarantee zip() doesn't drop anything @@ -262,12 +269,12 @@ def print_three_column(modinfos: list[ModuleInfo]): f"{len(self.failed_on_import)} failed on import)" ) - def check_strict_build(self): + def check_strict_build(self) -> None: """Fail if modules are missing and it's a strict build""" if self.strict_extensions_build and (self.failed_on_import or self.missing): raise RuntimeError("Failed to build some stdlib modules") - def list_module_names(self, *, all: bool = False) -> set: + def list_module_names(self, *, all: bool = False) -> set[str]: names = {modinfo.name for modinfo in self.modules} if all: names.update(WINDOWS_MODULES) @@ -280,9 +287,9 @@ def get_builddir(self) -> pathlib.Path: except FileNotFoundError: logger.error("%s must be run from the top build directory", __file__) raise - builddir = pathlib.Path(builddir) - logger.debug("%s: %s", self.pybuilddir_txt, builddir) - return builddir + builddir_path = pathlib.Path(builddir) + logger.debug("%s: %s", self.pybuilddir_txt, builddir_path) + return builddir_path def get_modules(self) -> list[ModuleInfo]: """Get module info from sysconfig and Modules/Setup* files""" @@ -367,7 +374,7 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: case ["*disabled*"]: state = ModuleState.DISABLED case ["*noconfig*"]: - state = None + continue case [*items]: if state == ModuleState.DISABLED: # *disabled* can disable multiple modules per line @@ -384,26 +391,33 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec: """Get ModuleSpec for builtin or extension module""" if modinfo.state == ModuleState.SHARED: - location = os.fspath(self.get_location(modinfo)) + mod_location = self.get_location(modinfo) + assert mod_location is not None + location = os.fspath(mod_location) loader = ExtensionFileLoader(modinfo.name, location) - return spec_from_file_location(modinfo.name, location, loader=loader) + spec = spec_from_file_location(modinfo.name, location, loader=loader) + assert spec is not None + return spec elif modinfo.state == ModuleState.BUILTIN: - return spec_from_loader(modinfo.name, loader=BuiltinImporter) + spec = spec_from_loader(modinfo.name, loader=BuiltinImporter) + assert spec is not None + return spec else: raise ValueError(modinfo) - def get_location(self, modinfo: ModuleInfo) -> pathlib.Path: + def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None: """Get shared library location in build directory""" if modinfo.state == ModuleState.SHARED: return self.builddir / f"{modinfo.name}{self.ext_suffix}" else: return None - def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec): + def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None: """Check that the module file is present and not empty""" - if spec.loader is BuiltinImporter: + if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap] return try: + assert spec.origin is not None st = os.stat(spec.origin) except FileNotFoundError: logger.error("%s (%s) is missing", modinfo.name, spec.origin) @@ -411,7 +425,7 @@ def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec): if not st.st_size: raise ImportError(f"{spec.origin} is an empty file") - def check_module_import(self, modinfo: ModuleInfo): + def check_module_import(self, modinfo: ModuleInfo) -> None: """Attempt to import module and report errors""" spec = self.get_spec(modinfo) self._check_file(modinfo, spec) @@ -430,7 +444,7 @@ def check_module_import(self, modinfo: ModuleInfo): logger.exception("Importing extension '%s' failed!", modinfo.name) raise - def check_module_cross(self, modinfo: ModuleInfo): + def check_module_cross(self, modinfo: ModuleInfo) -> None: """Sanity check for cross compiling""" spec = self.get_spec(modinfo) self._check_file(modinfo, spec) @@ -443,6 +457,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None: failed_name = f"{modinfo.name}_failed{self.ext_suffix}" builddir_path = self.get_location(modinfo) + assert builddir_path is not None if builddir_path.is_symlink(): symlink = builddir_path module_path = builddir_path.resolve().relative_to(os.getcwd()) @@ -466,7 +481,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None: logger.debug("Rename '%s' -> '%s'", module_path, failed_path) -def main(): +def main() -> None: args = parser.parse_args() if args.debug: args.verbose = True diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 88414cdbb37a8d..bda72539640611 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -1,9 +1,12 @@ # This script lists the names of standard library modules # to update Python/stdlib_module_names.h +from __future__ import annotations + import _imp import os.path import sys import sysconfig +from typing import TextIO from check_extension_modules import ModuleChecker @@ -48,12 +51,12 @@ } # Built-in modules -def list_builtin_modules(names): +def list_builtin_modules(names: set[str]) -> None: names |= set(sys.builtin_module_names) # Pure Python modules (Lib/*.py) -def list_python_modules(names): +def list_python_modules(names: set[str]) -> None: for filename in os.listdir(STDLIB_PATH): if not filename.endswith(".py"): continue @@ -62,7 +65,7 @@ def list_python_modules(names): # Packages in Lib/ -def list_packages(names): +def list_packages(names: set[str]) -> None: for name in os.listdir(STDLIB_PATH): if name in IGNORE: continue @@ -76,16 +79,16 @@ def list_packages(names): # Built-in and extension modules built by Modules/Setup* # includes Windows and macOS extensions. -def list_modules_setup_extensions(names): +def list_modules_setup_extensions(names: set[str]) -> None: checker = ModuleChecker() names.update(checker.list_module_names(all=True)) # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c). # Use the "./Programs/_testembed list_frozen" command. -def list_frozen(names): +def list_frozen(names: set[str]) -> None: submodules = set() - for name in _imp._frozen_module_names(): + for name in _imp._frozen_module_names(): # type: ignore[attr-defined] # To skip __hello__, __hello_alias__ and etc. if name.startswith('__'): continue @@ -101,8 +104,8 @@ def list_frozen(names): raise Exception(f'unexpected frozen submodules: {sorted(submodules)}') -def list_modules(): - names = set() +def list_modules() -> set[str]: + names: set[str] = set() list_builtin_modules(names) list_modules_setup_extensions(names) @@ -127,7 +130,7 @@ def list_modules(): return names -def write_modules(fp, names): +def write_modules(fp: TextIO, names: set[str]) -> None: print(f"// Auto-generated by {SCRIPT_NAME}.", file=fp) print("// List used to create sys.stdlib_module_names.", file=fp) @@ -138,7 +141,7 @@ def write_modules(fp, names): print("};", file=fp) -def main(): +def main() -> None: if not sysconfig.is_python_build(): print(f"ERROR: {sys.executable} is not a Python build", file=sys.stderr) diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 123dc895f90a1f..2abb21e51f1cf8 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -3,10 +3,12 @@ # Please, when adding new files here, also add them to: # .github/workflows/mypy.yml files = + Tools/build/check_extension_modules.py, Tools/build/compute-changes.py, Tools/build/deepfreeze.py, Tools/build/generate-build-details.py, Tools/build/generate_sbom.py, + Tools/build/generate_stdlib_module_names.py, Tools/build/verify_ensurepip_wheels.py, Tools/build/update_file.py, Tools/build/umarshal.py From 2f0d11b5e78f311ebc80c4779fb9f01287781309 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 8 Aug 2025 11:05:54 +0300 Subject: [PATCH 2/3] Fix lint --- Tools/build/check_extension_modules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 39ac7df17628df..5b175bc4ea5862 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -22,7 +22,6 @@ import _imp import argparse -import collections import enum import logging import os @@ -32,7 +31,9 @@ import sysconfig import warnings from collections.abc import Iterable -from importlib._bootstrap import _load as bootstrap_load # type: ignore[attr-defined] +from importlib._bootstrap import ( + _load as bootstrap_load, # type: ignore[attr-defined] +) from importlib.machinery import ( BuiltinImporter, ExtensionFileLoader, From 1bae534d655276f56cd198afeea7e17fb9a8bb89 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 8 Aug 2025 11:12:53 +0300 Subject: [PATCH 3/3] Fix lint --- Tools/build/check_extension_modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 5b175bc4ea5862..668db8df0bd181 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -31,8 +31,8 @@ import sysconfig import warnings from collections.abc import Iterable -from importlib._bootstrap import ( - _load as bootstrap_load, # type: ignore[attr-defined] +from importlib._bootstrap import ( # type: ignore[attr-defined] + _load as bootstrap_load, ) from importlib.machinery import ( BuiltinImporter,