Skip to content

[3.14] gh-133403: Check generate_stdlib_module_names and check_extension_modules with mypy (GH-137546) #137679

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 13, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/mypy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,11 @@ on:
- "Lib/test/libregrtest/**"
- "Lib/tomllib/**"
- "Misc/mypy/**"
- "Tools/build/check_extension_modules.py"
- "Tools/build/compute-changes.py"
- "Tools/build/deepfreeze.py"
- "Tools/build/generate_sbom.py"
- "Tools/build/generate_stdlib_module_names.py"
- "Tools/build/generate-build-details.py"
- "Tools/build/verify_ensurepip_wheels.py"
- "Tools/build/update_file.py"
Expand Down
80 changes: 48 additions & 32 deletions Tools/build/check_extension_modules.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@

See --help for more information
"""

from __future__ import annotations

import _imp
import argparse
import collections
import enum
import logging
import os
Expand All @@ -29,13 +31,16 @@
import sysconfig
import warnings
from collections.abc import Iterable
from importlib._bootstrap import _load as bootstrap_load
from importlib._bootstrap import ( # type: ignore[attr-defined]
_load as bootstrap_load,
)
from importlib.machinery import (
BuiltinImporter,
ExtensionFileLoader,
ModuleSpec,
)
from importlib.util import spec_from_file_location, spec_from_loader
from typing import NamedTuple

SRC_DIR = pathlib.Path(__file__).parent.parent.parent

Expand Down Expand Up @@ -112,6 +117,7 @@
)


@enum.unique
class ModuleState(enum.Enum):
# Makefile state "yes"
BUILTIN = "builtin"
Expand All @@ -123,21 +129,23 @@ class ModuleState(enum.Enum):
# disabled by Setup / makesetup rule
DISABLED_SETUP = "disabled_setup"

def __bool__(self):
def __bool__(self) -> bool:
return self.value in {"builtin", "shared"}


ModuleInfo = collections.namedtuple("ModuleInfo", "name state")
class ModuleInfo(NamedTuple):
name: str
state: ModuleState


class ModuleChecker:
pybuilddir_txt = "pybuilddir.txt"

setup_files = (
# see end of configure.ac
"Modules/Setup.local",
"Modules/Setup.stdlib",
"Modules/Setup.bootstrap",
pathlib.Path("Modules/Setup.local"),
pathlib.Path("Modules/Setup.stdlib"),
pathlib.Path("Modules/Setup.bootstrap"),
SRC_DIR / "Modules/Setup",
)

Expand All @@ -149,15 +157,15 @@ def __init__(self, cross_compiling: bool = False, strict: bool = False):
self.builddir = self.get_builddir()
self.modules = self.get_modules()

self.builtin_ok = []
self.shared_ok = []
self.failed_on_import = []
self.missing = []
self.disabled_configure = []
self.disabled_setup = []
self.notavailable = []
self.builtin_ok: list[ModuleInfo] = []
self.shared_ok: list[ModuleInfo] = []
self.failed_on_import: list[ModuleInfo] = []
self.missing: list[ModuleInfo] = []
self.disabled_configure: list[ModuleInfo] = []
self.disabled_setup: list[ModuleInfo] = []
self.notavailable: list[ModuleInfo] = []

def check(self):
def check(self) -> None:
if not hasattr(_imp, 'create_dynamic'):
logger.warning(
('Dynamic extensions not supported '
Expand Down Expand Up @@ -189,10 +197,10 @@ def check(self):
assert modinfo.state == ModuleState.SHARED
self.shared_ok.append(modinfo)

def summary(self, *, verbose: bool = False):
def summary(self, *, verbose: bool = False) -> None:
longest = max([len(e.name) for e in self.modules], default=0)

def print_three_column(modinfos: list[ModuleInfo]):
def print_three_column(modinfos: list[ModuleInfo]) -> None:
names = [modinfo.name for modinfo in modinfos]
names.sort(key=str.lower)
# guarantee zip() doesn't drop anything
Expand Down Expand Up @@ -262,12 +270,12 @@ def print_three_column(modinfos: list[ModuleInfo]):
f"{len(self.failed_on_import)} failed on import)"
)

def check_strict_build(self):
def check_strict_build(self) -> None:
"""Fail if modules are missing and it's a strict build"""
if self.strict_extensions_build and (self.failed_on_import or self.missing):
raise RuntimeError("Failed to build some stdlib modules")

def list_module_names(self, *, all: bool = False) -> set:
def list_module_names(self, *, all: bool = False) -> set[str]:
names = {modinfo.name for modinfo in self.modules}
if all:
names.update(WINDOWS_MODULES)
Expand All @@ -280,9 +288,9 @@ def get_builddir(self) -> pathlib.Path:
except FileNotFoundError:
logger.error("%s must be run from the top build directory", __file__)
raise
builddir = pathlib.Path(builddir)
logger.debug("%s: %s", self.pybuilddir_txt, builddir)
return builddir
builddir_path = pathlib.Path(builddir)
logger.debug("%s: %s", self.pybuilddir_txt, builddir_path)
return builddir_path

def get_modules(self) -> list[ModuleInfo]:
"""Get module info from sysconfig and Modules/Setup* files"""
Expand Down Expand Up @@ -367,7 +375,7 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]:
case ["*disabled*"]:
state = ModuleState.DISABLED
case ["*noconfig*"]:
state = None
continue
case [*items]:
if state == ModuleState.DISABLED:
# *disabled* can disable multiple modules per line
Expand All @@ -384,34 +392,41 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]:
def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec:
"""Get ModuleSpec for builtin or extension module"""
if modinfo.state == ModuleState.SHARED:
location = os.fspath(self.get_location(modinfo))
mod_location = self.get_location(modinfo)
assert mod_location is not None
location = os.fspath(mod_location)
loader = ExtensionFileLoader(modinfo.name, location)
return spec_from_file_location(modinfo.name, location, loader=loader)
spec = spec_from_file_location(modinfo.name, location, loader=loader)
assert spec is not None
return spec
elif modinfo.state == ModuleState.BUILTIN:
return spec_from_loader(modinfo.name, loader=BuiltinImporter)
spec = spec_from_loader(modinfo.name, loader=BuiltinImporter)
assert spec is not None
return spec
else:
raise ValueError(modinfo)

def get_location(self, modinfo: ModuleInfo) -> pathlib.Path:
def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None:
"""Get shared library location in build directory"""
if modinfo.state == ModuleState.SHARED:
return self.builddir / f"{modinfo.name}{self.ext_suffix}"
else:
return None

def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec):
def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None:
"""Check that the module file is present and not empty"""
if spec.loader is BuiltinImporter:
if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap]
return
try:
assert spec.origin is not None
st = os.stat(spec.origin)
except FileNotFoundError:
logger.error("%s (%s) is missing", modinfo.name, spec.origin)
raise
if not st.st_size:
raise ImportError(f"{spec.origin} is an empty file")

def check_module_import(self, modinfo: ModuleInfo):
def check_module_import(self, modinfo: ModuleInfo) -> None:
"""Attempt to import module and report errors"""
spec = self.get_spec(modinfo)
self._check_file(modinfo, spec)
Expand All @@ -430,7 +445,7 @@ def check_module_import(self, modinfo: ModuleInfo):
logger.exception("Importing extension '%s' failed!", modinfo.name)
raise

def check_module_cross(self, modinfo: ModuleInfo):
def check_module_cross(self, modinfo: ModuleInfo) -> None:
"""Sanity check for cross compiling"""
spec = self.get_spec(modinfo)
self._check_file(modinfo, spec)
Expand All @@ -443,6 +458,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None:

failed_name = f"{modinfo.name}_failed{self.ext_suffix}"
builddir_path = self.get_location(modinfo)
assert builddir_path is not None
if builddir_path.is_symlink():
symlink = builddir_path
module_path = builddir_path.resolve().relative_to(os.getcwd())
Expand All @@ -466,7 +482,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None:
logger.debug("Rename '%s' -> '%s'", module_path, failed_path)


def main():
def main() -> None:
args = parser.parse_args()
if args.debug:
args.verbose = True
Expand Down
23 changes: 13 additions & 10 deletions Tools/build/generate_stdlib_module_names.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
# This script lists the names of standard library modules
# to update Python/stdlib_module_names.h
from __future__ import annotations

import _imp
import os.path
import sys
import sysconfig
from typing import TextIO

from check_extension_modules import ModuleChecker

Expand Down Expand Up @@ -48,12 +51,12 @@
}

# Built-in modules
def list_builtin_modules(names):
def list_builtin_modules(names: set[str]) -> None:
names |= set(sys.builtin_module_names)


# Pure Python modules (Lib/*.py)
def list_python_modules(names):
def list_python_modules(names: set[str]) -> None:
for filename in os.listdir(STDLIB_PATH):
if not filename.endswith(".py"):
continue
Expand All @@ -62,7 +65,7 @@ def list_python_modules(names):


# Packages in Lib/
def list_packages(names):
def list_packages(names: set[str]) -> None:
for name in os.listdir(STDLIB_PATH):
if name in IGNORE:
continue
Expand All @@ -76,16 +79,16 @@ def list_packages(names):

# Built-in and extension modules built by Modules/Setup*
# includes Windows and macOS extensions.
def list_modules_setup_extensions(names):
def list_modules_setup_extensions(names: set[str]) -> None:
checker = ModuleChecker()
names.update(checker.list_module_names(all=True))


# List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
# Use the "./Programs/_testembed list_frozen" command.
def list_frozen(names):
def list_frozen(names: set[str]) -> None:
submodules = set()
for name in _imp._frozen_module_names():
for name in _imp._frozen_module_names(): # type: ignore[attr-defined]
# To skip __hello__, __hello_alias__ and etc.
if name.startswith('__'):
continue
Expand All @@ -101,8 +104,8 @@ def list_frozen(names):
raise Exception(f'unexpected frozen submodules: {sorted(submodules)}')


def list_modules():
names = set()
def list_modules() -> set[str]:
names: set[str] = set()

list_builtin_modules(names)
list_modules_setup_extensions(names)
Expand All @@ -127,7 +130,7 @@ def list_modules():
return names


def write_modules(fp, names):
def write_modules(fp: TextIO, names: set[str]) -> None:
print(f"// Auto-generated by {SCRIPT_NAME}.",
file=fp)
print("// List used to create sys.stdlib_module_names.", file=fp)
Expand All @@ -138,7 +141,7 @@ def write_modules(fp, names):
print("};", file=fp)


def main():
def main() -> None:
if not sysconfig.is_python_build():
print(f"ERROR: {sys.executable} is not a Python build",
file=sys.stderr)
Expand Down
2 changes: 2 additions & 0 deletions Tools/build/mypy.ini
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
# Please, when adding new files here, also add them to:
# .github/workflows/mypy.yml
files =
Tools/build/check_extension_modules.py,
Tools/build/compute-changes.py,
Tools/build/deepfreeze.py,
Tools/build/generate-build-details.py,
Tools/build/generate_sbom.py,
Tools/build/generate_stdlib_module_names.py,
Tools/build/verify_ensurepip_wheels.py,
Tools/build/update_file.py,
Tools/build/umarshal.py
Expand Down
Loading