Skip to content

gh-100238: Vendor distutils for peg_generator #104744

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

gh-100238: Vendor distutils for peg_generator #104744

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4470301
gh-92584: Vendor distutils for peg_generator
lysnikolaou May 18, 2023
b2233e9
📜🤖 Added by blurb_it.
blurb-it[bot] May 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 1 addition & 33 deletions Lib/test/support/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
# miscellaneous
"run_with_locale", "swap_item", "findfile", "infinite_recursion",
"swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict",
"run_with_tz", "PGO", "missing_compiler_executable",
"run_with_tz", "PGO",
"ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST",
"LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT",
"Py_DEBUG", "EXCEEDS_RECURSION_LIMIT",
Expand Down Expand Up @@ -1856,38 +1856,6 @@ def __del__(self):
test.assertTrue(done)


def missing_compiler_executable(cmd_names=[]):
"""Check if the compiler components used to build the interpreter exist.

Check for the existence of the compiler executables whose names are listed
in 'cmd_names' or all the compiler executables when 'cmd_names' is empty
and return the first missing executable or None when none is found
missing.

"""
# TODO (PEP 632): alternate check without using distutils
from distutils import ccompiler, sysconfig, spawn, errors
compiler = ccompiler.new_compiler()
sysconfig.customize_compiler(compiler)
if compiler.compiler_type == "msvc":
# MSVC has no executables, so check whether initialization succeeds
try:
compiler.initialize()
except errors.DistutilsPlatformError:
return "msvc"
for name in compiler.executables:
if cmd_names and name not in cmd_names:
continue
cmd = getattr(compiler, name)
if cmd_names:
assert cmd is not None, \
"the '%s' executable is not configured" % name
elif not cmd:
continue
if spawn.find_executable(cmd[0]) is None:
return cmd[0]


_is_android_emulator = None
def setswitchinterval(interval):
# Setting a very low gil interval on the Android emulator causes python
Expand Down
4 changes: 0 additions & 4 deletions Lib/test/test_peg_generator/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,6 @@
from test import support
from test.support import load_package_tests

# TODO: gh-92584: peg_generator uses distutils which was removed in Python 3.12
raise unittest.SkipTest("distutils has been removed in Python 3.12")


if support.check_sanitizer(address=True, memory=True):
# bpo-46633: Skip the test because it is too slow when Python is built
# with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions.
Expand Down
36 changes: 35 additions & 1 deletion Lib/test/test_peg_generator/test_c_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,40 @@ def test_parse(self):
"""


def _missing_compiler_executable(cmd_names=[]):
"""Check if the compiler components used to build the interpreter exist.

Check for the existence of the compiler executables whose names are listed
in 'cmd_names' or all the compiler executables when 'cmd_names' is empty
and return the first missing executable or None when none is found
missing.

"""
test_tools.skip_if_missing("distutils")
with test_tools.imports_under_tool(""):
from distutils import ccompiler, sysconfig, spawn, errors

compiler = ccompiler.new_compiler()
sysconfig.customize_compiler(compiler)
if compiler.compiler_type == "msvc":
# MSVC has no executables, so check whether initialization succeeds
try:
compiler.initialize()
except errors.DistutilsPlatformError:
return "msvc"
for name in compiler.executables:
if cmd_names and name not in cmd_names:
continue
cmd = getattr(compiler, name)
if cmd_names:
assert cmd is not None, \
"the '%s' executable is not configured" % name
elif not cmd:
continue
if spawn.find_executable(cmd[0]) is None:
return cmd[0]


@support.requires_subprocess()
class TestCParser(unittest.TestCase):

Expand All @@ -90,7 +124,7 @@ def setUpClass(cls):

def setUp(self):
self._backup_config_vars = dict(sysconfig._CONFIG_VARS)
cmd = support.missing_compiler_executable()
cmd = _missing_compiler_executable()
if cmd is not None:
self.skipTest("The %r command is not found" % cmd)
self.old_cwd = os.getcwd()
Expand Down
36 changes: 18 additions & 18 deletions Lib/test/test_peg_generator/test_pegen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,34 +552,34 @@ def test_mutually_left_recursive(self) -> None:
string="D",
start=(1, 0),
end=(1, 1),
line="D A C A E",
line="D A C A E\n",
),
TokenInfo(
type=NAME,
string="A",
start=(1, 2),
end=(1, 3),
line="D A C A E",
line="D A C A E\n",
),
],
TokenInfo(
type=NAME,
string="C",
start=(1, 4),
end=(1, 5),
line="D A C A E",
line="D A C A E\n",
),
],
TokenInfo(
type=NAME,
string="A",
start=(1, 6),
end=(1, 7),
line="D A C A E",
line="D A C A E\n",
),
],
TokenInfo(
type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E"
type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E\n"
),
],
)
Expand All @@ -594,22 +594,22 @@ def test_mutually_left_recursive(self) -> None:
string="B",
start=(1, 0),
end=(1, 1),
line="B C A E",
line="B C A E\n",
),
TokenInfo(
type=NAME,
string="C",
start=(1, 2),
end=(1, 3),
line="B C A E",
line="B C A E\n",
),
],
TokenInfo(
type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E"
type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E\n"
),
],
TokenInfo(
type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E"
type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E\n"
),
],
)
Expand Down Expand Up @@ -654,18 +654,18 @@ def test_lookahead(self) -> None:
node,
[
TokenInfo(
NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 ."
NAME, string="foo", start=(1, 0), end=(1, 3), line="foo = 12 + 12 .\n"
),
TokenInfo(
OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 ."
OP, string="=", start=(1, 4), end=(1, 5), line="foo = 12 + 12 .\n"
),
[
TokenInfo(
NUMBER,
string="12",
start=(1, 6),
end=(1, 8),
line="foo = 12 + 12 .",
line="foo = 12 + 12 .\n",
),
[
[
Expand All @@ -674,14 +674,14 @@ def test_lookahead(self) -> None:
string="+",
start=(1, 9),
end=(1, 10),
line="foo = 12 + 12 .",
line="foo = 12 + 12 .\n",
),
TokenInfo(
NUMBER,
string="12",
start=(1, 11),
end=(1, 13),
line="foo = 12 + 12 .",
line="foo = 12 + 12 .\n",
),
]
],
Expand Down Expand Up @@ -733,9 +733,9 @@ def test_cut(self) -> None:
self.assertEqual(
node,
[
TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)"),
TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)"),
TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)"),
TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)\n"),
TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)\n"),
TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)\n"),
],
)

Expand Down Expand Up @@ -794,7 +794,7 @@ def test_soft_keyword(self) -> None:
start:
| "number" n=NUMBER { eval(n.string) }
| "string" n=STRING { n.string }
| SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { f"{l.string} = {n.string}"}
| SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { l.string + " = " + n.string}
"""
parser_class = make_parser(grammar)
self.assertEqual(parse_string("number 1", parser_class), 1)
Expand Down
1 change: 1 addition & 0 deletions Misc/NEWS.d/next/Tools-Demos/2023-05-22-14-28-06.gh-issue-100238.7J7P3p.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Vendor all of distutils (except for tests) in Tools for use in the c-analyzer and the peg-generator.
8 changes: 7 additions & 1 deletion Tools/c-analyzer/c_parser/preprocessor/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import contextlib
import distutils.ccompiler
import logging
import shlex
import subprocess
Expand All @@ -16,6 +15,13 @@

logger = logging.getLogger(__name__)

try:
import distutils.ccompiler
except ImportError:
import pathlib
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent.parent.parent))
import distutils.ccompiler
sys.path.pop(0)

# XXX Add aggregate "source" class(es)?
# * expose all lines as single text string
Expand Down
2 changes: 0 additions & 2 deletions Tools/c-analyzer/distutils/README
View file
Open in desktop

This file was deleted.

Loading