Skip to content

Commit c5acc57

Browse files
borisyouknowone
boris
and
youknowone
committed
Convert whats_left into a CPython script
Co-Authored-By: Jeong YunWon <jeong@youknowone.org>
1 parent 9e9d610 commit c5acc57

File tree

2 files changed

+117
-92
lines changed

2 files changed

+117
-92
lines changed

‎extra_tests/not_impl_gen.py renamed to ‎whats_left.py

Lines changed: 117 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,56 @@
1-
# It's recommended to run this with `python3 -I not_impl_gen.py`, to make sure
2-
# that nothing in your global Python environment interferes with what's being
3-
# extracted here.
4-
#
1+
#!/usr/bin/env python3
2+
53
# This script generates Lib/snippets/whats_left_data.py with these variables defined:
64
# expected_methods - a dictionary mapping builtin objects to their methods
75
# cpymods - a dictionary mapping module names to their contents
86
# libdir - the location of RustPython's Lib/ directory.
97

10-
import inspect
11-
import io
8+
#
9+
# TODO: include this:
10+
# which finds all modules it has available and
11+
# creates a Python dictionary mapping module names to their contents, which is
12+
# in turn used to generate a second Python script that also finds which modules
13+
# it has available and compares that against the first dictionary we generated.
14+
# We then run this second generated script with RustPython.
15+
16+
import argparse
17+
import re
1218
import os
1319
import re
1420
import sys
21+
import json
1522
import warnings
23+
import inspect
24+
import subprocess
25+
import platform
1626
from pydoc import ModuleScanner
1727

28+
GENERATED_FILE = "extra_tests/snippets/not_impl.py"
29+
30+
implementation = platform.python_implementation()
31+
if implementation != "CPython":
32+
sys.exit("whats_left.py must be run under CPython, got {implementation} instead")
33+
34+
35+
def parse_args():
36+
parser = argparse.ArgumentParser(description="Process some integers.")
37+
parser.add_argument(
38+
"--signature",
39+
action="store_true",
40+
help="print functions whose signatures don't match CPython's",
41+
)
42+
parser.add_argument(
43+
"--json",
44+
action="store_true",
45+
help="print output as JSON (instead of line by line)",
46+
)
47+
48+
args = parser.parse_args()
49+
return args
50+
51+
52+
args = parse_args()
53+
1854

1955
# modules suggested for deprecation by PEP 594 (www.python.org/dev/peps/pep-0594/)
2056
# some of these might be implemented, but they are not a priority
@@ -57,9 +93,7 @@
5793
IGNORED_MODULES = {'this', 'antigravity'} | PEP_594_MODULES | CPYTHON_SPECIFIC_MODS
5894

5995
sys.path = [
60-
path
61-
for path in sys.path
62-
if ("site-packages" not in path and "dist-packages" not in path)
96+
path for path in sys.path if ("site-packages" not in path and "dist-packages" not in path)
6397
]
6498

6599

@@ -250,7 +284,7 @@ def gen_modules():
250284
output += gen_methods()
251285
output += f"""
252286
cpymods = {gen_modules()!r}
253-
libdir = {os.path.abspath("../Lib/").encode('utf8')!r}
287+
libdir = {os.path.abspath("Lib/").encode('utf8')!r}
254288
255289
"""
256290

@@ -279,6 +313,8 @@ def compare():
279313
import warnings
280314
from contextlib import redirect_stdout
281315

316+
import json
317+
import inspect
282318
import platform
283319

284320
def method_incompatability_reason(typ, method_name, real_method_value):
@@ -288,7 +324,7 @@ def method_incompatability_reason(typ, method_name, real_method_value):
288324

289325
is_inherited = not attr_is_not_inherited(typ, method_name)
290326
if is_inherited:
291-
return "inherited"
327+
return "(inherited)"
292328

293329
value = extra_info(getattr(typ, method_name))
294330
if value != real_method_value:
@@ -321,70 +357,41 @@ def method_incompatability_reason(typ, method_name, real_method_value):
321357

322358
rustpymods = {mod: dir_of_mod_or_error(mod) for mod in mod_names}
323359

324-
not_implemented = {}
325-
failed_to_import = {}
326-
missing_items = {}
327-
mismatched_items = {}
360+
result = {
361+
"cpython_modules": {},
362+
"implemented": {},
363+
"not_implemented": {},
364+
"failed_to_import": {},
365+
"missing_items": {},
366+
"mismatched_items": {},
367+
}
328368
for modname, cpymod in cpymods.items():
329369
rustpymod = rustpymods.get(modname)
330370
if rustpymod is None:
331-
not_implemented[modname] = None
371+
result["not_implemented"][modname] = None
332372
elif isinstance(rustpymod, Exception):
333-
failed_to_import[modname] = rustpymod
373+
result["failed_to_import"][modname] = rustpymod.__class__.__name__ + str(rustpymod)
334374
else:
335375
implemented_items = sorted(set(cpymod) & set(rustpymod))
336376
mod_missing_items = set(cpymod) - set(rustpymod)
337-
mod_missing_items = sorted(
338-
f"{modname}.{item}" for item in mod_missing_items
339-
)
377+
mod_missing_items = sorted(f"{modname}.{item}" for item in mod_missing_items)
340378
mod_mismatched_items = [
341379
(f"{modname}.{item}", rustpymod[item], cpymod[item])
342380
for item in implemented_items
343-
if rustpymod[item] != cpymod[item]
344-
and not isinstance(cpymod[item], Exception)
381+
if rustpymod[item] != cpymod[item] and not isinstance(cpymod[item], Exception)
345382
]
346-
if mod_missing_items:
347-
missing_items[modname] = mod_missing_items
348-
if mod_mismatched_items:
349-
mismatched_items[modname] = mod_mismatched_items
350-
351-
# missing entire module
352-
print("# modules")
353-
for modname in not_implemented:
354-
print(modname, "(entire module)")
355-
for modname, exception in failed_to_import.items():
356-
print(f"{modname} (exists but not importable: {exception})")
357-
358-
# missing from builtins
359-
print("\n# builtin items")
360-
for module, missing_methods in not_implementeds.items():
361-
for method, reason in missing_methods.items():
362-
print(f"{module}.{method}" + (f" ({reason})" if reason else ""))
363-
364-
# missing from modules
365-
print("\n# stdlib items")
366-
for modname, missing in missing_items.items():
367-
for item in missing:
368-
print(item)
383+
if mod_missing_items or mod_mismatched_items:
384+
if mod_missing_items:
385+
result["missing_items"][modname] = mod_missing_items
386+
if mod_mismatched_items:
387+
result["mismatched_items"][modname] = mod_mismatched_items
388+
else:
389+
result["implemented"][modname] = None
369390

370-
print("\n# mismatching signatures (warnings)")
371-
for modname, mismatched in mismatched_items.items():
372-
for (item, rustpy_value, cpython_value) in mismatched:
373-
if cpython_value == "ValueError('no signature found')":
374-
continue # these items will never match
375-
print(f"{item} {rustpy_value} != {cpython_value}")
376-
377-
result = {
378-
"not_implemented": not_implemented,
379-
"failed_to_import": failed_to_import,
380-
"missing_items": missing_items,
381-
"mismatched_items": mismatched_items,
382-
}
391+
result["cpython_modules"] = cpymods
392+
result["not_implementeds"] = not_implementeds
383393

384-
print()
385-
print("# out of", len(cpymods), "modules:")
386-
for error_type, modules in result.items():
387-
print("# ", error_type, len(modules))
394+
print(json.dumps(result))
388395

389396

390397
def remove_one_indent(s):
@@ -395,5 +402,52 @@ def remove_one_indent(s):
395402
compare_src = inspect.getsourcelines(compare)[0][1:]
396403
output += "".join(remove_one_indent(line) for line in compare_src)
397404

398-
with open("not_impl.py", "w") as f:
405+
with open(GENERATED_FILE, "w") as f:
399406
f.write(output + "\n")
407+
408+
409+
subprocess.run(["cargo", "build", "--release", "--features=ssl"], check=True)
410+
result = subprocess.run(
411+
["cargo", "run", "--release", "--features=ssl", "-q", "--", GENERATED_FILE],
412+
env={**os.environ.copy(), "RUSTPYTHONPATH": "Lib"},
413+
text=True,
414+
capture_output=True,
415+
)
416+
# The last line should be json output, the rest of the lines can contain noise
417+
# because importing certain modules can print stuff to stdout/stderr
418+
result = json.loads(result.stdout.splitlines()[-1])
419+
420+
if args.json:
421+
print(json.dumps(result))
422+
sys.exit()
423+
424+
425+
# missing entire modules
426+
print('# modules')
427+
for modname in result["not_implemented"]:
428+
print(modname, "(entire module)")
429+
for modname, exception in result["failed_to_import"].items():
430+
print(f"{modname} (exists but not importable: {exception})")
431+
432+
# missing from builtins
433+
print("\n# builtin items")
434+
for module, missing_methods in result["not_implementeds"].items():
435+
for method, reason in missing_methods.items():
436+
print(f"{module}.{method}" + (f" {reason}" if reason else ""))
437+
438+
# missing from modules
439+
print("\n# stdlib items")
440+
for modname, missing in result["missing_items"].items():
441+
for item in missing:
442+
print(item)
443+
444+
if args.signature:
445+
print("\n# mismatching signatures (warnings)")
446+
for modname, mismatched in result["mismatched_items"].items():
447+
for (item, rustpy_value, cpython_value) in mismatched:
448+
print(f"{item} {rustpy_value} != {cpython_value}")
449+
450+
print()
451+
print("# summary")
452+
for error_type, modules in result.items():
453+
print("# ", error_type, len(modules))

‎whats_left.sh

Lines changed: 0 additions & 29 deletions
This file was deleted.

0 commit comments

Comments
 (0)