RustPython · youknowone · Jul 23, 2022 · Feb 11, 2021 · Jul 23, 2022 · Jul 23, 2022
diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml
@@ -83,8 +83,8 @@ jobs:
         run: cargo build --release --verbose
       - name: Collect what is left data
         run: |
-          chmod +x ./whats_left.sh
-          ./whats_left.sh > whats_left.temp
+          chmod +x ./whats_left.py
+          ./whats_left.py > whats_left.temp
         env:
           RUSTPYTHONPATH: ${{ github.workspace }}/Lib
       - name: Upload data to the website

diff --git a/README.md b/README.md
@@ -198,7 +198,7 @@ To enhance CPython compatibility, try to increase unittest coverage by checking
 Another approach is to checkout the source code: builtin functions and object
 methods are often the simplest and easiest way to contribute.
 
-You can also simply run `./whats_left.sh` to assist in finding any unimplemented
+You can also simply run `./whats_left.py` to assist in finding any unimplemented
 method.
 
 ## Compiling to WebAssembly

diff --git a/extra_tests/not_impl_gen.py → whats_left.py b/extra_tests/not_impl_gen.py → whats_left.py
@@ -1,20 +1,61 @@
-# It's recommended to run this with `python3 -I not_impl_gen.py`, to make sure
-# that nothing in your global Python environment interferes with what's being
-# extracted here.
-#
+#!/usr/bin/env python3 -I
+
 # This script generates Lib/snippets/whats_left_data.py with these variables defined:
 # expected_methods - a dictionary mapping builtin objects to their methods
 # cpymods - a dictionary mapping module names to their contents
 # libdir - the location of RustPython's Lib/ directory.
 
-import inspect
-import io
+#
+# TODO: include this:
+# which finds all modules it has available and
+# creates a Python dictionary mapping module names to their contents, which is
+# in turn used to generate a second Python script that also finds which modules
+# it has available and compares that against the first dictionary we generated.
+# We then run this second generated script with RustPython.
+
+import argparse
+import re
 import os
 import re
 import sys
+import json
 import warnings
+import inspect
+import subprocess
+import platform
 from pydoc import ModuleScanner
 
+if not sys.flags.isolated:
+    print("running without -I option.")
+    print("python -I whats_left.py")
+    exit(1)
+
+GENERATED_FILE = "extra_tests/snippets/not_impl.py"
+
+implementation = platform.python_implementation()
+if implementation != "CPython":
+    sys.exit("whats_left.py must be run under CPython, got {implementation} instead")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Process some integers.")
+    parser.add_argument(
+        "--signature",
+        action="store_true",
+        help="print functions whose signatures don't match CPython's",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="print output as JSON (instead of line by line)",
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+args = parse_args()
+
 
 # modules suggested for deprecation by PEP 594 (www.python.org/dev/peps/pep-0594/)
 # some of these might be implemented, but they are not a priority
@@ -54,7 +95,7 @@
     '_testbuffer', '_testcapi', '_testimportmultiple', '_testinternalcapi', '_testmultiphase',
 }
 
-IGNORED_MODULES = {'this', 'antigravity'} | PEP_594_MODULES | CPYTHON_SPECIFIC_MODS
+IGNORED_MODULES = {"this", "antigravity"} | PEP_594_MODULES | CPYTHON_SPECIFIC_MODS
 
 sys.path = [
     path
@@ -188,6 +229,7 @@ def onerror(modname):
 def import_module(module_name):
     import io
     from contextlib import redirect_stdout
+
     # Importing modules causes ('Constant String', 2, None, 4) and
     # "Hello world!" to be printed to stdout.
     f = io.StringIO()
@@ -203,6 +245,7 @@ def import_module(module_name):
 
 def is_child(module, item):
     import inspect
+
     item_mod = inspect.getmodule(item)
     return item_mod is module
 
@@ -250,7 +293,7 @@ def gen_modules():
 output += gen_methods()
 output += f"""
 cpymods = {gen_modules()!r}
-libdir = {os.path.abspath("../Lib/").encode('utf8')!r}
+libdir = {os.path.abspath("Lib/").encode('utf8')!r}
 
 """
 
@@ -260,7 +303,7 @@ def gen_modules():
     extra_info,
     dir_of_mod_or_error,
     import_module,
-    is_child
+    is_child,
 ]
 for fn in REUSED:
     output += "".join(inspect.getsourcelines(fn)[0]) + "\n\n"
@@ -278,7 +321,7 @@ def compare():
     import sys
     import warnings
     from contextlib import redirect_stdout
-
+    import json
     import platform
 
     def method_incompatability_reason(typ, method_name, real_method_value):
@@ -288,7 +331,7 @@ def method_incompatability_reason(typ, method_name, real_method_value):
 
         is_inherited = not attr_is_not_inherited(typ, method_name)
         if is_inherited:
-            return "inherited"
+            return "(inherited)"
 
         value = extra_info(getattr(typ, method_name))
         if value != real_method_value:
@@ -321,16 +364,20 @@ def method_incompatability_reason(typ, method_name, real_method_value):
 
     rustpymods = {mod: dir_of_mod_or_error(mod) for mod in mod_names}
 
-    not_implemented = {}
-    failed_to_import = {}
-    missing_items = {}
-    mismatched_items = {}
+    result = {
+        "cpython_modules": {},
+        "implemented": {},
+        "not_implemented": {},
+        "failed_to_import": {},
+        "missing_items": {},
+        "mismatched_items": {},
+    }
     for modname, cpymod in cpymods.items():
         rustpymod = rustpymods.get(modname)
         if rustpymod is None:
-            not_implemented[modname] = None
+            result["not_implemented"][modname] = None
         elif isinstance(rustpymod, Exception):
-            failed_to_import[modname] = rustpymod
+            result["failed_to_import"][modname] = rustpymod.__class__.__name__ + str(rustpymod)
         else:
             implemented_items = sorted(set(cpymod) & set(rustpymod))
             mod_missing_items = set(cpymod) - set(rustpymod)
@@ -343,48 +390,18 @@ def method_incompatability_reason(typ, method_name, real_method_value):
                 if rustpymod[item] != cpymod[item]
                 and not isinstance(cpymod[item], Exception)
             ]
-            if mod_missing_items:
-                missing_items[modname] = mod_missing_items
-            if mod_mismatched_items:
-                mismatched_items[modname] = mod_mismatched_items
-
-    # missing entire module
-    print("# modules")
-    for modname in not_implemented:
-        print(modname, "(entire module)")
-    for modname, exception in failed_to_import.items():
-        print(f"{modname} (exists but not importable: {exception})")
-
-    # missing from builtins
-    print("\n# builtin items")
-    for module, missing_methods in not_implementeds.items():
-        for method, reason in missing_methods.items():
-            print(f"{module}.{method}" + (f" ({reason})" if reason else ""))
-
-    # missing from modules
-    print("\n# stdlib items")
-    for modname, missing in missing_items.items():
-        for item in missing:
-            print(item)
-
-    print("\n# mismatching signatures (warnings)")
-    for modname, mismatched in mismatched_items.items():
-        for (item, rustpy_value, cpython_value) in mismatched:
-            if cpython_value == "ValueError('no signature found')":
-                continue  # these items will never match
-            print(f"{item} {rustpy_value} != {cpython_value}")
+            if mod_missing_items or mod_mismatched_items:
+                if mod_missing_items:
+                    result["missing_items"][modname] = mod_missing_items
+                if mod_mismatched_items:
+                    result["mismatched_items"][modname] = mod_mismatched_items
+            else:
+                result["implemented"][modname] = None
 
-    result = {
-        "not_implemented": not_implemented,
-        "failed_to_import": failed_to_import,
-        "missing_items": missing_items,
-        "mismatched_items": mismatched_items,
-    }
+    result["cpython_modules"] = cpymods
+    result["not_implementeds"] = not_implementeds
 
-    print()
-    print("# out of", len(cpymods), "modules:")
-    for error_type, modules in result.items():
-        print("# ", error_type, len(modules))
+    print(json.dumps(result))
 
 
 def remove_one_indent(s):
@@ -395,5 +412,54 @@ def remove_one_indent(s):
 compare_src = inspect.getsourcelines(compare)[0][1:]
 output += "".join(remove_one_indent(line) for line in compare_src)
 
-with open("not_impl.py", "w") as f:
+with open(GENERATED_FILE, "w") as f:
     f.write(output + "\n")
+
+
+subprocess.run(["cargo", "build", "--release", "--features=ssl"], check=True)
+result = subprocess.run(
+    ["cargo", "run", "--release", "--features=ssl", "-q", "--", GENERATED_FILE],
+    env={**os.environ.copy(), "RUSTPYTHONPATH": "Lib"},
+    text=True,
+    capture_output=True,
+)
+# The last line should be json output, the rest of the lines can contain noise
+# because importing certain modules can print stuff to stdout/stderr
+result = json.loads(result.stdout.splitlines()[-1])
+
+if args.json:
+    print(json.dumps(result))
+    sys.exit()
+
+
+# missing entire modules
+print("# modules")
+for modname in result["not_implemented"]:
+    print(modname, "(entire module)")
+for modname, exception in result["failed_to_import"].items():
+    print(f"{modname} (exists but not importable: {exception})")
+
+# missing from builtins
+print("\n# builtin items")
+for module, missing_methods in result["not_implementeds"].items():
+    for method, reason in missing_methods.items():
+        print(f"{module}.{method}" + (f" {reason}" if reason else ""))
+
+# missing from modules
+print("\n# stdlib items")
+for modname, missing in result["missing_items"].items():
+    for item in missing:
+        print(item)
+
+if args.signature:
+    print("\n# mismatching signatures (warnings)")
+    for modname, mismatched in result["mismatched_items"].items():
+        for (item, rustpy_value, cpython_value) in mismatched:
+            if cpython_value == "ValueError('no signature found')":
+                continue # these items will never match
+            print(f"{item} {rustpy_value} != {cpython_value}")
+
+print()
+print("# summary")
+for error_type, modules in result.items():
+    print("# ", error_type, len(modules))
diff --git a/whats_left.sh b/whats_left.sh