RustPython · arihant2math · Apr 1, 2025 · Apr 23, 2025
diff --git a/scripts/checklist_template.md b/scripts/checklist_template.md
@@ -0,0 +1,21 @@
+{% macro display_line(i) %}- {% if i.completed == True %}[x] {% elif i.completed == False %}[ ] {% endif %}{{ i.name }}{% if i.pr != None %} {{ i.pr }}{% endif %}{% endmacro %}
+# List of libraries
+
+{% for lib in update_libs %}{{ display_line(lib) }}
+{% endfor %}
+
+# List of un-added libraries
+These libraries are not added yet. Pure python one will be possible while others are not.
+
+{% for lib in add_libs %}{{ display_line(lib) }}
+{% endfor %}
+
+# List of tests without python libraries
+
+{% for lib in update_tests %}{{ display_line(lib) }}
+{% endfor %}
+
+# List of un-added tests without python libraries
+
+{% for lib in add_tests %}{{ display_line(lib) }}
+{% endfor %}
diff --git a/scripts/find_eq.py b/scripts/find_eq.py
@@ -0,0 +1,76 @@
+# Run differential queries to find equivalent files in cpython and rustpython
+# Arguments
+# --cpython: Path to cpython source code
+# --print-diff: Print the diff between the files
+# --color: Output color
+# --files: Optional globbing pattern to match files in cpython source code 
+# --checklist: output as checklist
+
+import argparse
+import difflib
+import pathlib
+
+parser = argparse.ArgumentParser(description="Find equivalent files in cpython and rustpython")
+parser.add_argument("--cpython", type=pathlib.Path, required=True, help="Path to cpython source code")
+parser.add_argument("--print-diff", action="store_true", help="Print the diff between the files")
+parser.add_argument("--color", action="store_true", help="Output color")
+parser.add_argument("--files", type=str, default="*.py", help="Optional globbing pattern to match files in cpython source code")
+
+args = parser.parse_args()
+
+if not args.cpython.exists():
+    raise FileNotFoundError(f"Path {args.cpython} does not exist")
+if not args.cpython.is_dir():
+    raise NotADirectoryError(f"Path {args.cpython} is not a directory")
+if not args.cpython.is_absolute():
+    args.cpython = args.cpython.resolve()
+
+cpython_lib = args.cpython / "Lib"
+rustpython_lib = pathlib.Path(__file__).parent.parent / "Lib"
+assert rustpython_lib.exists(), "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place"
+
+# walk through the cpython lib directory
+cpython_files = []
+for path in cpython_lib.rglob(args.files):
+    if path.is_file():
+        # remove the cpython lib path from the file path
+        path = path.relative_to(cpython_lib)
+        cpython_files.append(path)
+
+for path in cpython_files:
+    # check if the file exists in the rustpython lib directory
+    rustpython_path = rustpython_lib / path
+    if rustpython_path.exists():
+        # open both files and compare them
+        try:
+            with open(cpython_lib / path, "r") as cpython_file:
+                cpython_code = cpython_file.read()
+            with open(rustpython_lib / path, "r") as rustpython_file:
+                rustpython_code = rustpython_file.read()
+            # compare the files
+            diff = difflib.unified_diff(cpython_code.splitlines(), rustpython_code.splitlines(), lineterm="", fromfile=str(path), tofile=str(path))
+            # print the diff if there are differences
+            diff = list(diff)
+            if len(diff) > 0:
+                if args.print_diff:
+                    print("Differences:")
+                    for line in diff:
+                        print(line)
+                else:
+                    print(f"File is not identical: {path}")
+            else:
+                print(f"File is identical: {path}")
+        except Exception as e:
+            print(f"Unable to check file {path}: {e}")
+    else:
+        print(f"File not found in RustPython: {path}")
+
+# check for files in rustpython lib directory that are not in cpython lib directory
+rustpython_files = []
+for path in rustpython_lib.rglob(args.files):
+    if path.is_file():
+        # remove the rustpython lib path from the file path
+        path = path.relative_to(rustpython_lib)
+        rustpython_files.append(path)
+        if path not in cpython_files:
+            print(f"File not found in CPython: {path}")
diff --git a/scripts/generate_checklist.py b/scripts/generate_checklist.py
@@ -0,0 +1,228 @@
+# Arguments
+# --cpython: Path to cpython source code
+# --updated-libs: Libraries that have been updated in RustPython
+
+
+import argparse
+import dataclasses
+import difflib
+import pathlib
+from typing import Optional
+import warnings
+
+import requests
+from jinja2 import Environment, FileSystemLoader
+
+parser = argparse.ArgumentParser(description="Find equivalent files in cpython and rustpython")
+parser.add_argument("--cpython", type=pathlib.Path, required=True, help="Path to cpython source code")
+parser.add_argument("--notes", type=pathlib.Path, required=False, help="Path to notes file")
+
+args = parser.parse_args()
+
+def check_pr(pr_id: str) -> bool:
+    if pr_id.startswith("#"):
+        pr_id = pr_id[1:]
+    int_pr_id = int(pr_id)
+    req = f"https://api.github.com/repos/RustPython/RustPython/pulls/{int_pr_id}"
+    response = requests.get(req).json()
+    return response["merged_at"] is not None
+
+@dataclasses.dataclass
+class LibUpdate:
+    pr: Optional[str] = None
+    done: bool = True
+
+def parse_updated_lib_issue(issue_body: str) -> dict[str, LibUpdate]:
+    lines = issue_body.splitlines()
+    updated_libs = {}
+    for line in lines:
+        if line.strip().startswith("- "):
+            line = line.strip()[2:]
+            out = line.split(" ")
+            out = [x for x in out if x]
+            assert len(out) < 3
+            if len(out) == 1:
+                updated_libs[out[0]] = LibUpdate()
+            elif len(out) == 2:
+                updated_libs[out[0]] = LibUpdate(out[1], check_pr(out[1]))
+    return updated_libs
+
+def get_updated_libs() -> dict[str, LibUpdate]:
+    issue_id = "5736"
+    req = f"https://api.github.com/repos/RustPython/RustPython/issues/{issue_id}"
+    response = requests.get(req).json()
+    return parse_updated_lib_issue(response["body"])
+
+updated_libs = get_updated_libs()
+
+if not args.cpython.exists():
+    raise FileNotFoundError(f"Path {args.cpython} does not exist")
+if not args.cpython.is_dir():
+    raise NotADirectoryError(f"Path {args.cpython} is not a directory")
+if not args.cpython.is_absolute():
+    args.cpython = args.cpython.resolve()
+
+notes: dict = {}
+if args.notes:
+    # check if the file exists in the rustpython lib directory
+    notes_path = args.notes
+    if notes_path.exists():
+        with open(notes_path) as f:
+            for line in f:
+                line = line.strip()
+                if not line.startswith("//") and line:
+                    line_split = line.split(" ")
+                    if len(line_split) > 1:
+                        rest = " ".join(line_split[1:])
+                        if line_split[0] in notes:
+                            notes[line_split[0]].append(rest)
+                        else:
+                            notes[line_split[0]] = [rest]
+                    else:
+                        raise ValueError(f"Invalid note: {line}")
+
+    else:
+        raise FileNotFoundError(f"Path {notes_path} does not exist")
+
+cpython_lib = args.cpython / "Lib"
+rustpython_lib = pathlib.Path(__file__).parent.parent / "Lib"
+assert rustpython_lib.exists(), "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place"
+
+ignored_objs = [
+    "__pycache__",
+    "test"
+]
+# loop through the top-level directories in the cpython lib directory
+libs = []
+for path in cpython_lib.iterdir():
+    if path.is_dir() and path.name not in ignored_objs:
+        # add the directory name to the list of libraries
+        libs.append(path.name)
+    elif path.is_file() and path.name.endswith(".py") and path.name not in ignored_objs:
+        # add the file name to the list of libraries
+        libs.append(path.name)
+
+tests = []
+cpython_lib_test = cpython_lib / "test"
+for path in cpython_lib_test.iterdir():
+    if path.is_dir() and path.name not in ignored_objs and path.name.startswith("test_"):
+        # add the directory name to the list of libraries
+        tests.append(path.name)
+    elif path.is_file() and path.name.endswith(".py") and path.name not in ignored_objs and path.name.startswith("test_"):
+        # add the file name to the list of libraries
+        file_name = path.name.replace("test_", "")
+        if file_name not in libs and file_name.replace(".py", "") not in libs:
+            tests.append(path.name)
+
+def check_diff(file1, file2):
+    try:
+        with open(file1, "r") as f1, open(file2, "r") as f2:
+            f1_lines = f1.readlines()
+            f2_lines = f2.readlines()
+            diff = difflib.unified_diff(f1_lines, f2_lines, lineterm="")
+            diff_lines = list(diff)
+            return len(diff_lines)
+    except UnicodeDecodeError:
+        return False
+
+def check_completion_pr(display_name):
+    for lib in updated_libs:
+        if lib == str(display_name):
+            return updated_libs[lib].done, updated_libs[lib].pr
+    return False, None
+
+def check_test_completion(rustpython_path, cpython_path):
+    if rustpython_path.exists() and rustpython_path.is_file():
+        if cpython_path.exists() and cpython_path.is_file():
+            if not rustpython_path.exists() or not rustpython_path.is_file():
+                return False
+            elif check_diff(rustpython_path, cpython_path) > 0:
+                return False
+            return True
+    return False
+
+def check_lib_completion(rustpython_path, cpython_path):
+    test_name = "test_" + rustpython_path.name
+    rustpython_test_path = rustpython_lib / "test" / test_name
+    cpython_test_path = cpython_lib / "test" / test_name
+    if cpython_test_path.exists() and not check_test_completion(rustpython_test_path, cpython_test_path):
+        return False
+    if rustpython_path.exists() and rustpython_path.is_file():
+        if check_diff(rustpython_path, cpython_path) > 0:
+            return False
+        return True
+    return False
+
+def handle_notes(display_path) -> list[str]:
+    if str(display_path) in notes:
+        res = notes[str(display_path)]
+        # remove the note from the notes list
+        del notes[str(display_path)]
+        return res
+    return []
+
+@dataclasses.dataclass
+class Output:
+    name: str
+    pr: Optional[str]
+    completed: Optional[bool]
+    notes: list[str]
+
+update_libs_output = []
+add_libs_output = []
+for path in libs:
+    # check if the file exists in the rustpython lib directory
+    rustpython_path = rustpython_lib / path
+    # remove the file extension if it exists
+    display_path = pathlib.Path(path).with_suffix("")
+    (completed, pr) = check_completion_pr(display_path)
+    if rustpython_path.exists():
+        if not completed:
+            # check if the file exists in the cpython lib directory
+            cpython_path = cpython_lib / path
+            # check if the file exists in the rustpython lib directory
+            if rustpython_path.exists() and rustpython_path.is_file():
+                completed = check_lib_completion(rustpython_path, cpython_path)
+        update_libs_output.append(Output(str(display_path), pr, completed, handle_notes(display_path)))
+    else:
+        if pr is not None and completed:
+            update_libs_output.append(Output(str(display_path), pr, None, handle_notes(display_path)))
+        else:
+            add_libs_output.append(Output(str(display_path), pr, None, handle_notes(display_path)))
+
+update_tests_output = []
+add_tests_output = []
+for path in tests:
+    # check if the file exists in the rustpython lib directory
+    rustpython_path = rustpython_lib / "test" / path
+    # remove the file extension if it exists
+    display_path = pathlib.Path(path).with_suffix("")
+    (completed, pr) = check_completion_pr(display_path)
+    if rustpython_path.exists():
+        if not completed:
+            # check if the file exists in the cpython lib directory
+            cpython_path = cpython_lib / "test" / path
+            # check if the file exists in the rustpython lib directory
+            if rustpython_path.exists() and rustpython_path.is_file():
+                completed = check_lib_completion(rustpython_path, cpython_path)
+        update_tests_output.append(Output(str(display_path), pr, completed, handle_notes(display_path)))
+    else:
+        if pr is not None and completed:
+            update_tests_output.append(Output(str(display_path), pr, None, handle_notes(display_path)))
+        else:
+            add_tests_output.append(Output(str(display_path), pr, None, handle_notes(display_path)))
+
+for note in notes:
+    # add a warning for each note that is not attached to a file
+    for n in notes[note]:
+        warnings.warn(f"Unattached Note: {note} - {n}")
+
+env = Environment(loader=FileSystemLoader('.'))
+template = env.get_template("checklist_template.md")
+output = template.render(
+    update_libs=update_libs_output,
+    add_libs=add_libs_output,
+    update_tests=update_tests_output,
+    add_tests=add_tests_output
+)
+print(output)
diff --git a/scripts/notes.txt b/scripts/notes.txt
@@ -0,0 +1,48 @@
+__future__ Related test is `test_future_stmt`
+abc `_collections_abc.py`
+abc `_py_abc.py`
+code Related test is `test_code_module`
+codecs `_pycodecs.py`
+collections See also #3418
+ctypes #5572
+datetime `_pydatetime.py`
+decimal `_pydecimal.py`
+dis See also #3846
+importlib #4565
+io `_pyio.py`
+io #3960
+io #4702
+locale #3850
+mailbox #4072
+multiprocessing #3965
+os Blocker: Some tests requires async comprehension
+os #3960
+os #4053
+pickle #3876
+pickle `_compat_pickle.py`
+pickle `test/pickletester.py` supports `test_pickle.py`
+pickle `test/test_picklebuffer.py`
+pydoc `pydoc_data`
+queue See also #3608
+re Don't forget sre files `sre_compile.py`, `sre_constants.py`, `sre_parse.py`
+shutil #3960
+site Don't forget `_sitebuiltins.py`
+venv #3960
+warnings #4013
+
+// test
+
+test_array #3876
+test_gc #4158
+test_marshal #3458
+test_mmap #3847
+test_posix #4496
+test_property #3430
+test_set #3992
+test_structseq #4063
+test_super #3865
+test_support #4538
+test_syntax #4469
+test_sys #4541
+test_time #3850
+test_time #4157