diff --git a/scripts/find_eq.py b/scripts/find_eq.py index 5e31746041..b79982807b 100644 --- a/scripts/find_eq.py +++ b/scripts/find_eq.py @@ -3,18 +3,29 @@ # --cpython: Path to cpython source code # --print-diff: Print the diff between the files # --color: Output color -# --files: Optional globbing pattern to match files in cpython source code +# --files: Optional globbing pattern to match files in cpython source code # --checklist: output as checklist import argparse import difflib import pathlib -parser = argparse.ArgumentParser(description="Find equivalent files in cpython and rustpython") -parser.add_argument("--cpython", type=pathlib.Path, required=True, help="Path to cpython source code") -parser.add_argument("--print-diff", action="store_true", help="Print the diff between the files") +parser = argparse.ArgumentParser( + description="Find equivalent files in cpython and rustpython" +) +parser.add_argument( + "--cpython", type=pathlib.Path, required=True, help="Path to cpython source code" +) +parser.add_argument( + "--print-diff", action="store_true", help="Print the diff between the files" +) parser.add_argument("--color", action="store_true", help="Output color") -parser.add_argument("--files", type=str, default="*.py", help="Optional globbing pattern to match files in cpython source code") +parser.add_argument( + "--files", + type=str, + default="*.py", + help="Optional globbing pattern to match files in cpython source code", +) args = parser.parse_args() @@ -27,7 +38,9 @@ cpython_lib = args.cpython / "Lib" rustpython_lib = pathlib.Path(__file__).parent.parent / "Lib" -assert rustpython_lib.exists(), "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place" +assert rustpython_lib.exists(), ( + "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place" +) # walk through the cpython lib directory cpython_files = [] @@ -48,7 +61,13 @@ with open(rustpython_lib / path, "r") as rustpython_file: rustpython_code = rustpython_file.read() # compare the files - diff = difflib.unified_diff(cpython_code.splitlines(), rustpython_code.splitlines(), lineterm="", fromfile=str(path), tofile=str(path)) + diff = difflib.unified_diff( + cpython_code.splitlines(), + rustpython_code.splitlines(), + lineterm="", + fromfile=str(path), + tofile=str(path), + ) # print the diff if there are differences diff = list(diff) if len(diff) > 0: diff --git a/scripts/generate_checklist.py b/scripts/generate_checklist.py index 4df87d0d52..9a444b16e2 100644 --- a/scripts/generate_checklist.py +++ b/scripts/generate_checklist.py @@ -13,12 +13,19 @@ import requests from jinja2 import Environment, FileSystemLoader -parser = argparse.ArgumentParser(description="Find equivalent files in cpython and rustpython") -parser.add_argument("--cpython", type=pathlib.Path, required=True, help="Path to cpython source code") -parser.add_argument("--notes", type=pathlib.Path, required=False, help="Path to notes file") +parser = argparse.ArgumentParser( + description="Find equivalent files in cpython and rustpython" +) +parser.add_argument( + "--cpython", type=pathlib.Path, required=True, help="Path to cpython source code" +) +parser.add_argument( + "--notes", type=pathlib.Path, required=False, help="Path to notes file" +) args = parser.parse_args() + def check_pr(pr_id: str) -> bool: if pr_id.startswith("#"): pr_id = pr_id[1:] @@ -27,11 +34,13 @@ def check_pr(pr_id: str) -> bool: response = requests.get(req).json() return response["merged_at"] is not None + @dataclasses.dataclass class LibUpdate: pr: Optional[str] = None done: bool = True + def parse_updated_lib_issue(issue_body: str) -> dict[str, LibUpdate]: lines = issue_body.splitlines() updated_libs = {} @@ -47,12 +56,14 @@ def parse_updated_lib_issue(issue_body: str) -> dict[str, LibUpdate]: updated_libs[out[0]] = LibUpdate(out[1], check_pr(out[1])) return updated_libs + def get_updated_libs() -> dict[str, LibUpdate]: issue_id = "5736" req = f"https://api.github.com/repos/RustPython/RustPython/issues/{issue_id}" response = requests.get(req).json() return parse_updated_lib_issue(response["body"]) + updated_libs = get_updated_libs() if not args.cpython.exists(): @@ -86,12 +97,11 @@ def get_updated_libs() -> dict[str, LibUpdate]: cpython_lib = args.cpython / "Lib" rustpython_lib = pathlib.Path(__file__).parent.parent / "Lib" -assert rustpython_lib.exists(), "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place" +assert rustpython_lib.exists(), ( + "RustPython lib directory does not exist, ensure the find_eq.py script is located in the right place" +) -ignored_objs = [ - "__pycache__", - "test" -] +ignored_objs = ["__pycache__", "test"] # loop through the top-level directories in the cpython lib directory libs = [] for path in cpython_lib.iterdir(): @@ -105,15 +115,25 @@ def get_updated_libs() -> dict[str, LibUpdate]: tests = [] cpython_lib_test = cpython_lib / "test" for path in cpython_lib_test.iterdir(): - if path.is_dir() and path.name not in ignored_objs and path.name.startswith("test_"): + if ( + path.is_dir() + and path.name not in ignored_objs + and path.name.startswith("test_") + ): # add the directory name to the list of libraries tests.append(path.name) - elif path.is_file() and path.name.endswith(".py") and path.name not in ignored_objs and path.name.startswith("test_"): + elif ( + path.is_file() + and path.name.endswith(".py") + and path.name not in ignored_objs + and path.name.startswith("test_") + ): # add the file name to the list of libraries file_name = path.name.replace("test_", "") if file_name not in libs and file_name.replace(".py", "") not in libs: tests.append(path.name) + def check_diff(file1, file2): try: with open(file1, "r") as f1, open(file2, "r") as f2: @@ -125,12 +145,14 @@ def check_diff(file1, file2): except UnicodeDecodeError: return False + def check_completion_pr(display_name): for lib in updated_libs: if lib == str(display_name): return updated_libs[lib].done, updated_libs[lib].pr return False, None + def check_test_completion(rustpython_path, cpython_path): if rustpython_path.exists() and rustpython_path.is_file(): if cpython_path.exists() and cpython_path.is_file(): @@ -141,11 +163,14 @@ def check_test_completion(rustpython_path, cpython_path): return True return False + def check_lib_completion(rustpython_path, cpython_path): test_name = "test_" + rustpython_path.name rustpython_test_path = rustpython_lib / "test" / test_name cpython_test_path = cpython_lib / "test" / test_name - if cpython_test_path.exists() and not check_test_completion(rustpython_test_path, cpython_test_path): + if cpython_test_path.exists() and not check_test_completion( + rustpython_test_path, cpython_test_path + ): return False if rustpython_path.exists() and rustpython_path.is_file(): if check_diff(rustpython_path, cpython_path) > 0: @@ -153,6 +178,7 @@ def check_lib_completion(rustpython_path, cpython_path): return True return False + def handle_notes(display_path) -> list[str]: if str(display_path) in notes: res = notes[str(display_path)] @@ -161,6 +187,7 @@ def handle_notes(display_path) -> list[str]: return res return [] + @dataclasses.dataclass class Output: name: str @@ -168,6 +195,7 @@ class Output: completed: Optional[bool] notes: list[str] + update_libs_output = [] add_libs_output = [] for path in libs: @@ -183,12 +211,18 @@ class Output: # check if the file exists in the rustpython lib directory if rustpython_path.exists() and rustpython_path.is_file(): completed = check_lib_completion(rustpython_path, cpython_path) - update_libs_output.append(Output(str(display_path), pr, completed, handle_notes(display_path))) + update_libs_output.append( + Output(str(display_path), pr, completed, handle_notes(display_path)) + ) else: if pr is not None and completed: - update_libs_output.append(Output(str(display_path), pr, None, handle_notes(display_path))) + update_libs_output.append( + Output(str(display_path), pr, None, handle_notes(display_path)) + ) else: - add_libs_output.append(Output(str(display_path), pr, None, handle_notes(display_path))) + add_libs_output.append( + Output(str(display_path), pr, None, handle_notes(display_path)) + ) update_tests_output = [] add_tests_output = [] @@ -205,24 +239,30 @@ class Output: # check if the file exists in the rustpython lib directory if rustpython_path.exists() and rustpython_path.is_file(): completed = check_lib_completion(rustpython_path, cpython_path) - update_tests_output.append(Output(str(display_path), pr, completed, handle_notes(display_path))) + update_tests_output.append( + Output(str(display_path), pr, completed, handle_notes(display_path)) + ) else: if pr is not None and completed: - update_tests_output.append(Output(str(display_path), pr, None, handle_notes(display_path))) + update_tests_output.append( + Output(str(display_path), pr, None, handle_notes(display_path)) + ) else: - add_tests_output.append(Output(str(display_path), pr, None, handle_notes(display_path))) + add_tests_output.append( + Output(str(display_path), pr, None, handle_notes(display_path)) + ) for note in notes: # add a warning for each note that is not attached to a file for n in notes[note]: warnings.warn(f"Unattached Note: {note} - {n}") -env = Environment(loader=FileSystemLoader('.')) +env = Environment(loader=FileSystemLoader(".")) template = env.get_template("checklist_template.md") output = template.render( update_libs=update_libs_output, add_libs=add_libs_output, update_tests=update_tests_output, - add_tests=add_tests_output + add_tests=add_tests_output, ) print(output)