From 3c708fd566e6508d7c82bdfeead0ac372f7d8f22 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 12 Jan 2024 15:36:55 +0100 Subject: [PATCH 1/2] gh-113317: Move global utility functions into libclinic Establish Tools/clinic/libclinic/utils.py and move the following functions over there: - compute_checksum() - create_regex() - write_file() --- Tools/clinic/clinic.py | 66 +++++------------------------- Tools/clinic/libclinic/__init__.py | 10 +++++ Tools/clinic/libclinic/utils.py | 44 ++++++++++++++++++++ 3 files changed, 64 insertions(+), 56 deletions(-) create mode 100644 Tools/clinic/libclinic/utils.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index f6f95580f1a177..d45159f47a0989 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -16,7 +16,6 @@ import dataclasses as dc import enum import functools -import hashlib import inspect import io import itertools @@ -1792,21 +1791,6 @@ def render_function( return clinic.get_destination('block').dump() -def create_regex( - before: str, - after: str, - word: bool = True, - whole_line: bool = True -) -> re.Pattern[str]: - """Create an re object for matching marker lines.""" - group_re = r"\w+" if word else ".+" - pattern = r'{}({}){}' - if whole_line: - pattern = '^' + pattern + '$' - pattern = pattern.format(re.escape(before), group_re, re.escape(after)) - return re.compile(pattern) - - @dc.dataclass(slots=True, repr=False) class Block: r""" @@ -1905,8 +1889,9 @@ def __init__( self.language = language before, _, after = language.start_line.partition('{dsl_name}') assert _ == '{dsl_name}' - self.find_start_re = create_regex(before, after, whole_line=False) - self.start_re = create_regex(before, after) + self.find_start_re = libclinic.create_regex(before, after, + whole_line=False) + self.start_re = libclinic.create_regex(before, after) self.verify = verify self.last_checksum_re: re.Pattern[str] | None = None self.last_dsl_name: str | None = None @@ -1995,7 +1980,7 @@ def is_stop_line(line: str) -> bool: else: before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}') assert _ == '{arguments}' - checksum_re = create_regex(before, after, word=False) + checksum_re = libclinic.create_regex(before, after, word=False) self.last_dsl_name = dsl_name self.last_checksum_re = checksum_re assert checksum_re is not None @@ -2029,7 +2014,7 @@ def is_stop_line(line: str) -> bool: else: checksum = d['checksum'] - computed = compute_checksum(output, len(checksum)) + computed = libclinic.compute_checksum(output, len(checksum)) if checksum != computed: fail("Checksum mismatch! " f"Expected {checksum!r}, computed {computed!r}. " @@ -2142,8 +2127,8 @@ def print_block( write(output) arguments = "output={output} input={input}".format( - output=compute_checksum(output, 16), - input=compute_checksum(input, 16) + output=libclinic.compute_checksum(output, 16), + input=libclinic.compute_checksum(input, 16) ) write(self.language.checksum_line.format(dsl_name=dsl_name, arguments=arguments)) write("\n") @@ -2245,27 +2230,6 @@ def dump(self) -> str: extensions['py'] = PythonLanguage -def write_file(filename: str, new_contents: str) -> None: - try: - with open(filename, encoding="utf-8") as fp: - old_contents = fp.read() - - if old_contents == new_contents: - # no change: avoid modifying the file modification time - return - except FileNotFoundError: - pass - # Atomic write using a temporary file and os.replace() - filename_new = f"{filename}.new" - with open(filename_new, "w", encoding="utf-8") as fp: - fp.write(new_contents) - try: - os.replace(filename_new, filename) - except: - os.unlink(filename_new) - raise - - ClassDict = dict[str, "Class"] DestinationDict = dict[str, Destination] ModuleDict = dict[str, "Module"] @@ -2505,7 +2469,8 @@ def parse(self, input: str) -> str: core_includes=True, limited_capi=self.limited_capi, header_includes=self.includes) - write_file(destination.filename, printer_2.f.getvalue()) + libclinic.write_file(destination.filename, + printer_2.f.getvalue()) continue return printer.f.getvalue() @@ -2578,18 +2543,7 @@ def parse_file( limited_capi=limited_capi) cooked = clinic.parse(raw) - write_file(output, cooked) - - -def compute_checksum( - input: str | None, - length: int | None = None -) -> str: - input = input or '' - s = hashlib.sha1(input.encode('utf-8')).hexdigest() - if length: - s = s[:length] - return s + libclinic.write_file(output, cooked) class PythonParser: diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index d4e7a0c5cf7b76..f26bf3ef6d55fe 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -15,6 +15,11 @@ wrap_declarations, wrapped_c_string_literal, ) +from .utils import ( + create_regex, + compute_checksum, + write_file, +) __all__ = [ @@ -32,6 +37,11 @@ "suffix_all_lines", "wrap_declarations", "wrapped_c_string_literal", + + # Utility functions + "create_regex", + "compute_checksum", + "write_file", ] diff --git a/Tools/clinic/libclinic/utils.py b/Tools/clinic/libclinic/utils.py new file mode 100644 index 00000000000000..273c34374ea2d1 --- /dev/null +++ b/Tools/clinic/libclinic/utils.py @@ -0,0 +1,44 @@ +import hashlib +import re +import os + + +def write_file(filename: str, new_contents: str) -> None: + """Write new content to file, iff the content changed.""" + try: + with open(filename, encoding="utf-8") as fp: + old_contents = fp.read() + + if old_contents == new_contents: + # no change: avoid modifying the file modification time + return + except FileNotFoundError: + pass + # Atomic write using a temporary file and os.replace() + filename_new = f"{filename}.new" + with open(filename_new, "w", encoding="utf-8") as fp: + fp.write(new_contents) + try: + os.replace(filename_new, filename) + except: + os.unlink(filename_new) + raise + + +def compute_checksum(input_: str, length: int | None = None) -> str: + checksum = hashlib.sha1(input_.encode("utf-8")).hexdigest() + if length: + checksum = checksum[:length] + return checksum + + +def create_regex( + before: str, after: str, word: bool = True, whole_line: bool = True +) -> re.Pattern[str]: + """Create a regex object for matching marker lines.""" + group_re = r"\w+" if word else ".+" + pattern = r"{}({}){}" + if whole_line: + pattern = "^" + pattern + "$" + pattern = pattern.format(re.escape(before), group_re, re.escape(after)) + return re.compile(pattern) From a13702bfb388dd47cbef85295830438f10cb8916 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 14 Jan 2024 19:02:48 +0100 Subject: [PATCH 2/2] Update Tools/clinic/libclinic/utils.py --- Tools/clinic/libclinic/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Tools/clinic/libclinic/utils.py b/Tools/clinic/libclinic/utils.py index 273c34374ea2d1..1514558cb5665c 100644 --- a/Tools/clinic/libclinic/utils.py +++ b/Tools/clinic/libclinic/utils.py @@ -37,8 +37,9 @@ def create_regex( ) -> re.Pattern[str]: """Create a regex object for matching marker lines.""" group_re = r"\w+" if word else ".+" - pattern = r"{}({}){}" + before = re.escape(before) + after = re.escape(after) + pattern = fr"{before}({group_re}){after}" if whole_line: - pattern = "^" + pattern + "$" - pattern = pattern.format(re.escape(before), group_re, re.escape(after)) + pattern = fr"^{pattern}$" return re.compile(pattern)