From cec1f7a40b508e8bc80949c060156c205f02ecb0 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 25 Apr 2025 21:52:14 +0500 Subject: [PATCH 1/6] Adjust c-analyzer max_sizes for typeobject.c --- Tools/c-analyzer/cpython/_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 037fe11ea223c7..682512801cd138 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -323,7 +323,7 @@ def clean_lines(text): _abs('Modules/_testcapimodule.c'): (20_000, 400), _abs('Modules/expat/expat.h'): (10_000, 400), _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), - _abs('Objects/typeobject.c'): (35_000, 200), + _abs('Objects/typeobject.c'): (380_000, 13_000), _abs('Python/compile.c'): (20_000, 500), _abs('Python/optimizer.c'): (100_000, 5_000), _abs('Python/parking_lot.c'): (40_000, 1000), From 45be756b3dd96b5fd0902dab825badad4c28913c Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 1 May 2025 00:18:18 +0500 Subject: [PATCH 2/6] Extend error messages if too much code to c-analyzer --- Tools/c-analyzer/c_parser/parser/__init__.py | 24 ++++++++++++++++++-- Tools/c-analyzer/c_parser/parser/_info.py | 10 ++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index ff4f303c4a2bec..e4a3d07a3c0d0f 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -208,7 +208,27 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): return # At this point either the file ended prematurely # or there's "too much" text. - filename, lno, text = srcinfo.filename, srcinfo._start, srcinfo.text + filename, lno_from, lno_to = srcinfo.filename, srcinfo.start, srcinfo.end + text = srcinfo.text if len(text) > 500: text = text[:500] + '...' - raise Exception(f'unmatched text ({filename} starting at line {lno}):\n{text}') + + if srcinfo.too_much_text(maxtext): + msg = [ + 'too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py', + f'{filename} starting at line {lno_from} to {lno_to}', + f'has code with length {len(text)} greater than {maxtext}:', + text + ] + raise Exception('\n'.join(msg)) + + if srcinfo.too_much_lines(maxlines): + msg = [ + 'too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py', + f'{filename} starting at line {lno_from} to {lno_to}', + f'has code with number of lines {lno_to - lno_from} greater than {maxlines}:', + text + ] + raise Exception('\n'.join(msg)) + + raise Exception(f'unmatched text ({filename} starting at line {lno_from}):\n{text}') diff --git a/Tools/c-analyzer/c_parser/parser/_info.py b/Tools/c-analyzer/c_parser/parser/_info.py index 340223db933c90..fbfc1d74f9a27a 100644 --- a/Tools/c-analyzer/c_parser/parser/_info.py +++ b/Tools/c-analyzer/c_parser/parser/_info.py @@ -123,10 +123,16 @@ def resolve(self, kind, data, name, parent=None): def done(self): self._set_ready() + def too_much_text(self, maxtext): + return maxtext and len(self.text) > maxtext + + def too_much_lines(self, maxlines): + return maxlines and self.end - self.start > maxlines + def too_much(self, maxtext, maxlines): - if maxtext and len(self.text) > maxtext: + if self.too_much_text(maxtext): pass - elif maxlines and self.end - self.start > maxlines: + elif self.too_much_lines(maxlines): pass else: return False From ce4b06cb8f4343de0f2c3cf80de3a69623ae1175 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Thu, 1 May 2025 00:28:28 +0500 Subject: [PATCH 3/6] Make linter happy --- Tools/c-analyzer/c_parser/parser/_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/c_parser/parser/_info.py b/Tools/c-analyzer/c_parser/parser/_info.py index fbfc1d74f9a27a..3e831f977922a2 100644 --- a/Tools/c-analyzer/c_parser/parser/_info.py +++ b/Tools/c-analyzer/c_parser/parser/_info.py @@ -125,7 +125,7 @@ def done(self): def too_much_text(self, maxtext): return maxtext and len(self.text) > maxtext - + def too_much_lines(self, maxlines): return maxlines and self.end - self.start > maxlines From c514255ccd08574336db0c59f192c3abda0ee2fe Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Tue, 22 Jul 2025 22:24:08 +0500 Subject: [PATCH 4/6] Address review comments --- Tools/c-analyzer/c_parser/parser/__init__.py | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index e4a3d07a3c0d0f..bb3b9263aae457 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -214,21 +214,21 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): text = text[:500] + '...' if srcinfo.too_much_text(maxtext): - msg = [ - 'too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py', - f'{filename} starting at line {lno_from} to {lno_to}', - f'has code with length {len(text)} greater than {maxtext}:', - text - ] - raise Exception('\n'.join(msg)) + import textwrap + msg = f'''too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py + {filename} starting at line {lno_from} to {lno_to} + has code with length {len(text)} greater than {maxtext}: + {text} + ''' + raise RuntimeError(textwrap.dedent(msg)) if srcinfo.too_much_lines(maxlines): - msg = [ - 'too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py', - f'{filename} starting at line {lno_from} to {lno_to}', - f'has code with number of lines {lno_to - lno_from} greater than {maxlines}:', - text - ] - raise Exception('\n'.join(msg)) - - raise Exception(f'unmatched text ({filename} starting at line {lno_from}):\n{text}') + import textwrap + msg = f'''too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py + {filename} starting at line {lno_from} to {lno_to} + has code with number of lines {lno_to - lno_from} greater than {maxlines}: + {text} + ''' + raise RuntimeError(textwrap.dedent(msg)) + + raise RuntimeError(f'unmatched text ({filename} starting at line {lno_from}):\n{text}') From 4369d2d4107abc6593555659441645f7e58b870a Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Tue, 22 Jul 2025 22:34:40 +0500 Subject: [PATCH 5/6] Fix dedent usage --- Tools/c-analyzer/c_parser/parser/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index bb3b9263aae457..e1ceca9f32c030 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -215,7 +215,8 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): if srcinfo.too_much_text(maxtext): import textwrap - msg = f'''too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py + msg = f''' + too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py {filename} starting at line {lno_from} to {lno_to} has code with length {len(text)} greater than {maxtext}: {text} @@ -224,7 +225,8 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): if srcinfo.too_much_lines(maxlines): import textwrap - msg = f'''too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py + msg = f''' + too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py {filename} starting at line {lno_from} to {lno_to} has code with number of lines {lno_to - lno_from} greater than {maxlines}: {text} From e60d28c21e22341ec8d297106233b2bcfb5eda02 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 23 Jul 2025 10:00:48 +0500 Subject: [PATCH 6/6] Update Tools/c-analyzer/c_parser/parser/__init__.py Co-authored-by: Peter Bierma --- Tools/c-analyzer/c_parser/parser/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/c_parser/parser/__init__.py b/Tools/c-analyzer/c_parser/parser/__init__.py index e1ceca9f32c030..9ea4184e233846 100644 --- a/Tools/c-analyzer/c_parser/parser/__init__.py +++ b/Tools/c-analyzer/c_parser/parser/__init__.py @@ -226,7 +226,7 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False): if srcinfo.too_much_lines(maxlines): import textwrap msg = f''' - too much lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py + too many lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py {filename} starting at line {lno_from} to {lno_to} has code with number of lines {lno_to - lno_from} greater than {maxlines}: {text}