Skip to content

Commit 9f02b47

Browse files
ericsnowcurrentlyericsnowcurrently
authored
bpo-36876: [c-analyzer tool] Tighten up the results and output. (pythonGH-23431)
We also update the "ignored" file with a temporary list of all known globals.
1 parent a993e90 commit 9f02b47

File tree

13 files changed

+3202
-109
lines changed

13 files changed

+3202
-109
lines changed

‎Tools/c-analyzer/c_analyzer/__main__.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import re
66
import sys
77

8+
from c_common import fsutil
89
from c_common.logging import VERBOSITY, Printer
910
from c_common.scriptutil import (
1011
add_verbosity_cli,
@@ -298,9 +299,9 @@ def cmd_check(filenames, *,
298299
checks=None,
299300
ignored=None,
300301
fmt=None,
301-
relroot=None,
302302
failfast=False,
303303
iter_filenames=None,
304+
relroot=fsutil.USE_CWD,
304305
track_progress=None,
305306
verbosity=VERBOSITY,
306307
_analyze=_analyze,
@@ -317,14 +318,14 @@ def cmd_check(filenames, *,
317318
(handle_failure, handle_after, div
318319
) = _get_check_handlers(fmt, printer, verbosity)
319320

320-
filenames = filter_filenames(filenames, iter_filenames)
321+
filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
322+
filenames = filter_filenames(filenames, iter_filenames, relroot)
321323
if track_progress:
322324
filenames = track_progress(filenames)
323325

324326
logger.info('analyzing files...')
325327
analyzed = _analyze(filenames, **kwargs)
326-
if relroot:
327-
analyzed.fix_filenames(relroot)
328+
analyzed.fix_filenames(relroot, normalize=False)
328329
decls = filter_forward(analyzed, markpublic=True)
329330

330331
logger.info('checking analysis results...')
@@ -374,6 +375,7 @@ def _cli_analyze(parser, **kwargs):
374375
def cmd_analyze(filenames, *,
375376
fmt=None,
376377
iter_filenames=None,
378+
relroot=fsutil.USE_CWD,
377379
track_progress=None,
378380
verbosity=None,
379381
_analyze=_analyze,
@@ -387,12 +389,14 @@ def cmd_analyze(filenames, *,
387389
except KeyError:
388390
raise ValueError(f'unsupported fmt {fmt!r}')
389391

390-
filenames = filter_filenames(filenames, iter_filenames)
392+
filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
393+
filenames = filter_filenames(filenames, iter_filenames, relroot)
391394
if track_progress:
392395
filenames = track_progress(filenames)
393396

394397
logger.info('analyzing files...')
395398
analyzed = _analyze(filenames, **kwargs)
399+
analyzed.fix_filenames(relroot, normalize=False)
396400
decls = filter_forward(analyzed, markpublic=True)
397401

398402
for line in do_fmt(decls):
@@ -434,7 +438,7 @@ def cmd_data(datacmd, filenames, known=None, *,
434438
_analyze=_analyze,
435439
formats=FORMATS,
436440
extracolumns=None,
437-
relroot=None,
441+
relroot=fsutil.USE_CWD,
438442
track_progress=None,
439443
**kwargs
440444
):
@@ -447,9 +451,11 @@ def cmd_data(datacmd, filenames, known=None, *,
447451
for line in do_fmt(known):
448452
print(line)
449453
elif datacmd == 'dump':
454+
filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
450455
if track_progress:
451456
filenames = track_progress(filenames)
452457
analyzed = _analyze(filenames, **kwargs)
458+
analyzed.fix_filenames(relroot, normalize=False)
453459
if known is None or usestdout:
454460
outfile = io.StringIO()
455461
_datafiles.write_known(analyzed, outfile, extracolumns,

‎Tools/c-analyzer/c_analyzer/datafiles.py

Lines changed: 43 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
import os.path
2+
3+
from c_common import fsutil
14
import c_common.tables as _tables
25
import c_parser.info as _info
36
import c_parser.match as _match
@@ -13,31 +16,10 @@
1316
]
1417

1518

16-
def analyze_known(known, *,
17-
analyze_resolved=None,
18-
handle_unresolved=True,
19-
):
20-
knowntypes = knowntypespecs = {}
21-
collated = _match.group_by_kinds(known)
22-
types = {decl: None for decl in collated['type']}
23-
typespecs = _analyze.get_typespecs(types)
24-
def analyze_decl(decl):
25-
return _analyze.analyze_decl(
26-
decl,
27-
typespecs,
28-
knowntypespecs,
29-
types,
30-
knowntypes,
31-
analyze_resolved=analyze_resolved,
32-
)
33-
_analyze.analyze_type_decls(types, analyze_decl, handle_unresolved)
34-
return types, typespecs
35-
36-
3719
def get_known(known, extracolumns=None, *,
3820
analyze_resolved=None,
3921
handle_unresolved=True,
40-
relroot=None,
22+
relroot=fsutil.USE_CWD,
4123
):
4224
if isinstance(known, str):
4325
known = read_known(known, extracolumns, relroot)
@@ -48,7 +30,7 @@ def get_known(known, extracolumns=None, *,
4830
)
4931

5032

51-
def read_known(infile, extracolumns=None, relroot=None):
33+
def read_known(infile, extracolumns=None, relroot=fsutil.USE_CWD):
5234
extracolumns = EXTRA_COLUMNS + (
5335
list(extracolumns) if extracolumns else []
5436
)
@@ -58,8 +40,29 @@ def read_known(infile, extracolumns=None, relroot=None):
5840
return known
5941

6042

43+
def analyze_known(known, *,
44+
analyze_resolved=None,
45+
handle_unresolved=True,
46+
):
47+
knowntypes = knowntypespecs = {}
48+
collated = _match.group_by_kinds(known)
49+
types = {decl: None for decl in collated['type']}
50+
typespecs = _analyze.get_typespecs(types)
51+
def analyze_decl(decl):
52+
return _analyze.analyze_decl(
53+
decl,
54+
typespecs,
55+
knowntypespecs,
56+
types,
57+
knowntypes,
58+
analyze_resolved=analyze_resolved,
59+
)
60+
_analyze.analyze_type_decls(types, analyze_decl, handle_unresolved)
61+
return types, typespecs
62+
63+
6164
def write_known(rows, outfile, extracolumns=None, *,
62-
relroot=None,
65+
relroot=fsutil.USE_CWD,
6366
backup=True,
6467
):
6568
extracolumns = EXTRA_COLUMNS + (
@@ -86,22 +89,34 @@ def write_known(rows, outfile, extracolumns=None, *,
8689
IGNORED_HEADER = '\t'.join(IGNORED_COLUMNS)
8790

8891

89-
def read_ignored(infile):
90-
return dict(_iter_ignored(infile))
92+
def read_ignored(infile, relroot=fsutil.USE_CWD):
93+
return dict(_iter_ignored(infile, relroot))
9194

9295

93-
def _iter_ignored(infile):
96+
def _iter_ignored(infile, relroot):
97+
if relroot and relroot is not fsutil.USE_CWD:
98+
relroot = os.path.abspath(relroot)
99+
bogus = {_tables.EMPTY, _tables.UNKNOWN}
94100
for row in _tables.read_table(infile, IGNORED_HEADER, sep='\t'):
95101
*varidinfo, reason = row
102+
if _tables.EMPTY in varidinfo or _tables.UNKNOWN in varidinfo:
103+
varidinfo = tuple(None if v in bogus else v
104+
for v in varidinfo)
105+
if reason in bogus:
106+
reason = None
96107
varid = _info.DeclID.from_row(varidinfo)
108+
varid = varid.fix_filename(relroot, formatted=False, fixroot=False)
97109
yield varid, reason
98110

99111

100-
def write_ignored(variables, outfile):
112+
def write_ignored(variables, outfile, relroot=fsutil.USE_CWD):
101113
raise NotImplementedError
114+
if relroot and relroot is not fsutil.USE_CWD:
115+
relroot = os.path.abspath(relroot)
102116
reason = '???'
103117
#if not isinstance(varid, DeclID):
104118
# varid = getattr(varid, 'parsed', varid).id
119+
decls = (d.fix_filename(relroot, fixroot=False) for d in decls)
105120
_tables.write_table(
106121
outfile,
107122
IGNORED_HEADER,

‎Tools/c-analyzer/c_analyzer/info.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
from collections import namedtuple
2+
import os.path
23

4+
from c_common import fsutil
35
from c_common.clsutil import classonly
46
import c_common.misc as _misc
57
from c_parser.info import (
@@ -223,8 +225,9 @@ def is_known(self):
223225
else:
224226
return UNKNOWN not in self.typedecl
225227

226-
def fix_filename(self, relroot):
227-
self.item.fix_filename(relroot)
228+
def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
229+
self.item.fix_filename(relroot, **kwargs)
230+
return self
228231

229232
def as_rowdata(self, columns=None):
230233
# XXX finsih!
@@ -309,9 +312,11 @@ def __getitem__(self, key):
309312
else:
310313
return self._analyzed[key]
311314

312-
def fix_filenames(self, relroot):
315+
def fix_filenames(self, relroot=fsutil.USE_CWD, **kwargs):
316+
if relroot and relroot is not fsutil.USE_CWD:
317+
relroot = os.path.abspath(relroot)
313318
for item in self._analyzed:
314-
item.fix_filename(relroot)
319+
item.fix_filename(relroot, fixroot=False, **kwargs)
315320

316321
def _add_result(self, info, resolved):
317322
analyzed = type(self).build_item(info, resolved)

‎Tools/c-analyzer/c_common/fsutil.py

Lines changed: 95 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88
from .iterutil import iter_many
99

1010

11+
USE_CWD = object()
12+
13+
1114
C_SOURCE_SUFFIXES = ('.c', '.h')
1215

1316

@@ -29,6 +32,78 @@ def create_backup(old, backup=None):
2932
return backup
3033

3134

35+
##################################
36+
# filenames
37+
38+
def fix_filename(filename, relroot=USE_CWD, *,
39+
fixroot=True,
40+
_badprefix=f'..{os.path.sep}',
41+
):
42+
"""Return a normalized, absolute-path copy of the given filename."""
43+
if not relroot or relroot is USE_CWD:
44+
return os.path.abspath(filename)
45+
if fixroot:
46+
relroot = os.path.abspath(relroot)
47+
return _fix_filename(filename, relroot)
48+
49+
50+
def _fix_filename(filename, relroot, *,
51+
_badprefix=f'..{os.path.sep}',
52+
):
53+
orig = filename
54+
55+
# First we normalize.
56+
filename = os.path.normpath(filename)
57+
if filename.startswith(_badprefix):
58+
raise ValueError(f'bad filename {orig!r} (resolves beyond relative root')
59+
60+
# Now make sure it is absolute (relative to relroot).
61+
if not os.path.isabs(filename):
62+
filename = os.path.join(relroot, filename)
63+
else:
64+
relpath = os.path.relpath(filename, relroot)
65+
if os.path.join(relroot, relpath) != filename:
66+
raise ValueError(f'expected {relroot!r} as lroot, got {orig!r}')
67+
68+
return filename
69+
70+
71+
def fix_filenames(filenames, relroot=USE_CWD):
72+
if not relroot or relroot is USE_CWD:
73+
filenames = (os.path.abspath(v) for v in filenames)
74+
else:
75+
relroot = os.path.abspath(relroot)
76+
filenames = (_fix_filename(v, relroot) for v in filenames)
77+
return filenames, relroot
78+
79+
80+
def format_filename(filename, relroot=USE_CWD, *,
81+
fixroot=True,
82+
normalize=True,
83+
_badprefix=f'..{os.path.sep}',
84+
):
85+
"""Return a consistent relative-path representation of the filename."""
86+
orig = filename
87+
if normalize:
88+
filename = os.path.normpath(filename)
89+
if relroot is None:
90+
# Otherwise leave it as-is.
91+
return filename
92+
elif relroot is USE_CWD:
93+
# Make it relative to CWD.
94+
filename = os.path.relpath(filename)
95+
else:
96+
# Make it relative to "relroot".
97+
if fixroot:
98+
relroot = os.path.abspath(relroot)
99+
elif not relroot:
100+
raise ValueError('missing relroot')
101+
filename = os.path.relpath(filename, relroot)
102+
if filename.startswith(_badprefix):
103+
raise ValueError(f'bad filename {orig!r} (resolves beyond relative root')
104+
return filename
105+
106+
32107
##################################
33108
# find files
34109

@@ -54,34 +129,29 @@ def match_glob(filename, pattern):
54129
return fnmatch.fnmatch(filename, pattern.replace('**/', '', 1))
55130

56131

57-
def iter_filenames(filenames, *,
58-
start=None,
59-
include=None,
60-
exclude=None,
61-
):
132+
def process_filenames(filenames, *,
133+
start=None,
134+
include=None,
135+
exclude=None,
136+
relroot=USE_CWD,
137+
):
138+
if relroot and relroot is not USE_CWD:
139+
relroot = os.path.abspath(relroot)
140+
if start:
141+
start = fix_filename(start, relroot, fixroot=False)
142+
if include:
143+
include = set(fix_filename(v, relroot, fixroot=False)
144+
for v in include)
145+
if exclude:
146+
exclude = set(fix_filename(v, relroot, fixroot=False)
147+
for v in exclude)
148+
62149
onempty = Exception('no filenames provided')
63150
for filename, solo in iter_many(filenames, onempty):
151+
filename = fix_filename(filename, relroot, fixroot=False)
152+
relfile = format_filename(filename, relroot, fixroot=False, normalize=False)
64153
check, start = _get_check(filename, start, include, exclude)
65-
yield filename, check, solo
66-
# filenames = iter(filenames or ())
67-
# try:
68-
# first = next(filenames)
69-
# except StopIteration:
70-
# raise Exception('no filenames provided')
71-
# try:
72-
# second = next(filenames)
73-
# except StopIteration:
74-
# check, _ = _get_check(first, start, include, exclude)
75-
# yield first, check, False
76-
# return
77-
#
78-
# check, start = _get_check(first, start, include, exclude)
79-
# yield first, check, True
80-
# check, start = _get_check(second, start, include, exclude)
81-
# yield second, check, True
82-
# for filename in filenames:
83-
# check, start = _get_check(filename, start, include, exclude)
84-
# yield filename, check, True
154+
yield filename, relfile, check, solo
85155

86156

87157
def expand_filenames(filenames):

0 commit comments

Comments
 (0)