diff --git a/Tools/c-analyzer/c_parser/preprocessor/__init__.py b/Tools/c-analyzer/c_parser/preprocessor/__init__.py index 30a86cbd7dc494..f8d2f805cb1b19 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/__init__.py +++ b/Tools/c-analyzer/c_parser/preprocessor/__init__.py @@ -16,6 +16,7 @@ from . import ( pure as _pure, gcc as _gcc, + clang as _clang, ) @@ -234,7 +235,7 @@ def handling_errors(ignore_exc=None, *, log_err=None): 'bcpp': None, # aliases/extras: 'gcc': _gcc.preprocess, - 'clang': None, + 'clang': _clang.preprocess, } diff --git a/Tools/c-analyzer/c_parser/preprocessor/clang.py b/Tools/c-analyzer/c_parser/preprocessor/clang.py new file mode 100644 index 00000000000000..574a23f8f6d6f9 --- /dev/null +++ b/Tools/c-analyzer/c_parser/preprocessor/clang.py @@ -0,0 +1,104 @@ +import os.path +import re, sys + +from . import common as _common +from . import gcc as _gcc + +_normpath = _gcc._normpath + +TOOL = 'clang' + +META_FILES = { + '', + '', +} + + +def preprocess(filename, + incldirs=None, + includes=None, + macros=None, + samefiles=None, + cwd=None, + ): + if not cwd or not os.path.isabs(cwd): + cwd = os.path.abspath(cwd or '.') + filename = _normpath(filename, cwd) + + postargs = _gcc.POST_ARGS + basename = os.path.basename(filename) + dirname = os.path.basename(os.path.dirname(filename)) + if (basename not in _gcc.FILES_WITHOUT_INTERNAL_CAPI + and dirname not in _gcc.DIRS_WITHOUT_INTERNAL_CAPI): + postargs += ('-DPy_BUILD_CORE=1',) + + text = _common.preprocess( + TOOL, + filename, + incldirs=incldirs, + includes=includes, + macros=macros, + #preargs=PRE_ARGS, + postargs=postargs, + executable=['clang'], + compiler='unix', + cwd=cwd, + ) + return _iter_lines(text, filename, samefiles, cwd) + + +EXIT_MARKERS = {'# 2 "" 2', '# 3 "" 2', '# 4 "" 2'} + + +def _iter_lines(text, reqfile, samefiles, cwd, raw=False): + lines = iter(text.splitlines()) + + # The first line is special. + # The subsequent lines are consistent. + firstlines = [ + f'# 1 "{reqfile}"', + '# 1 "" 1', + '# 1 "" 3', + '# 370 "" 3', + '# 1 "" 1', + '# 1 "" 2', + ] + for expected in firstlines: + line = next(lines) + if line != expected: + raise NotImplementedError((line, expected)) + + # Do all the CLI-provided includes. + filter_reqfile = (lambda f: _gcc._filter_reqfile(f, reqfile, samefiles)) + make_info = (lambda lno: _common.FileInfo(reqfile, lno)) + last = None + for line in lines: + assert last != reqfile, (last,) + # NOTE:condition is clang specific + if not line: + continue + lno, included, flags = _gcc._parse_marker_line(line, reqfile) + if not included: + raise NotImplementedError((line,)) + if included == reqfile: + # This will be the last one. + assert 2 in flags, (line, flags) + else: + # NOTE:first condition is specific to clang + if _normpath(included, cwd) == reqfile: + assert 1 in flags or 2 in flags, (line, flags, included, reqfile) + else: + assert 1 in flags, (line, flags, included, reqfile) + yield from _gcc._iter_top_include_lines( + lines, + _normpath(included, cwd), + cwd, + filter_reqfile, + make_info, + raw, + EXIT_MARKERS + ) + last = included + # The last one is always the requested file. + # NOTE:_normpath is clang specific + assert _normpath(included, cwd) == reqfile, (line,) diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index d20cd19f6e6d5e..4a55a1a24ee1be 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -65,6 +65,8 @@ '-E', ) +EXIT_MARKERS = {'# 0 "" 2', '# 1 "" 2'} + def preprocess(filename, incldirs=None, @@ -138,6 +140,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False): filter_reqfile, make_info, raw, + EXIT_MARKERS ) last = included # The last one is always the requested file. @@ -146,7 +149,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False): def _iter_top_include_lines(lines, topfile, cwd, filter_reqfile, make_info, - raw): + raw, exit_markers): partial = 0 # depth files = [topfile] # We start at 1 in case there are source lines (including blank ones) @@ -154,12 +157,20 @@ def _iter_top_include_lines(lines, topfile, cwd, # _parse_marker_line() that the preprocessor reported lno as 1. lno = 1 for line in lines: - if line == '# 0 "" 2' or line == '# 1 "" 2': + if line in exit_markers: # We're done with this top-level include. return _lno, included, flags = _parse_marker_line(line) if included: + # HACK: + # Mixes curses.h and ncurses.h marker lines + # gcc silently passes this, while clang fails + # See: /Include/py_curses.h #if-elif directives + # And compare with preprocessor output + if os.path.basename(included) == 'curses.h': + included = os.path.join(os.path.dirname(included), 'ncurses.h') + lno = _lno included = _normpath(included, cwd) # We hit a marker line. diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 037fe11ea223c7..a2df613db46966 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -351,6 +351,10 @@ def clean_lines(text): # Catch-alls: _abs('Include/**/*.h'): (5_000, 500), + + # Specific to clang + _abs('Modules/selectmodule.c'): (40_000, 3000), + _abs('Modules/_testcapi/pyatomic.c'): (30_000, 1000), } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 3c3cb2f9c86f16..301784f773d31f 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -400,6 +400,7 @@ Modules/_tkinter.c - tcl_lock - Modules/_tkinter.c - excInCmd - Modules/_tkinter.c - valInCmd - Modules/_tkinter.c - trbInCmd - +Modules/socketmodule.c - netdb_lock - ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index a33619b1b345e2..cd3b8945709cba 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -16,6 +16,7 @@ filename funcname name reason ## indicators for resource availability/capability # (set during first init) Python/bootstrap_hash.c py_getrandom getrandom_works - +Python/bootstrap_hash.c py_getentropy getentropy_works - Python/fileutils.c - _Py_open_cloexec_works - Python/fileutils.c set_inheritable ioctl_works - # (set lazily, *after* first init)