From 84ce7c408b6df3989bf746592d4e1064f5ea1d82 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Mon, 22 Apr 2019 21:07:39 +0200 Subject: [PATCH 1/2] Temporary commit message --- Tools/scripts/duplicate_meth_defs.py | 255 +++++++++++++++++++++++++++ Tools/scripts/duplicates_ignored.txt | 46 +++++ 2 files changed, 301 insertions(+) create mode 100755 Tools/scripts/duplicate_meth_defs.py create mode 100644 Tools/scripts/duplicates_ignored.txt diff --git a/Tools/scripts/duplicate_meth_defs.py b/Tools/scripts/duplicate_meth_defs.py new file mode 100755 index 00000000000000..c29d9a599188e1 --- /dev/null +++ b/Tools/scripts/duplicate_meth_defs.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python2 +"""Print duplicate method definitions in the same scope.""" + +from __future__ import print_function + +import sys +import os +import ast +import re +import argparse +import traceback +from contextlib import contextmanager + +class ClassFunctionVisitor(ast.NodeVisitor): + def __init__(self, duplicates): + self.duplicates = duplicates + self.scopes = [] + + @contextmanager + def _visit_new_scope(self, node): + prev_scope = self.scopes[-1] if len(self.scopes) else None + new_scope = Scope(node, prev_scope) + try: + yield prev_scope, new_scope + finally: + self.scopes.append(new_scope) + self.generic_visit(node) + self.scopes.pop() + + def visit_FunctionDef(self, node): + with self._visit_new_scope(node) as (previous, new): + if previous is not None and previous.type == 'ClassDef': + new.check_duplicate_meth(self.duplicates) + + visit_AsyncFunctionDef = visit_FunctionDef + + def visit_ClassDef(self, node): + with self._visit_new_scope(node) as (previous, new): + pass + +class Scope: + def __init__(self, node, previous): + self.node = node + self.previous = previous + self.type = node.__class__.__name__ + self.methods = [] + if previous is None: + self.id = [node.name] + else: + self.id = previous.id + [node.name] + + def check_duplicate_meth(self, duplicates): + node = self.node + name = node.name + # Skip property methods. + if 'decorator_list' in node._fields and node.decorator_list: + for decorator in node.decorator_list: + if (isinstance(decorator, ast.Attribute) and + 'attr' in decorator._fields and + decorator.attr in + ('setter', 'getter', 'deleter')): + return + + if name in self.previous.methods: + duplicates.append((self.id, node.lineno)) + else: + self.previous.methods.append(name) + +def get_duplicates(source, fname): + duplicates = [] + tree = ast.parse(source, fname) + ClassFunctionVisitor(duplicates).visit(tree) + return duplicates + +def duplicates_exist(source, fname='', dups_to_ignore=[]): + """Print duplicates and return True if duplicates exist. + + >>> test_basic = ''' + ... class C: + ... def foo(self): pass + ... def foo(self): pass + ... ''' + >>> duplicates_exist(test_basic) + :4 C.foo + True + + >>> test_ignore = ''' + ... class C: + ... def foo(self): pass + ... def foo(self): pass + ... ''' + >>> duplicates_exist(test_ignore, dups_to_ignore=['C.foo']) + False + + >>> test_nested = ''' + ... def foo(): + ... def bar(): pass + ... + ... class C: + ... def foo(self): pass + ... def bar(self): pass + ... def bar(self): pass + ... + ... class D: + ... def foo(self): pass + ... def bar(self): pass + ... def bar(self): pass + ... + ... def bar(self): pass + ... + ... def bar(): pass + ... ''' + >>> duplicates_exist(test_nested) + :8 foo.C.bar + :13 foo.C.D.bar + :15 foo.C.bar + True + + >>> test_properties = ''' + ... class C: + ... @property + ... def foo(self): pass + ... + ... @foo.setter + ... def foo(self, a, b): pass + ... + ... def bar(self): pass + ... def bar(self): pass + ... + ... class D: + ... @property + ... def foo(self): pass + ... + ... @foo.setter + ... def foo(self, a, b): pass + ... + ... def bar(self): pass + ... def bar(self): pass + ... def foo(self): pass + ... ''' + >>> duplicates_exist(test_properties) + :10 C.bar + :20 C.D.bar + :21 C.D.foo + True + """ + + cnt = 0 + duplicates = get_duplicates(source, fname) + if duplicates: + for (id, lineno) in duplicates: + id = '.'.join(id) + if dups_to_ignore is None or id not in dups_to_ignore: + print('%s:%d %s' % (fname, lineno, id)) + cnt += 1 + return False if cnt == 0 else True + +def iter_modules(paths): + for f in paths: + if os.path.isdir(f): + for path, dirs, filenames in os.walk(f): + for fname in filenames: + if os.path.splitext(fname)[1] == '.py': + yield os.path.normpath(os.path.join(path, fname)) + else: + yield os.path.normpath(f) + +def ignored_dups(f): + """Parse the ignore duplicates file.""" + + # Comments or empty lines. + comments = re.compile(r'^\s*#.*$|^\s*$') + + # Mapping of filename to the list of duplicates to ignore. + ignored = {} + if f is None: + return ignored + with f: + for item in (l.rstrip(': \n\t').split(':') for l in f + if not comments.match(l)): + path = item[0].strip() + if not path: + continue + path = os.path.normpath(path) + if len(item) == 1: + ignored[path] = [] + else: + dupname = item[1].strip() + if path in ignored: + ignored[path].append(dupname) + else: + ignored[path] = [dupname] + return ignored + +def parse_args(argv): + parser = argparse.ArgumentParser(description=__doc__.strip()) + parser.add_argument('files', metavar='path', nargs='*', + help='python module or directory pathname - when a directory, ' + 'the python modules in the directory tree are searched ' + 'recursively for duplicates.') + parser.add_argument('-i', '--ignore', metavar='fname', type=open, + help='ignore duplicates or files listed in , see' + ' Tools/scripts/duplicates_ignored.txt for the' + ' specification of the format of the entries in this file') + parser.add_argument('-x', '--exclude', metavar='prefixes', + help="comma separated list of path names prefixes to exclude" + " (default: '%(default)s')") + parser.add_argument('-t', '--test', action='store_true', + help='run the doctests') + return parser.parse_args(argv) + +def _main(args): + ignored = ignored_dups(args.ignore) + if args.exclude: + prefixes = [p.strip() for p in args.exclude.split(',')] + else: + prefixes = [] + + # Find duplicates. + rc = 0 + for fname in iter_modules(args.files): + def excluded(f): + for p in prefixes: + if f.startswith(p): + return True + + # Skip files whose prefix is excluded or that are configured in the + # '--ignore' file. + if excluded(fname): + continue + dups_to_ignore = ignored.get(fname) + if dups_to_ignore == []: + continue + + try: + with open(fname) as f: + if duplicates_exist(f.read(), fname, dups_to_ignore): + rc = 1 + except (UnicodeDecodeError, SyntaxError) as e: + print('%s: %s' % (fname, e), file=sys.stderr) + traceback.print_tb(sys.exc_info()[2]) + rc = 1 + + return rc + +def main(argv): + args = parse_args(argv) + if args.test: + import doctest + doctest.testmod() + else: + return _main(args) + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/Tools/scripts/duplicates_ignored.txt b/Tools/scripts/duplicates_ignored.txt new file mode 100644 index 00000000000000..3c08a7def28440 --- /dev/null +++ b/Tools/scripts/duplicates_ignored.txt @@ -0,0 +1,46 @@ +# Files or duplicates ignored by Tools/scripts/duplicate_meth_defs.py +# when run with the '--ignore' command line option. +# +# Format: +# filename[: mfqn] +# +# The whole file is ignored when mfqn (method fully qualified name) is +# missing. Use instead the '--exclude' command line option of +# duplicate_meth_defs.py to ignore a directory tree. + +# Files with bad encoding or bad syntax. +Lib/lib2to3/tests/data/py3_test_grammar.py +Lib/test/bad_coding.py +Lib/test/bad_coding2.py +Lib/test/bad_coding3.py +Lib/test/badsyntax_nocaret.py +PCbuild/get_external.py + +# False positives. +Demo/rpc/xdr.py: Packer.pack_uint +Demo/rpc/xdr.py: Unpacker.unpack_uint +Lib/ctypes/__init__.py: c_char_p.__repr__ +Lib/idlelib/ScriptBinding.py: ScriptBinding.run_module_event +Lib/os.py: _Environ.__delitem__ +Lib/pickle.py: Pickler.save_string +Lib/subprocess.py: Popen._get_handles +Lib/subprocess.py: Popen._execute_child +Lib/subprocess.py: Popen._internal_poll +Lib/subprocess.py: Popen.wait +Lib/subprocess.py: Popen._communicate +Lib/subprocess.py: Popen.send_signal +Lib/subprocess.py: Popen.terminate +Lib/tempfile.py: _TemporaryFileWrapper.__exit__ +Lib/test/test_bz2.py: BaseTest.decompress + + +# Duplicates that must be fixed and that are temporarily ignored. +# Issue bpo-19113 +Lib/ctypes/test/test_functions.py: FunctionTestCase.test_errors + +# Issue bpo-XXX +Lib/email/feedparser.py: BufferedSubFile.pushlines +Lib/email/test/test_email_renamed.py: TestEncoders.test_default_cte + +# Issue bpo-XXX +Lib/ctypes/test/test_unicode.py: StringTestCase.test_ascii_replace From 8e5fe2806cd728307b38e074b4accf2b757726d2 Mon Sep 17 00:00:00 2001 From: Xavier de Gaye Date: Wed, 24 Apr 2019 17:37:55 +0200 Subject: [PATCH 2/2] Add the `duplicate_meth_defs.py` tool to report duplicate method definitions --- .../Tools-Demos/2019-04-24-17-37-06.bpo-16079.NJAUsv.rst | 2 ++ Tools/scripts/duplicates_ignored.txt | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2019-04-24-17-37-06.bpo-16079.NJAUsv.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2019-04-24-17-37-06.bpo-16079.NJAUsv.rst b/Misc/NEWS.d/next/Tools-Demos/2019-04-24-17-37-06.bpo-16079.NJAUsv.rst new file mode 100644 index 00000000000000..c9c08a69f4e943 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2019-04-24-17-37-06.bpo-16079.NJAUsv.rst @@ -0,0 +1,2 @@ +Add the `duplicate_meth_defs.py` tool to report duplicate method definitions +in a given list of files or directories. diff --git a/Tools/scripts/duplicates_ignored.txt b/Tools/scripts/duplicates_ignored.txt index 3c08a7def28440..63b9d7b87485a8 100644 --- a/Tools/scripts/duplicates_ignored.txt +++ b/Tools/scripts/duplicates_ignored.txt @@ -38,9 +38,11 @@ Lib/test/test_bz2.py: BaseTest.decompress # Issue bpo-19113 Lib/ctypes/test/test_functions.py: FunctionTestCase.test_errors -# Issue bpo-XXX +# Issue bpo-36711 Lib/email/feedparser.py: BufferedSubFile.pushlines + +# Issue bpo-36712 Lib/email/test/test_email_renamed.py: TestEncoders.test_default_cte -# Issue bpo-XXX +# Issue bpo-36713 Lib/ctypes/test/test_unicode.py: StringTestCase.test_ascii_replace