Skip to content

[2.7] bpo-16079: Add the duplicate_meth_defs.py tool (GH-12886) #12940

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

[2.7] bpo-16079: Add the duplicate_meth_defs.py tool (GH-12886) #12940

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
84ce7c4
Temporary commit message
xdegaye Apr 22, 2019
8e5fe28
Add the `duplicate_meth_defs.py` tool to report duplicate method defi…
xdegaye Apr 24, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Tools-Demos/2019-04-24-17-37-06.bpo-16079.NJAUsv.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Add the `duplicate_meth_defs.py` tool to report duplicate method definitions
in a given list of files or directories.
255 changes: 255 additions & 0 deletions Tools/scripts/duplicate_meth_defs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,255 @@
#!/usr/bin/env python2
"""Print duplicate method definitions in the same scope."""

from __future__ import print_function

import sys
import os
import ast
import re
import argparse
import traceback
from contextlib import contextmanager

class ClassFunctionVisitor(ast.NodeVisitor):
def __init__(self, duplicates):
self.duplicates = duplicates
self.scopes = []

@contextmanager
def _visit_new_scope(self, node):
prev_scope = self.scopes[-1] if len(self.scopes) else None
new_scope = Scope(node, prev_scope)
try:
yield prev_scope, new_scope
finally:
self.scopes.append(new_scope)
self.generic_visit(node)
self.scopes.pop()

def visit_FunctionDef(self, node):
with self._visit_new_scope(node) as (previous, new):
if previous is not None and previous.type == 'ClassDef':
new.check_duplicate_meth(self.duplicates)

visit_AsyncFunctionDef = visit_FunctionDef

def visit_ClassDef(self, node):
with self._visit_new_scope(node) as (previous, new):
pass

class Scope:
def __init__(self, node, previous):
self.node = node
self.previous = previous
self.type = node.__class__.__name__
self.methods = []
if previous is None:
self.id = [node.name]
else:
self.id = previous.id + [node.name]

def check_duplicate_meth(self, duplicates):
node = self.node
name = node.name
# Skip property methods.
if 'decorator_list' in node._fields and node.decorator_list:
for decorator in node.decorator_list:
if (isinstance(decorator, ast.Attribute) and
'attr' in decorator._fields and
decorator.attr in
('setter', 'getter', 'deleter')):
return

if name in self.previous.methods:
duplicates.append((self.id, node.lineno))
else:
self.previous.methods.append(name)

def get_duplicates(source, fname):
duplicates = []
tree = ast.parse(source, fname)
ClassFunctionVisitor(duplicates).visit(tree)
return duplicates

def duplicates_exist(source, fname='<unknown>', dups_to_ignore=[]):
"""Print duplicates and return True if duplicates exist.

>>> test_basic = '''
... class C:
... def foo(self): pass
... def foo(self): pass
... '''
>>> duplicates_exist(test_basic)
<unknown>:4 C.foo
True

>>> test_ignore = '''
... class C:
... def foo(self): pass
... def foo(self): pass
... '''
>>> duplicates_exist(test_ignore, dups_to_ignore=['C.foo'])
False

>>> test_nested = '''
... def foo():
... def bar(): pass
...
... class C:
... def foo(self): pass
... def bar(self): pass
... def bar(self): pass
...
... class D:
... def foo(self): pass
... def bar(self): pass
... def bar(self): pass
...
... def bar(self): pass
...
... def bar(): pass
... '''
>>> duplicates_exist(test_nested)
<unknown>:8 foo.C.bar
<unknown>:13 foo.C.D.bar
<unknown>:15 foo.C.bar
True

>>> test_properties = '''
... class C:
... @property
... def foo(self): pass
...
... @foo.setter
... def foo(self, a, b): pass
...
... def bar(self): pass
... def bar(self): pass
...
... class D:
... @property
... def foo(self): pass
...
... @foo.setter
... def foo(self, a, b): pass
...
... def bar(self): pass
... def bar(self): pass
... def foo(self): pass
... '''
>>> duplicates_exist(test_properties)
<unknown>:10 C.bar
<unknown>:20 C.D.bar
<unknown>:21 C.D.foo
True
"""

cnt = 0
duplicates = get_duplicates(source, fname)
if duplicates:
for (id, lineno) in duplicates:
id = '.'.join(id)
if dups_to_ignore is None or id not in dups_to_ignore:
print('%s:%d %s' % (fname, lineno, id))
cnt += 1
return False if cnt == 0 else True

def iter_modules(paths):
for f in paths:
if os.path.isdir(f):
for path, dirs, filenames in os.walk(f):
for fname in filenames:
if os.path.splitext(fname)[1] == '.py':
yield os.path.normpath(os.path.join(path, fname))
else:
yield os.path.normpath(f)

def ignored_dups(f):
"""Parse the ignore duplicates file."""

# Comments or empty lines.
comments = re.compile(r'^\s*#.*$|^\s*$')

# Mapping of filename to the list of duplicates to ignore.
ignored = {}
if f is None:
return ignored
with f:
for item in (l.rstrip(': \n\t').split(':') for l in f
if not comments.match(l)):
path = item[0].strip()
if not path:
continue
path = os.path.normpath(path)
if len(item) == 1:
ignored[path] = []
else:
dupname = item[1].strip()
if path in ignored:
ignored[path].append(dupname)
else:
ignored[path] = [dupname]
return ignored

def parse_args(argv):
parser = argparse.ArgumentParser(description=__doc__.strip())
parser.add_argument('files', metavar='path', nargs='*',
help='python module or directory pathname - when a directory, '
'the python modules in the directory tree are searched '
'recursively for duplicates.')
parser.add_argument('-i', '--ignore', metavar='fname', type=open,
help='ignore duplicates or files listed in <fname>, see'
' Tools/scripts/duplicates_ignored.txt for the'
' specification of the format of the entries in this file')
parser.add_argument('-x', '--exclude', metavar='prefixes',
help="comma separated list of path names prefixes to exclude"
" (default: '%(default)s')")
parser.add_argument('-t', '--test', action='store_true',
help='run the doctests')
return parser.parse_args(argv)

def _main(args):
ignored = ignored_dups(args.ignore)
if args.exclude:
prefixes = [p.strip() for p in args.exclude.split(',')]
else:
prefixes = []

# Find duplicates.
rc = 0
for fname in iter_modules(args.files):
def excluded(f):
for p in prefixes:
if f.startswith(p):
return True

# Skip files whose prefix is excluded or that are configured in the
# '--ignore' file.
if excluded(fname):
continue
dups_to_ignore = ignored.get(fname)
if dups_to_ignore == []:
continue

try:
with open(fname) as f:
if duplicates_exist(f.read(), fname, dups_to_ignore):
rc = 1
except (UnicodeDecodeError, SyntaxError) as e:
print('%s: %s' % (fname, e), file=sys.stderr)
traceback.print_tb(sys.exc_info()[2])
rc = 1

return rc

def main(argv):
args = parse_args(argv)
if args.test:
import doctest
doctest.testmod()
else:
return _main(args)

if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))
48 changes: 48 additions & 0 deletions Tools/scripts/duplicates_ignored.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# Files or duplicates ignored by Tools/scripts/duplicate_meth_defs.py
# when run with the '--ignore' command line option.
#
# Format:
# filename[: mfqn]
#
# The whole file is ignored when mfqn (method fully qualified name) is
# missing. Use instead the '--exclude' command line option of
# duplicate_meth_defs.py to ignore a directory tree.

# Files with bad encoding or bad syntax.
Lib/lib2to3/tests/data/py3_test_grammar.py
Lib/test/bad_coding.py
Lib/test/bad_coding2.py
Lib/test/bad_coding3.py
Lib/test/badsyntax_nocaret.py
PCbuild/get_external.py

# False positives.
Demo/rpc/xdr.py: Packer.pack_uint
Demo/rpc/xdr.py: Unpacker.unpack_uint
Lib/ctypes/__init__.py: c_char_p.__repr__
Lib/idlelib/ScriptBinding.py: ScriptBinding.run_module_event
Lib/os.py: _Environ.__delitem__
Lib/pickle.py: Pickler.save_string
Lib/subprocess.py: Popen._get_handles
Lib/subprocess.py: Popen._execute_child
Lib/subprocess.py: Popen._internal_poll
Lib/subprocess.py: Popen.wait
Lib/subprocess.py: Popen._communicate
Lib/subprocess.py: Popen.send_signal
Lib/subprocess.py: Popen.terminate
Lib/tempfile.py: _TemporaryFileWrapper.__exit__
Lib/test/test_bz2.py: BaseTest.decompress


# Duplicates that must be fixed and that are temporarily ignored.
# Issue bpo-19113
Lib/ctypes/test/test_functions.py: FunctionTestCase.test_errors

# Issue bpo-36711
Lib/email/feedparser.py: BufferedSubFile.pushlines

# Issue bpo-36712
Lib/email/test/test_email_renamed.py: TestEncoders.test_default_cte

# Issue bpo-36713
Lib/ctypes/test/test_unicode.py: StringTestCase.test_ascii_replace