Skip to content

Commit 2037913

Browse files
mdickinsonmdickinson
committed
Update Demo/parser directory; backport unparse fixes from py3k.
1 parent 4aa5f6f commit 2037913

File tree

5 files changed

+297
-170
lines changed

5 files changed

+297
-170
lines changed

‎Demo/parser/README

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,27 @@ It also contains examples for the AST parser.
66
Files:
77
------
88

9-
FILES -- list of files associated with the parser module.
9+
FILES -- list of files associated with the parser module.
1010

11-
README -- this file.
11+
README -- this file.
1212

13-
example.py -- module that uses the `parser' module to extract
14-
information from the parse tree of Python source
15-
code.
13+
docstring.py -- sample source file containing only a module docstring.
1614

17-
docstring.py -- sample source file containing only a module docstring.
15+
example.py -- module that uses the `parser' module to extract
16+
information from the parse tree of Python source
17+
code.
1818

19-
simple.py -- sample source containing a "short form" definition.
19+
simple.py -- sample source containing a "short form" definition.
2020

21-
source.py -- sample source code used to demonstrate ability to
22-
handle nested constructs easily using the functions
23-
and classes in example.py.
21+
source.py -- sample source code used to demonstrate ability to
22+
handle nested constructs easily using the functions
23+
and classes in example.py.
2424

25-
test_parser.py program to put the parser module through its paces.
25+
test_parser.py program to put the parser module through its paces.
2626

27-
unparse.py AST (2.5) based example to recreate source code
28-
from an AST. This is incomplete; contributions
29-
are welcome.
27+
test_unparse.py tests for the unparse module
28+
29+
unparse.py AST (2.7) based example to recreate source code
30+
from an AST.
3031

3132
Enjoy!

‎Demo/parser/test_parser.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,19 @@ def testChunk(t, fileName):
1111
global _numFailed
1212
print '----', fileName,
1313
try:
14-
ast = parser.suite(t)
15-
tup = parser.ast2tuple(ast)
16-
# this discards the first AST; a huge memory savings when running
14+
st = parser.suite(t)
15+
tup = parser.st2tuple(st)
16+
# this discards the first ST; a huge memory savings when running
1717
# against a large source file like Tkinter.py.
18-
ast = None
19-
new = parser.tuple2ast(tup)
18+
st = None
19+
new = parser.tuple2st(tup)
2020
except parser.ParserError, err:
2121
print
2222
print 'parser module raised exception on input file', fileName + ':'
2323
traceback.print_exc()
2424
_numFailed = _numFailed + 1
2525
else:
26-
if tup != parser.ast2tuple(new):
26+
if tup != parser.st2tuple(new):
2727
print
2828
print 'parser module failed on input file', fileName
2929
_numFailed = _numFailed + 1

‎Demo/parser/test_unparse.py

Lines changed: 160 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,20 @@
11
import unittest
22
from test import test_support
3-
43
import cStringIO
4+
import sys
5+
import os
6+
import tokenize
57
import ast
6-
import _ast
78
import unparse
89

9-
forelse = """\
10+
def read_pyfile(filename):
11+
"""Read and return the contents of a Python source file (as a
12+
string), taking into account the file encoding."""
13+
with open(filename, "r") as pyfile:
14+
source = pyfile.read()
15+
return source
16+
17+
for_else = """\
1018
def f():
1119
for x in range(10):
1220
break
@@ -15,7 +23,7 @@ def f():
1523
z = 3
1624
"""
1725

18-
whileelse = """\
26+
while_else = """\
1927
def g():
2028
while True:
2129
break
@@ -24,16 +32,63 @@ def g():
2432
z = 3
2533
"""
2634

27-
class UnparseTestCase(unittest.TestCase):
28-
# Tests for specific bugs found in earlier versions of unparse
35+
relative_import = """\
36+
from . import fred
37+
from .. import barney
38+
from .australia import shrimp as prawns
39+
"""
40+
41+
class_decorator = """\
42+
@f1(arg)
43+
@f2
44+
class Foo: pass
45+
"""
46+
47+
elif1 = """\
48+
if cond1:
49+
suite1
50+
elif cond2:
51+
suite2
52+
else:
53+
suite3
54+
"""
55+
56+
elif2 = """\
57+
if cond1:
58+
suite1
59+
elif cond2:
60+
suite2
61+
"""
62+
63+
try_except_finally = """\
64+
try:
65+
suite1
66+
except ex1:
67+
suite2
68+
except ex2:
69+
suite3
70+
else:
71+
suite4
72+
finally:
73+
suite5
74+
"""
75+
76+
class ASTTestCase(unittest.TestCase):
77+
def assertASTEqual(self, ast1, ast2):
78+
dump1 = ast.dump(ast1)
79+
dump2 = ast.dump(ast2)
80+
self.assertEqual(ast.dump(ast1), ast.dump(ast2))
2981

3082
def check_roundtrip(self, code1, filename="internal"):
31-
ast1 = compile(code1, filename, "exec", _ast.PyCF_ONLY_AST)
83+
ast1 = compile(code1, filename, "exec", ast.PyCF_ONLY_AST)
3284
unparse_buffer = cStringIO.StringIO()
3385
unparse.Unparser(ast1, unparse_buffer)
3486
code2 = unparse_buffer.getvalue()
35-
ast2 = compile(code2, filename, "exec", _ast.PyCF_ONLY_AST)
36-
self.assertEqual(ast.dump(ast1), ast.dump(ast2))
87+
ast2 = compile(code2, filename, "exec", ast.PyCF_ONLY_AST)
88+
self.assertASTEqual(ast1, ast2)
89+
90+
class UnparseTestCase(ASTTestCase):
91+
# Tests for specific bugs found in earlier versions of unparse
3792

3893
def test_del_statement(self):
3994
self.check_roundtrip("del x, y, z")
@@ -43,23 +98,116 @@ def test_shifts(self):
4398
self.check_roundtrip("13 >> 7")
4499

45100
def test_for_else(self):
46-
self.check_roundtrip(forelse)
101+
self.check_roundtrip(for_else)
47102

48103
def test_while_else(self):
49-
self.check_roundtrip(whileelse)
104+
self.check_roundtrip(while_else)
50105

51106
def test_unary_parens(self):
52107
self.check_roundtrip("(-1)**7")
108+
self.check_roundtrip("(-1.)**8")
109+
self.check_roundtrip("(-1j)**6")
53110
self.check_roundtrip("not True or False")
54111
self.check_roundtrip("True or not False")
55112

113+
def test_integer_parens(self):
114+
self.check_roundtrip("3 .__abs__()")
115+
116+
def test_huge_float(self):
117+
self.check_roundtrip("1e1000")
118+
self.check_roundtrip("-1e1000")
119+
self.check_roundtrip("1e1000j")
120+
self.check_roundtrip("-1e1000j")
121+
122+
def test_min_int(self):
123+
self.check_roundtrip(str(-sys.maxint-1))
124+
self.check_roundtrip("-(%s)" % (sys.maxint + 1))
125+
126+
def test_imaginary_literals(self):
127+
self.check_roundtrip("7j")
128+
self.check_roundtrip("-7j")
129+
self.check_roundtrip("-(7j)")
130+
self.check_roundtrip("0j")
131+
self.check_roundtrip("-0j")
132+
self.check_roundtrip("-(0j)")
133+
134+
def test_negative_zero(self):
135+
self.check_roundtrip("-0")
136+
self.check_roundtrip("-(0)")
137+
self.check_roundtrip("-0b0")
138+
self.check_roundtrip("-(0b0)")
139+
self.check_roundtrip("-0o0")
140+
self.check_roundtrip("-(0o0)")
141+
self.check_roundtrip("-0x0")
142+
self.check_roundtrip("-(0x0)")
143+
144+
def test_lambda_parentheses(self):
145+
self.check_roundtrip("(lambda: int)()")
146+
56147
def test_chained_comparisons(self):
57148
self.check_roundtrip("1 < 4 <= 5")
58149
self.check_roundtrip("a is b is c is not d")
59150

151+
def test_function_arguments(self):
152+
self.check_roundtrip("def f(): pass")
153+
self.check_roundtrip("def f(a): pass")
154+
self.check_roundtrip("def f(b = 2): pass")
155+
self.check_roundtrip("def f(a, b): pass")
156+
self.check_roundtrip("def f(a, b = 2): pass")
157+
self.check_roundtrip("def f(a = 5, b = 2): pass")
158+
self.check_roundtrip("def f(*args, **kwargs): pass")
159+
160+
def test_relative_import(self):
161+
self.check_roundtrip(relative_import)
162+
163+
def test_bytes(self):
164+
self.check_roundtrip("b'123'")
165+
166+
def test_set_literal(self):
167+
self.check_roundtrip("{'a', 'b', 'c'}")
168+
169+
def test_set_comprehension(self):
170+
self.check_roundtrip("{x for x in range(5)}")
171+
172+
def test_dict_comprehension(self):
173+
self.check_roundtrip("{x: x*x for x in range(10)}")
174+
175+
def test_class_decorators(self):
176+
self.check_roundtrip(class_decorator)
177+
178+
def test_elifs(self):
179+
self.check_roundtrip(elif1)
180+
self.check_roundtrip(elif2)
181+
182+
def test_try_except_finally(self):
183+
self.check_roundtrip(try_except_finally)
184+
185+
class DirectoryTestCase(ASTTestCase):
186+
"""Test roundtrip behaviour on all files in Lib and Lib/test."""
187+
188+
# test directories, relative to the root of the distribution
189+
test_directories = 'Lib', os.path.join('Lib', 'test')
190+
191+
def test_files(self):
192+
# get names of files to test
193+
dist_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
194+
195+
names = []
196+
for d in self.test_directories:
197+
test_dir = os.path.join(dist_dir, d)
198+
for n in os.listdir(test_dir):
199+
if n.endswith('.py') and not n.startswith('bad'):
200+
names.append(os.path.join(test_dir, n))
201+
202+
for filename in names:
203+
if test_support.verbose:
204+
print('Testing %s' % filename)
205+
source = read_pyfile(filename)
206+
self.check_roundtrip(source)
207+
60208

61209
def test_main():
62-
test_support.run_unittest(UnparseTestCase)
210+
test_support.run_unittest(UnparseTestCase, DirectoryTestCase)
63211

64212
if __name__ == '__main__':
65213
test_main()

‎Demo/parser/texipre.dat

Lines changed: 0 additions & 100 deletions
This file was deleted.

0 commit comments

Comments
 (0)