Skip to content

Commit 927e70d

Browse files
ncwncw
committed
parser: fix reporting of errors and parsing of decimals with leading zeros
1 parent 60d12b3 commit 927e70d

8 files changed

+408
-318
lines changed

‎notes.txt

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ Limitations
99
* string keys only in dictionaries
1010
* ints only 64 bit
1111

12-
FIXME parsing 00007 works fine whereas it should be throwing an error
13-
1412
FIXME interesting crash with compiling `int("42"), sausage=11)`
1513
Compile error: SystemError: [interface conversion: *ast.Call is not ast.SetCtxer: missing method SetCtx]
1614

‎parser/grammar.y

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1335,7 +1335,7 @@ comp_op:
13351335
| LTGT
13361336
{
13371337
// panic("FIXME no coverage")
1338-
yylex.Error("Invalid syntax")
1338+
yylex.(*yyLex).SyntaxError("Invalid syntax")
13391339
}
13401340
| PLINGEQ
13411341
{
@@ -1495,14 +1495,14 @@ strings:
14951495
case py.String:
14961496
$$ = a + b
14971497
default:
1498-
yylex.Error("SyntaxError: cannot mix string and nonstring literals")
1498+
yylex.(*yyLex).SyntaxError("cannot mix string and nonstring literals")
14991499
}
15001500
case py.Bytes:
15011501
switch b := $2.(type) {
15021502
case py.Bytes:
15031503
$$ = append(a, b...)
15041504
default:
1505-
yylex.Error("SyntaxError: cannot mix bytes and nonbytes literals")
1505+
yylex.(*yyLex).SyntaxError("cannot mix bytes and nonbytes literals")
15061506
}
15071507
}
15081508
}
@@ -1823,7 +1823,7 @@ arglist:
18231823
call := $1
18241824
call.Starargs = $3
18251825
if len($4.Args) != 0 {
1826-
yylex.Error("SyntaxError: only named arguments may follow *expression")
1826+
yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
18271827
}
18281828
call.Keywords = append(call.Keywords, $4.Keywords...)
18291829
$$ = call
@@ -1834,7 +1834,7 @@ arglist:
18341834
call.Starargs = $3
18351835
call.Kwargs = $7
18361836
if len($4.Args) != 0 {
1837-
yylex.Error("SyntaxError: only named arguments may follow *expression")
1837+
yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
18381838
}
18391839
call.Keywords = append(call.Keywords, $4.Keywords...)
18401840
$$ = call
@@ -1868,7 +1868,7 @@ argument:
18681868
if name, ok := test.(*ast.Name); ok {
18691869
$$.Keywords = []*ast.Keyword{&ast.Keyword{Pos: name.Pos, Arg: name.Id, Value: $3}}
18701870
} else {
1871-
yylex.Error("SyntaxError: keyword can't be an expression")
1871+
yylex.(*yyLex).SyntaxError("keyword can't be an expression")
18721872
}
18731873
}
18741874

‎parser/grammar_data_test.go

Lines changed: 292 additions & 282 deletions
Large diffs are not rendered by default.

‎parser/grammar_test.go

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,44 @@ import (
77
"testing"
88

99
"github.com/ncw/gpython/ast"
10+
"github.com/ncw/gpython/py"
1011
)
1112

1213
var debugLevel = flag.Int("debugLevel", 0, "Debug level 0-4")
1314

1415
// FIXME test pos is correct
1516

16-
// FIXME add tests to test the error cases
17-
1817
func TestGrammar(t *testing.T) {
1918
SetDebug(*debugLevel)
2019
for _, test := range grammarTestData {
2120
Ast, err := ParseString(test.in, test.mode)
2221
if err != nil {
23-
t.Errorf("Parse(%q) returned error: %v", test.in, err)
22+
if test.exceptionType == nil {
23+
t.Errorf("%s: Got exception %v when not expecting one", test.in, err)
24+
return
25+
} else if exc, ok := err.(*py.Exception); !ok {
26+
t.Errorf("%s: Got non python exception %T %v", test.in, err, err)
27+
return
28+
} else if exc.Type() != test.exceptionType {
29+
t.Errorf("%s: want exception type %v got %v", test.in, test.exceptionType, exc.Type())
30+
return
31+
} else if exc.Type() != test.exceptionType {
32+
t.Errorf("%s: want exception type %v got %v", test.in, test.exceptionType, exc.Type())
33+
return
34+
} else {
35+
msg := string(exc.Args.(py.Tuple)[0].(py.String))
36+
if msg != test.errString {
37+
t.Errorf("%s: want exception text %q got %q", test.in, test.errString, msg)
38+
}
39+
}
2440
} else {
25-
out := ast.Dump(Ast)
26-
if out != test.out {
27-
t.Errorf("Parse(%q)\nwant> %q\n got> %q\n", test.in, test.out, out)
41+
if test.exceptionType != nil {
42+
t.Errorf("%s: expecting exception %q", test.in, test.errString)
43+
} else {
44+
out := ast.Dump(Ast)
45+
if out != test.out {
46+
t.Errorf("Parse(%q)\nwant> %q\n got> %q\n", test.in, test.out, out)
47+
}
2848
}
2949
}
3050
}

‎parser/lexer.go

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ func (x *yyLex) refill() {
9090
x.eof = true
9191
default:
9292
x.eof = true
93-
x.Errorf("Error reading input: %v", err)
93+
x.SyntaxErrorf("Error reading input: %v", err)
9494
}
9595
// If this is exec input, add a newline to the end of the
9696
// string if there isn't one already.
@@ -129,7 +129,7 @@ func countIndent(s string) int {
129129
// a b
130130
indent += tabSize - (indent & (tabSize - 1))
131131
default:
132-
panic("bad indent")
132+
panic(py.ExceptionNewf(py.IndentationError, "unexpected indent"))
133133
}
134134

135135
}
@@ -399,7 +399,7 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
399399
goto foundIndent
400400
}
401401
}
402-
x.Error("Inconsistent indent")
402+
x.SyntaxError("Inconsistent indent")
403403
return eof
404404
foundIndent:
405405
x.indentStack = x.indentStack[:len(x.indentStack)-1]
@@ -489,7 +489,7 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
489489
}
490490

491491
// Nothing we recognise found
492-
x.Error("invalid syntax")
492+
x.SyntaxError("invalid syntax")
493493
return eof
494494
case checkEof:
495495
if x.eof {
@@ -683,8 +683,9 @@ isNumber:
683683
value = py.Complex(complex(0, f))
684684
} else {
685685
// Discard numbers with leading 0 except all 0s
686-
if illegalDecimalInteger.FindString(x.line) != "" {
687-
x.Error("illegal decimal with leading zero")
686+
if illegalDecimalInteger.FindString(s) != "" {
687+
// FIXME where is this error going in the grammar?
688+
x.SyntaxError("illegal decimal with leading zero")
688689
return eofError, nil
689690
}
690691
value, err = py.IntFromString(s, 10)
@@ -806,12 +807,12 @@ found:
806807
}
807808
}
808809
if !multiLineString {
809-
x.Errorf("Unterminated %sx%s string", stringEnd, stringEnd)
810+
x.SyntaxErrorf("Unterminated %sx%s string", stringEnd, stringEnd)
810811
return eofError, nil
811812
}
812813
readMore:
813814
if x.eof {
814-
x.Errorf("Unterminated %sx%s string", stringEnd, stringEnd)
815+
x.SyntaxErrorf("Unterminated %sx%s string", stringEnd, stringEnd)
815816
return eofError, nil
816817
}
817818
x.refill()
@@ -821,7 +822,7 @@ foundEndOfString:
821822
var err error
822823
buf, err = DecodeEscape(buf, byteString)
823824
if err != nil {
824-
x.Errorf("Decode error: %v", err)
825+
x.SyntaxErrorf("Decode error: %v", err)
825826
return eofError, nil
826827
}
827828
}
@@ -834,23 +835,31 @@ foundEndOfString:
834835
// The parser calls this method on a parse error.
835836
func (x *yyLex) Error(s string) {
836837
x.error = true
837-
x.errorString = s
838838
if yyDebug >= 1 {
839839
log.Printf("Parse error: %s", s)
840840
log.Printf("Parse buffer %q", x.line)
841841
log.Printf("State %#v", x)
842842
}
843843
}
844844

845+
// The parser calls this method on a parse error.
846+
func (x *yyLex) SyntaxError(s string) {
847+
x.errorString = s
848+
x.Error(s)
849+
}
850+
845851
// Call this to write formatted errors
846-
func (x *yyLex) Errorf(format string, a ...interface{}) {
847-
x.Error(fmt.Sprintf(format, a...))
852+
func (x *yyLex) SyntaxErrorf(format string, a ...interface{}) {
853+
x.SyntaxError(fmt.Sprintf(format, a...))
848854
}
849855

850856
// Returns an python error for the current yyLex
851857
func (x *yyLex) ErrorReturn() error {
852858
if x.error {
853-
return py.ExceptionNewf(py.SyntaxError, "Syntax Error: %s", x.errorString)
859+
if x.errorString == "" {
860+
x.errorString = "invalid syntax"
861+
}
862+
return py.ExceptionNewf(py.SyntaxError, "%s", x.errorString)
854863
}
855864
return nil
856865
}

‎parser/lexer_test.go

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,31 @@ func TestLex(t *testing.T) {
233233
{DEDENT, nil},
234234
{ENDMARKER, nil},
235235
}},
236+
{"1", "", "eval", LexTokens{
237+
{EVAL_INPUT, nil},
238+
{NUMBER, py.Int(1)},
239+
{ENDMARKER, nil},
240+
}},
241+
{"01", "illegal decimal with leading zero", "eval", LexTokens{
242+
{EVAL_INPUT, nil},
243+
}},
244+
{"1", "", "exec", LexTokens{
245+
{FILE_INPUT, nil},
246+
{NUMBER, py.Int(1)},
247+
{NEWLINE, nil},
248+
{ENDMARKER, nil},
249+
}},
250+
{"1 2 3", "", "exec", LexTokens{
251+
{FILE_INPUT, nil},
252+
{NUMBER, py.Int(1)},
253+
{NUMBER, py.Int(2)},
254+
{NUMBER, py.Int(3)},
255+
{NEWLINE, nil},
256+
{ENDMARKER, nil},
257+
}},
258+
{"01", "illegal decimal with leading zero", "exec", LexTokens{
259+
{FILE_INPUT, nil},
260+
}},
236261
{"1\n 2\n 3\n4\n", "", "exec", LexTokens{
237262
{FILE_INPUT, nil},
238263
{NUMBER, py.Int(1)},
@@ -343,7 +368,7 @@ func TestLex(t *testing.T) {
343368
errString = err.Error()
344369
}
345370
if test.errString != "" {
346-
test.errString = "SyntaxError: [Syntax Error: " + test.errString + "]"
371+
test.errString = "SyntaxError: [" + test.errString + "]"
347372
}
348373
if errString != test.errString || !lts.Eq(test.lts) {
349374
t.Errorf("Lex(%q) expecting (%v, %q) got (%v, %q)", test.in, test.lts, test.errString, lts, errString)

‎parser/make_grammar_test.py

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@
2424
("b'abc'", "eval"),
2525
("b'abc' b'''123'''", "eval"),
2626
("1234", "eval"),
27+
("01234", "eval", SyntaxError, "illegal decimal with leading zero"),
28+
("1234d", "eval", SyntaxError, "invalid syntax"),
29+
("1234d", "exec", SyntaxError),
2730
("0x1234", "eval"),
2831
("12.34", "eval"),
2932
("1,", "eval"),
@@ -359,6 +362,7 @@
359362

360363
# Assign
361364
("a = b", "exec"),
365+
("a = 007", "exec", SyntaxError, "illegal decimal with leading zero"),
362366
("a = b = c", "exec"),
363367
("a, b = 1, 2", "exec"),
364368
("a, b = c, d = 1, 2", "exec"),
@@ -466,13 +470,37 @@ def main():
466470
467471
package parser
468472
473+
import (
474+
"github.com/ncw/gpython/py"
475+
)
476+
469477
var grammarTestData = []struct {
470478
in string
471479
mode string
472480
out string
481+
exceptionType *py.Type
482+
errString string
473483
}{"""]
474-
for source, mode in inp:
475-
out.append('{"%s", "%s", "%s"},' % (escape(source), mode, escape(dump(source, mode))))
484+
for x in inp:
485+
source, mode = x[:2]
486+
if len(x) > 2:
487+
exc = x[2]
488+
errString = (x[3] if len(x) > 3 else "")
489+
try:
490+
dump(source, mode)
491+
except exc as e:
492+
error = e.msg
493+
else:
494+
raise ValueError("Expecting exception %s" % exc)
495+
if errString != "":
496+
error = errString # override error string
497+
dmp = ""
498+
exc_name = "py.%s" % exc.__name__
499+
else:
500+
dmp = dump(source, mode)
501+
exc_name = "nil"
502+
error = ""
503+
out.append('{"%s", "%s", "%s", %s, "%s"},' % (escape(source), mode, escape(dmp), exc_name, escape(error)))
476504
out.append("}")
477505
print("Writing %s" % path)
478506
with open(path, "w") as f:

‎parser/y.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2063,7 +2063,7 @@ yydefault:
20632063
//line grammar.y:1336
20642064
{
20652065
// panic("FIXME no coverage")
2066-
yylex.Error("Invalid syntax")
2066+
yylex.(*yyLex).SyntaxError("Invalid syntax")
20672067
}
20682068
case 209:
20692069
//line grammar.y:1341
@@ -2234,14 +2234,14 @@ yydefault:
22342234
case py.String:
22352235
yyVAL.obj = a + b
22362236
default:
2237-
yylex.Error("SyntaxError: cannot mix string and nonstring literals")
2237+
yylex.(*yyLex).SyntaxError("cannot mix string and nonstring literals")
22382238
}
22392239
case py.Bytes:
22402240
switch b := yyS[yypt-0].obj.(type) {
22412241
case py.Bytes:
22422242
yyVAL.obj = append(a, b...)
22432243
default:
2244-
yylex.Error("SyntaxError: cannot mix bytes and nonbytes literals")
2244+
yylex.(*yyLex).SyntaxError("cannot mix bytes and nonbytes literals")
22452245
}
22462246
}
22472247
}
@@ -2582,7 +2582,7 @@ yydefault:
25822582
call := yyS[yypt-3].call
25832583
call.Starargs = yyS[yypt-1].expr
25842584
if len(yyS[yypt-0].call.Args) != 0 {
2585-
yylex.Error("SyntaxError: only named arguments may follow *expression")
2585+
yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
25862586
}
25872587
call.Keywords = append(call.Keywords, yyS[yypt-0].call.Keywords...)
25882588
yyVAL.call = call
@@ -2594,7 +2594,7 @@ yydefault:
25942594
call.Starargs = yyS[yypt-4].expr
25952595
call.Kwargs = yyS[yypt-0].expr
25962596
if len(yyS[yypt-3].call.Args) != 0 {
2597-
yylex.Error("SyntaxError: only named arguments may follow *expression")
2597+
yylex.(*yyLex).SyntaxError("only named arguments may follow *expression")
25982598
}
25992599
call.Keywords = append(call.Keywords, yyS[yypt-3].call.Keywords...)
26002600
yyVAL.call = call
@@ -2628,7 +2628,7 @@ yydefault:
26282628
if name, ok := test.(*ast.Name); ok {
26292629
yyVAL.call.Keywords = []*ast.Keyword{&ast.Keyword{Pos: name.Pos, Arg: name.Id, Value: yyS[yypt-0].expr}}
26302630
} else {
2631-
yylex.Error("SyntaxError: keyword can't be an expression")
2631+
yylex.(*yyLex).SyntaxError("keyword can't be an expression")
26322632
}
26332633
}
26342634
case 303:

0 commit comments

Comments
 (0)