Skip to content

Commit 4639e27

Browse files
bburanfrmdstryrMatthieuDartiailh
authored
Enaml syntaxerror detail (nucleic#530)
* Patch for Python 3.11 SyntaxError * Create tests to detect correct syntaxerror detail This does not actually capture the error I am attempting to fix. * Add compatibility for Python 3.8 and 3.9 This also fixes a bug in the unit test (the assert statement was missing). * Fix typo in unit test for parser * Include location of next token when raising indentation error * Wrong expected error message in test_parser * Regenerate Enaml parser * Update release notes with bugfix detail * Add information on how to regenerate Enaml parser * Regenerate parser with pegen 0.1.0 Pegen 0.2.0 results in some IndentationError exceptions being raised as SyntaxError instead. * Pin pegen to 0.1.0 * Update releasenotes.rst --------- Co-authored-by: frm dstryr <frmdstryr@protonmail.com> Co-authored-by: Matthieu Dartiailh <marul@laposte.net>
1 parent 87e9c31 commit 4639e27

File tree

7 files changed

+255
-47
lines changed

7 files changed

+255
-47
lines changed

‎development.rst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Regenerating the Enaml parser
2+
=============================
3+
4+
To regenerate the parser:
5+
6+
python -m enaml.core.parser.generate_enaml_parser
7+
black enaml/core/parser/enaml_parser.py

‎enaml/core/parser/base_python_parser.py

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
from pegen.parser import Parser
1717
from pegen.tokenizer import Tokenizer
1818

19+
from enaml.compat import PY310
20+
1921
# Singleton ast nodes, created once for efficiency
2022
Load = ast.Load()
2123
Store = ast.Store()
@@ -76,7 +78,13 @@ def parse(self, rule: str) -> Optional[ast.AST]:
7678
raise self._exception
7779
else:
7880
token = self._tokenizer.diagnose()
79-
raise SyntaxError("invalid syntax", (self.filename, token.start, 0, token.line))
81+
lineno, offset = token.start
82+
end_lineno, end_offset = token.end
83+
if PY310:
84+
args = (self.filename, lineno, offset, token.line, end_lineno, end_offset)
85+
else:
86+
args = (self.filename, lineno, offset, token.line)
87+
raise SyntaxError("invalid syntax", args)
8088

8189
return res
8290

@@ -93,7 +101,9 @@ def check_version(self, min_version: Tuple[int, ...], error_msg: str, node: Node
93101

94102
def raise_indentation_error(self, msg) -> None:
95103
"""Raise an indentation error."""
96-
raise IndentationError(msg)
104+
node = self._tokenizer.peek()
105+
self.store_syntax_error_known_location(msg, node, IndentationError)
106+
raise self._exception
97107

98108
def get_expr_name(self, node) -> str:
99109
"""Get a descriptive name for an expression."""
@@ -245,7 +255,8 @@ def _store_syntax_error(
245255
self,
246256
message: str,
247257
start: Optional[Tuple[int, int]] = None,
248-
end: Optional[Tuple[int, int]] = None
258+
end: Optional[Tuple[int, int]] = None,
259+
exc_type: type = SyntaxError
249260
) -> None:
250261
line_from_token = start is None and end is None
251262
if start is None or end is None:
@@ -264,7 +275,7 @@ def _store_syntax_error(
264275
args = (self.filename, start[0], start[1], line)
265276
if sys.version_info >= (3, 10):
266277
args += (end[0], end[1])
267-
self._exception = SyntaxError(message, args)
278+
self._exception = exc_type(message, args)
268279

269280
def store_syntax_error(self, message: str) -> None:
270281
self._store_syntax_error(message)
@@ -273,7 +284,12 @@ def make_syntax_error(self, message: str) -> None:
273284
self._store_syntax_error(message)
274285
return self._exception
275286

276-
def store_syntax_error_known_location(self, message: str, node) -> None:
287+
def store_syntax_error_known_location(
288+
self,
289+
message: str,
290+
node,
291+
exc_type: type = SyntaxError
292+
) -> None:
277293
"""Store a syntax error that occured at a given AST node."""
278294
if isinstance(node, tokenize.TokenInfo):
279295
start = node.start
@@ -282,7 +298,7 @@ def store_syntax_error_known_location(self, message: str, node) -> None:
282298
start = node.lineno, node.col_offset
283299
end = node.end_lineno, node.end_col_offset
284300

285-
self._store_syntax_error(message, start, end)
301+
self._store_syntax_error(message, start, end, exc_type)
286302

287303
def store_syntax_error_known_range(
288304
self,

‎enaml/core/parser/enaml_parser.py

Lines changed: 37 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,21 @@
88
# NOTE This file was generated using enaml/core/parser/generate_enaml_parser.py
99
# DO NOT EDIT DIRECTLY
1010
import ast
11-
import itertools
1211
import sys
12+
import itertools
1313
from typing import Any, List, NoReturn, Optional, Tuple, TypeVar, Union
1414

15-
from pegen.parser import Parser, logger, memoize, memoize_left_rec
16-
1715
from enaml.core import enaml_ast
16+
from pegen.parser import Parser, logger, memoize, memoize_left_rec
1817

1918
from .base_enaml_parser import BaseEnamlParser as Parser
2019

2120
# Singleton ast nodes, created once for efficiency
2221
Load = ast.Load()
2322
Store = ast.Store()
2423
Del = ast.Del()
24+
25+
2526
# Keywords and soft keywords are listed at the end of the parser definition.
2627
class EnamlParser(Parser):
2728
@memoize
@@ -9697,54 +9698,54 @@ def _tmp_241(self) -> Optional[Any]:
96979698
return None
96989699

96999700
KEYWORDS = (
9700-
"nonlocal",
9701-
"global",
9702-
"True",
9703-
"while",
9704-
"for",
9705-
"elif",
9706-
"del",
9707-
"from",
9708-
"pass",
9709-
"def",
9710-
"except",
9711-
"None",
9712-
"with",
9701+
"class",
97139702
"or",
9714-
"return",
9703+
"del",
97159704
"not",
9716-
"and",
9717-
"await",
9718-
"finally",
97199705
"in",
9706+
"await",
9707+
"except",
9708+
"return",
9709+
"try",
97209710
"import",
9721-
"yield",
9722-
"raise",
9723-
"assert",
9711+
"break",
97249712
"else",
9725-
"False",
9726-
"lambda",
9727-
"class",
9713+
"for",
9714+
"with",
97289715
"async",
9729-
"break",
9716+
"None",
9717+
"assert",
9718+
"global",
97309719
"if",
9731-
"try",
97329720
"as",
9733-
"continue",
9721+
"True",
9722+
"yield",
9723+
"raise",
9724+
"and",
9725+
"finally",
97349726
"is",
9727+
"while",
9728+
"pass",
9729+
"from",
9730+
"False",
9731+
"elif",
9732+
"lambda",
9733+
"nonlocal",
9734+
"def",
9735+
"continue",
97359736
)
97369737
SOFT_KEYWORDS = (
9737-
"event",
9738+
"case",
97389739
"template",
9740+
"match",
9741+
"event",
9742+
"alias",
97399743
"attr",
9744+
"enamldef",
97409745
"const",
9741-
"func",
9742-
"pragma",
9743-
"case",
9744-
"alias",
97459746
"_",
9746-
"enamldef",
9747-
"match",
9747+
"pragma",
9748+
"func",
97489749
)
97499750

97509751

‎pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ dependencies = [
3333
"atom>=0.9.0",
3434
"kiwisolver>=1.2.0",
3535
"bytecode>=0.14.2",
36-
"pegen>=0.1.0",
36+
"pegen>=0.1.0,<0.2.0",
3737
]
3838
dynamic=["version"]
3939

‎releasenotes.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ Enaml Release Notes
33

44
Dates are written as DD/MM/YYYY
55

6+
XXXXX
7+
-----
8+
- fix bug in Enaml parser that was not showing proper location of syntax and
9+
indentation errors in tracebacks when the error was in an Enaml file.
10+
611
0.16.1 - 05/05/2023
712
-------------------
813
- fix typo causing a crash in dock area PR #523

‎tests/conftest.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ def validate_parser_is_up_to_date():
8585
last_source_modif
8686
), (
8787
"Generated parser appears outdated compared to its sources, "
88-
"re-generate it using enaml/core/parser/generate_enaml_parser.enaml"
88+
"re-generate it using `python -m enaml.core.parser.generate_enaml_parser`"
8989
)
9090

9191

0 commit comments

Comments
 (0)