Skip to content

Commit 7bb1d4c

Browse files
yanneyanne
committed
separate lexer for resource files
1 parent 68edf3f commit 7bb1d4c

File tree

7 files changed

+66
-61
lines changed

7 files changed

+66
-61
lines changed

‎src/robot/parsing/lexer/__init__.py

Lines changed: 13 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,17 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
from __future__ import print_function # TODO: Remove once "main" is removed
17-
18-
from .context import TestCaseFileContext
19-
from .lexers import TestCaseFileLexer
16+
from .context import TestCaseFileContext, ResourceFileContext
17+
from .lexers import FileLexer
2018
from .splitter import Splitter
2119
from .tokens import EOS, Token
2220

2321

24-
class RobotFrameworkLexer(object):
22+
class BaseLexer(object):
23+
context_class = None
2524

26-
def __init__(self, data_only=False, lexer=None):
27-
self.lexer = lexer or TestCaseFileLexer()
25+
def __init__(self, data_only=True):
26+
self.lexer = FileLexer()
2827
self.statements = []
2928
self._data_only = data_only
3029

@@ -40,9 +39,8 @@ def input(self, content):
4039
self.lexer.input(data)
4140

4241
def get_tokens(self):
43-
self.lexer.lex(TestCaseFileContext())
42+
self.lexer.lex(self.context_class())
4443
if self._data_only:
45-
# TODO: Should whole statement be ignored if there's ERROR?
4644
ignore = {Token.IGNORE, Token.COMMENT_HEADER, Token.COMMENT,
4745
Token.OLD_FOR_INDENT}
4846
else:
@@ -85,22 +83,9 @@ def _get_first_data_token(self, statement):
8583
return None
8684

8785

88-
# TODO: Remove when integrated...
89-
if __name__ == '__main__':
90-
import sys
91-
if len(sys.argv) == 2:
92-
data_only = False
93-
formatter = str
94-
end = ''
95-
ignore = ()
96-
else:
97-
data_only = True
98-
formatter = repr
99-
end = '\n'
100-
ignore = (Token.EOS,)
101-
lxr = RobotFrameworkLexer(data_only=data_only)
102-
with open(sys.argv[1]) as f:
103-
lxr.input(f.read())
104-
for token in lxr.get_tokens():
105-
if token.type not in ignore:
106-
print(formatter(token), end=end)
86+
class TestCaseFileLexer(BaseLexer):
87+
context_class = TestCaseFileContext
88+
89+
90+
class ResourceFileLexer(BaseLexer):
91+
context_class = ResourceFileContext

‎src/robot/parsing/lexer/context.py

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,28 +13,40 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
from .settings import KeywordSettings, TestCaseFileSettings, TestCaseSettings
16+
from .settings import (KeywordSettings, TestCaseFileSettings,
17+
TestCaseSettings, ResourceFileSettings)
1718

1819

1920
class LexingContext(object):
2021

21-
def __init__(self, settings=None):
22+
def __init__(self, settings):
2223
self.settings = settings
2324

2425
def lex_setting(self, statement):
2526
self.settings.lex(statement)
2627

27-
def test_case_context(self):
28-
return TestCaseContext(TestCaseSettings(self.settings))
29-
3028
def keyword_context(self):
3129
return KeywordContext(KeywordSettings())
3230

3331

3432
class TestCaseFileContext(LexingContext):
3533

36-
def __init__(self, settings=None):
37-
LexingContext.__init__(self, settings or TestCaseFileSettings())
34+
def __init__(self):
35+
LexingContext.__init__(self, TestCaseFileSettings())
36+
37+
def test_case_context(self):
38+
return TestCaseContext(TestCaseSettings(self.settings))
39+
40+
41+
class ResourceFileContext(LexingContext):
42+
43+
def __init__(self):
44+
LexingContext.__init__(self, ResourceFileSettings())
45+
46+
def test_case_context(self):
47+
# Lex test cases for resource/init files.
48+
# The error will be reported when the model is built
49+
return KeywordContext(TestCaseSettings(self.settings))
3850

3951

4052
class TestCaseContext(LexingContext):
@@ -45,7 +57,6 @@ def template_set(self):
4557

4658

4759
class KeywordContext(LexingContext):
48-
settings_class = KeywordSettings
4960

5061
@property
5162
def template_set(self):

‎src/robot/parsing/lexer/lexers.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ def _lex_with_priority(self, ctx, priority):
8989
lexer.lex(ctx)
9090

9191

92-
class TestCaseFileLexer(BlockLexer):
92+
class FileLexer(BlockLexer):
9393

9494
def lex(self, ctx):
9595
self._lex_with_priority(ctx, priority=SettingSectionLexer)

‎src/robot/parsing/newparser/builder.py

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,41 @@
11
import os
22

3+
from robot.errors import DataError
34
from robot.output import LOGGER
45
from robot.parsing.vendor import yacc
5-
from robot.parsing.lexer import RobotFrameworkLexer
6+
from robot.parsing.lexer import TestCaseFileLexer, ResourceFileLexer
67
from robot.utils import Utf8Reader
78

9+
from .nodes import TestCaseSection
810
from .parser import RobotFrameworkParser
911

1012

11-
class Builder(object):
13+
def get_test_case_file_ast(source):
14+
lexer = TestCaseFileLexer()
15+
parser = yacc.yacc(module=RobotFrameworkParser())
16+
return parser.parse(lexer=LexerWrapper(lexer, source))
1217

13-
def read(self, source):
14-
data = Utf8Reader(source).read()
15-
parser = yacc.yacc(module=RobotFrameworkParser())
16-
return parser.parse(lexer=LexerWrapper(data, source))
18+
19+
def get_resource_file_ast(source):
20+
lexer = ResourceFileLexer()
21+
parser = yacc.yacc(module=RobotFrameworkParser())
22+
ast = parser.parse(lexer=LexerWrapper(lexer, source))
23+
if any(isinstance(s, TestCaseSection) for s in ast.sections):
24+
raise DataError("Resource file '%s' cannot contain tests or tasks." % source)
25+
return ast
1726

1827

1928
class LexerWrapper(object):
2029

21-
def __init__(self, data, source):
30+
def __init__(self, lexer, source):
2231
self.source = source
2332
self.curdir = os.path.dirname(source).replace('\\', '\\\\')
24-
lexer = RobotFrameworkLexer(data_only=True)
25-
lexer.input(data)
33+
lexer.input(Utf8Reader(source).read())
2634
self.tokens = lexer.get_tokens()
2735

2836
def token(self):
2937
token = next(self.tokens, None)
38+
# print(repr(token))
3039
if token and token.type == token.ERROR:
3140
self._report_error(token)
3241
return self._next_token_after_eos()
@@ -39,7 +48,9 @@ def _report_error(self, token):
3948
LOGGER.error("Error in file '%s': %s" % (self.source, token.error))
4049

4150
def _next_token_after_eos(self):
42-
token = self.token()
43-
while token.type != token.EOS:
51+
while True:
4452
token = self.token()
45-
return self.token()
53+
if token is None:
54+
return None
55+
if token.type == token.EOS:
56+
return self.token()

‎src/robot/parsing/newparser/nodes.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from ast import AST
22

3+
34
class Node(AST):
45
_fields = ()
56

‎src/robot/parsing/newparser/parser.py

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,7 @@ def p_section(self, p):
3535
def p_setting_section(self, p):
3636
'''setting_section : SETTING_HEADER EOS
3737
| SETTING_HEADER EOS settings'''
38-
if len(p) == 4:
39-
p[0] = SettingSection(p[3])
38+
p[0] = SettingSection(p[3] if len(p) == 4 else [])
4039

4140
def p_settings(self, p):
4241
'''settings : setting
@@ -152,8 +151,7 @@ def p_return(self, p):
152151
def p_variable_section(self, p):
153152
'''variable_section : VARIABLE_HEADER EOS
154153
| VARIABLE_HEADER EOS variables'''
155-
if len(p) == 4:
156-
p[0] = VariableSection(p[3])
154+
p[0] = VariableSection(p[3] if len(p) == 4 else [])
157155

158156
def p_variables(self, p):
159157
'''variables : variable
@@ -167,14 +165,12 @@ def p_variable(self, p):
167165
def p_testcase_section(self, p):
168166
'''testcase_section : TESTCASE_HEADER EOS
169167
| TESTCASE_HEADER EOS tests'''
170-
if len(p) == 4:
171-
p[0] = TestCaseSection(p[3])
168+
p[0] = TestCaseSection(p[3] if len(p) == 4 else [])
172169

173170
def p_keyword_section(self, p):
174171
'''keyword_section : KEYWORD_HEADER EOS
175172
| KEYWORD_HEADER EOS keywords'''
176-
if len(p) == 4:
177-
p[0] = KeywordSection(p[3])
173+
p[0] = KeywordSection(p[3] if len(p) == 4 else [])
178174

179175
def p_tests(self, p):
180176
'''tests : test

‎src/robot/running/newbuilder.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
from robot.errors import DataError
66
from robot.parsing import TEST_EXTENSIONS
7-
from robot.parsing.newparser.builder import Builder
7+
from robot.parsing.newparser import builder
88
from robot.model import SuiteNamePatterns
99
from robot.running.model import TestSuite, Keyword, ForLoop, ResourceFile
1010
from robot.utils import abspath
@@ -347,6 +347,7 @@ def _get_extensions(self, extension):
347347
return extensions
348348

349349
def _parse_and_build(self, path, parent_defaults=None, include_suites=None, include_extensions=None):
350+
path = abspath(path)
350351
name = format_name(path)
351352
if os.path.isdir(path):
352353
LOGGER.info("Parsing directory '%s'." % path)
@@ -370,14 +371,14 @@ def _build_suite(self, source, name, parent_defaults):
370371
suite = TestSuite(name=name, source=source)
371372
defaults = TestDefaults(parent_defaults)
372373
if data:
373-
print(ast.dump(data))
374+
#print(ast.dump(data))
374375
SettingsBuilder(suite, defaults).visit(data)
375376
SuiteBuilder(suite, defaults).visit(data)
376377
return suite, defaults
377378

378379
def _parse(self, path):
379380
try:
380-
return Builder().read(abspath(path))
381+
return builder.get_test_case_file_ast(path)
381382
except DataError as err:
382383
raise DataError("Parsing '%s' failed: %s" % (path, err.message))
383384

@@ -461,7 +462,7 @@ class ResourceFileBuilder(object):
461462

462463
def build(self, path):
463464
resource = ResourceFile(source=path)
464-
data = Builder().read(abspath(path))
465+
data = builder.get_resource_file_ast(path)
465466
if data.sections:
466467
ResourceBuilder(resource).visit(data)
467468
else:

0 commit comments

Comments
 (0)