Split standalone comments and empty lines to separate statements

yanne · yanne · commit 1dba783e578d · 2019-09-03T12:19:51.000+03:00
Also add a separate EOL token
diff --git a/src/robot/parsing/lexer/__init__.py b/src/robot/parsing/lexer/__init__.py
@@ -13,6 +13,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+from itertools import chain
+
 from .context import TestCaseFileContext, ResourceFileContext
 from .lexers import FileLexer
 from .splitter import Splitter
@@ -45,7 +47,13 @@ def get_tokens(self):
                       Token.OLD_FOR_INDENT}
         else:
             ignore = {Token.IGNORE}
-        for statement in self._handle_old_for(self.statements):
+        statements = self._handle_old_for(self.statements)
+        if not self._data_only:
+            statements = chain.from_iterable(
+                self._split_trailing_comment_and_empty_lines(s)
+                for s in statements
+            )
+        for statement in statements:
             name_token = last_token = None
             for token in statement:
                 if token.type in ignore:
@@ -82,6 +90,33 @@ def _get_first_data_token(self, statement):
                 return token
         return None
 
+    def _split_trailing_comment_and_empty_lines(self, statement):
+        lines = list(self._split_to_lines(statement))
+        split_statements = []
+        for line in reversed(lines):
+            is_split = False
+            for token in line:
+                if token.type not in (token.IGNORE, token.SEPARATOR):
+                    is_split = token.type in (token.EOL, token.COMMENT)
+                    break
+            if not is_split:
+                break
+            split_statements.append(line)
+            lines.pop()
+        yield list(chain.from_iterable(lines))
+        for split in reversed(split_statements):
+            yield split
+
+    def _split_to_lines(self, statement):
+        current = []
+        for tok in statement:
+            current.append(tok)
+            if tok.type == tok.EOL:
+                yield current
+                current = []
+        if current:
+            yield current
+
 
 class TestCaseFileLexer(BaseLexer):
     context_class = TestCaseFileContext
diff --git a/src/robot/parsing/lexer/splitter.py b/src/robot/parsing/lexer/splitter.py
@@ -50,7 +50,7 @@ def _split_line(self, line, lineno, data_only=False):
         if not data_only:
             trailing_whitespace = re.search(r'\s+$', line, flags=re.UNICODE)
             if trailing_whitespace:
-                yield Token(sepa, trailing_whitespace.group(), lineno, columnno)
+                yield Token(Token.EOL, trailing_whitespace.group(), lineno, columnno)
 
     def _split_from_spaces(self, line):
         for index, value in enumerate(self._space_splitter.split(line)):
diff --git a/src/robot/parsing/lexer/tokens.py b/src/robot/parsing/lexer/tokens.py
@@ -53,6 +53,7 @@ class Token(object):
     END = 'END'
 
     SEPARATOR = 'SEPARATOR'
+    EOL = 'EOL'
     COMMENT = 'COMMENT'
     CONTINUATION = 'CONTINUATION'
     IGNORE = 'IGNORE'
diff --git a/utest/parsing/test_splitter.py b/utest/parsing/test_splitter.py
