tidy: make all formatting changes to parsing model

yanne · yanne · commit 6b9d2b61daf8 · 2020-01-09T09:08:11.000+02:00
This allows also programmatic access to the tidied AST
diff --git a/src/robot/parsing/__init__.py b/src/robot/parsing/__init__.py
@@ -51,5 +51,5 @@ def print_suite(suite):
 
 from .builders import get_model, get_resource_model
 from .lexer import get_tokens, get_resource_tokens, Token
-from .model import ModelVisitor
+from .model import ModelTransformer
 from .suitestructure import SuiteStructureBuilder, SuiteStructureVisitor
diff --git a/src/robot/parsing/builders.py b/src/robot/parsing/builders.py
@@ -90,6 +90,8 @@ class TestCaseSectionBuilder(SectionBuilder):
 
     def statement(self, statement):
         if statement.type == Token.EOL:
+            if self.model.body.items:
+                self.model.body.add(statement)
             return self
         model = TestCase(statement)
         self.model.body.add(model)
@@ -100,6 +102,8 @@ class KeywordSectionBuilder(SectionBuilder):
 
     def statement(self, statement):
         if statement.type == Token.EOL:
+            if self.model.body.items:
+                self.model.body.add(statement)
             return self
         model = Keyword(statement)
         self.model.body.add(model)
diff --git a/src/robot/parsing/model/__init__.py b/src/robot/parsing/model/__init__.py
@@ -16,4 +16,4 @@
 from .blocks import (File, SettingSection, VariableSection, TestCaseSection,
                      KeywordSection, CommentSection, TestCase, Keyword, ForLoop)
 from .statements import get_statements
-from .visitor import ModelVisitor
+from .visitor import ModelTransformer
diff --git a/src/robot/parsing/model/statements.py b/src/robot/parsing/model/statements.py
@@ -77,15 +77,12 @@ def _tokens(self, *types):
 
     @property
     def lines(self):
-        if self.type in Token.SETTING_TOKENS and len(self.tokens) == 1:
-            return
         line = []
         for token in self.tokens:
-            if token.type != Token.CONTINUATION:
-                line.append(token)
-            else:
+            line.append(token)
+            if token.type == Token.EOL:
                 yield line
-                line = [token]
+                line = []
         if line:
             yield line
 
@@ -424,4 +421,4 @@ def error(self):
 
 
 class EmptyLine(Statement):
-    pass
+    type = Token.EOL
diff --git a/src/robot/parsing/model/visitor.py b/src/robot/parsing/model/visitor.py
@@ -16,11 +16,7 @@
 import ast
 
 
-class ModelVisitor(ast.NodeVisitor):
-
-    def visit(self, node):
-        visitor = self._find_visitor(type(node)) or self.generic_visit
-        return visitor(node)
+class VisitorFinder(object):
 
     def _find_visitor(self, cls):
         if cls is ast.AST:
@@ -32,3 +28,17 @@ def _find_visitor(self, cls):
             visitor = self._find_visitor(base)
             if visitor:
                 return visitor
+
+
+class ModelVisitor(ast.NodeVisitor, VisitorFinder):
+
+    def visit(self, node):
+        visitor = self._find_visitor(type(node)) or self.generic_visit
+        visitor(node)
+
+
+class ModelTransformer(ast.NodeTransformer, VisitorFinder):
+
+    def visit(self, node):
+        visitor = self._find_visitor(type(node)) or self.generic_visit
+        return visitor(node)
diff --git a/src/robot/tidypkg/datafilewriter.py b/src/robot/tidypkg/datafilewriter.py
@@ -16,7 +16,7 @@
 from robot.errors import DataError
 from robot.utils import file_writer
 
-from .transformers import Aligner, Cleaner, Formatter, SeparatorRemover
+from .transformers import Aligner, Cleaner, Formatter, SeparatorCleaner, PipeAdder, NewlineAdder
 
 
 class DataFileWriter(object):
@@ -39,8 +39,12 @@ def write(self, model):
         """
 
         with WritingContext(model, **self._options) as ctx:
-            SeparatorRemover().visit(model)
             Cleaner().visit(model)
+            NewlineAdder().visit(model)
+            if ctx.pipe_separated:
+                PipeAdder().visit(model)
+            else:
+                SeparatorCleaner(ctx).visit(model)
             Aligner(ctx).visit(model)
             Formatter(ctx, RowWriter(ctx)).visit(model)
 
@@ -112,24 +116,7 @@ class RowWriter(object):
 
     def __init__(self, ctx):
         self.output = ctx.output
-        self.pipes = ctx.pipe_separated
-        self.separator, self.indent_string = self.get_separator_and_indent(ctx)
 
-    def get_separator_and_indent(self, ctx):
-        if ctx.pipe_separated:
-            return ' | ', '   | '
-        separator = ' ' * ctx.txt_separating_spaces
-        return separator, separator
-
-    def write(self, tokens, indent):
-        indent = indent * self.indent_string
+    def write(self, tokens):
         values = [t.value for t in tokens]
-        row = indent + self.separator.join(values)
-        if self.pipes:
-            row = '| ' + row + ' |'
-        else:
-            row = row.rstrip()
-        self.output.write(row)
-
-    def write_newline(self):
-        self.output.write('\n')
+        self.output.write(''.join(values).rstrip(' '))
diff --git a/src/robot/tidypkg/transformers.py b/src/robot/tidypkg/transformers.py
