Validate output.xml schema as part of acceptance tests.

pekkaklarck · pekkaklarck · commit 2fc7278db25a · 2021-03-07T21:17:38.000+02:00
This isn't done for all outputs by default because it slows down execution a bit too much. That can be enabled by setting ATEST_VALIDATE_OUTPUT environment varibale to TRUE. Part of schema update issue robotframework#3726.
diff --git a/atest/README.rst b/atest/README.rst
@@ -228,6 +228,18 @@ documentation from Java source files. In addition to setting ``CLASSPATH``
 explicitly, it is possible to put ``tools.jar`` into the ``ext-lib``
 directory in the project root and ``CLASSPATH`` is set automatically.
 
+Schema validation
+-----------------
+
+Created output.xml has `<../doc/schema>`_ that can be tested as part of acceptance
+test run. The schema is always used to validate selected outputs in
+`<robot/rebot/compatibility.robot>`_, but validating all outputs would slow down
+execution a bit too much.
+
+It is, however, possible to enable validating all outputs by setting
+`ATEST_VALIDATE_OUTPUT` environment variable to `TRUE` (case-insensitive).
+This is recommended especially if the schema is updated or output.xml changed.
+
 Telnet tests
 ------------
 
diff --git a/atest/resources/TestCheckerLibrary.py b/atest/resources/TestCheckerLibrary.py
@@ -1,6 +1,8 @@
 import os
 import re
 
+from xmlschema import XMLSchema
+
 from robot import utils
 from robot.api import logger
 from robot.utils.asserts import assert_equal
@@ -60,14 +62,22 @@ class NoSlotsTestSuite(TestSuite):
 
 
 class TestCheckerLibrary:
+    ROBOT_LIBRARY_SCOPE = 'GLOBAL'
+
+    def __init__(self):
+        self.schema = XMLSchema('doc/schema/robot.02.xsd')
 
-    def process_output(self, path):
+    def process_output(self, path, validate=None):
         set_suite_variable = BuiltIn().set_suite_variable
         if not path or path.upper() == 'NONE':
             set_suite_variable('$SUITE', None)
             logger.info("Not processing output.")
             return
         path = path.replace('/', os.sep)
+        if validate is None:
+            validate = os.getenv('ATEST_VALIDATE_OUTPUT', False)
+        if utils.is_truthy(validate):
+            self.schema.validate(path)
         try:
             logger.info("Processing output '%s'." % path)
             result = Result(root_suite=NoSlotsTestSuite())
diff --git a/atest/resources/atest_resource.robot b/atest/resources/atest_resource.robot
@@ -52,11 +52,11 @@ Run Tests Without Processing Output
     [Return]    ${result}
 
 Run Rebot
-    [Arguments]    ${options}=    ${sources}=    ${default options}=${COMMON DEFAULTS}    ${output}=${OUTFILE}
+    [Arguments]    ${options}=    ${sources}=    ${default options}=${COMMON DEFAULTS}    ${output}=${OUTFILE}    ${validate output}=True
     [Documentation]    *OUTDIR:* file://${OUTDIR} (regenerated for every run)
     ${result} =    Execute    ${INTERPRETER.rebot}   ${options}    ${sources}    ${default options}
     Log Many    RC: ${result.rc}    STDERR:\n${result.stderr}    STDOUT:\n${result.stdout}
-    Process Output    ${output}
+    Process Output    ${output}    validate=${validate output}
     [Return]    ${result}
 
 Run Rebot Without Processing Output
diff --git a/atest/robot/rebot/compatibility.robot b/atest/robot/rebot/compatibility.robot
@@ -7,7 +7,7 @@ Resource          rebot_resource.robot
 
 *** Test Cases ***
 RF 3.2 compatibility
-    Run Rebot And Validate Statistics    rebot/output-3.2.2.xml    172    10
+    Run Rebot And Validate Statistics    rebot/output-3.2.2.xml    172    10    validate=False
 
 RF 4.0 compatibility
     Run Rebot And Validate Statistics    rebot/output-4.0.xml      172    10
@@ -21,8 +21,8 @@ Message directly under test
 
 *** Keywords ***
 Run Rebot And Validate Statistics
-    [Arguments]    ${path}    ${passed}    ${failed}
-    Run Rebot    ${EMPTY}    ${path}
+    [Arguments]    ${path}    ${passed}    ${failed}    ${validate}=True
+    Run Rebot    ${EMPTY}    ${path}    validate output=${validate}
     ${total}    ${passed}    ${failed} =    Evaluate    ${passed} + ${failed}, ${passed}, ${failed}
     Should Be Equal    ${SUITE.statistics.total}     ${total}
     Should Be Equal    ${SUITE.statistics.passed}    ${passed}
