diff --git a/atest/robot/running/continue_on_failure_tag.robot b/atest/robot/running/continue_on_failure_tag.robot new file mode 100644 index 00000000000..f1b411b15ec --- /dev/null +++ b/atest/robot/running/continue_on_failure_tag.robot @@ -0,0 +1,70 @@ +*** Settings *** +Suite Setup Run Tests ${EMPTY} running/continue_on_failure_tag.robot +Resource atest_resource.robot + +*** Test Cases *** +Continue in test with tag + Check Test Case ${TESTNAME} + +Continue in test with Set Tags + Check Test Case ${TESTNAME} + +Continue in user keyword with tag + Check Test Case ${TESTNAME} + +Continue in test with tag and UK without tag + Check Test Case ${TESTNAME} + +Continue in test with tag and nested UK with and without tag + Check Test Case ${TESTNAME} + +Continue in test with tag and two nested UK with tag + Check Test Case ${TESTNAME} + +Continue in FOR loop with tag + Check Test Case ${TESTNAME} + +Continue in FOR loop with Set Tags + Check Test Case ${TESTNAME} + +No continue in FOR loop without tag + Check Test Case ${TESTNAME} + +Continue in FOR loop in UK with tag + Check Test Case ${TESTNAME} + +Continue in FOR loop in UK without tag + Check Test Case ${TESTNAME} + +Continue in IF with tag + Check Test Case ${TESTNAME} + +Continue in IF with set and remove tag + Check Test Case ${TESTNAME} + +No continue in IF without tag + Check Test Case ${TESTNAME} + +Continue in IF in UK with tag + Check Test Case ${TESTNAME} + +No continue in IF in UK without tag + Check Test Case ${TESTNAME} + +Continue in Run Keywords with tag + Check Test Case ${TESTNAME} + +Recursive continue in test with tag and two nested UK without tag + Check Test Case ${TESTNAME} + +Recursive continue in test with Set Tags and two nested UK without tag + Check Test Case ${TESTNAME} + +Recursive continue in test with tag and two nested UK with and without tag + Check Test Case ${TESTNAME} + +Recursive continue in user keyword + Check Test Case ${TESTNAME} + +No recursive continue in user keyword + Check Test Case ${TESTNAME} diff --git a/atest/robot/tags/tag_stat_include_and_exclude.robot b/atest/robot/tags/tag_stat_include_and_exclude.robot index a7f8de6b916..a13b5ba9e46 100644 --- a/atest/robot/tags/tag_stat_include_and_exclude.robot +++ b/atest/robot/tags/tag_stat_include_and_exclude.robot @@ -5,6 +5,7 @@ Resource atest_resource.robot *** Variables *** ${DATA SOURCE} tags/include_and_exclude.robot ${F} force +${INTERNAL} robot:just-an-example ${I1} incl1 ${I2} incl 2 ${I3} incl_3 @@ -32,6 +33,14 @@ Include With Patterns --TagStatInc incl_? @{INCL} --TagStatInc *cl3 --TagStatInc i*2 ${E3} ${I2} ${I3} +Include to show internal tags + --tagstatinclude incl1 --tagstatinclude robot:* ${I1} ${INTERNAL} + --tagstatinclude robot:* ${INTERNAL} + --tagstatinclude * @{ALL} ${INTERNAL} + +Include and exclude internal + --tagstatinclude incl1 --tagstatinclude robot:* --tagstatexclude robot:* ${I1} + One Exclude --tagstatexclude excl1 ${E2} ${E3} ${F} @{INCL} diff --git a/atest/robot/tags/tag_stat_include_and_exclude_with_rebot.robot b/atest/robot/tags/tag_stat_include_and_exclude_with_rebot.robot index e1181aa11a3..8d1db4ebbeb 100644 --- a/atest/robot/tags/tag_stat_include_and_exclude_with_rebot.robot +++ b/atest/robot/tags/tag_stat_include_and_exclude_with_rebot.robot @@ -8,6 +8,7 @@ Resource rebot_resource.robot ${DATA SOURCE} tags/include_and_exclude.robot ${INPUT FILE} %{TEMPDIR}${/}robot-test-tagstat.xml ${F} force +${INTERNAL} robot:just-an-example ${I1} incl1 ${I2} incl 2 ${I3} incl_3 @@ -35,6 +36,14 @@ Include With Patterns --TagStatInc incl_? @{INCL} --TagStatInc *cl3 --TagStatInc i*2 ${E3} ${I2} ${I3} +Include to show internal tags + --tagstatinclude incl1 --tagstatinclude robot:* ${I1} ${INTERNAL} + --tagstatinclude robot:* ${INTERNAL} + --tagstatinclude * @{ALL} ${INTERNAL} + +Include and exclude internal + --tagstatinclude incl1 --tagstatinclude robot:* --tagstatexclude robot:* ${I1} + One Exclude --tagstatexclude excl1 ${E2} ${E3} ${F} @{INCL} diff --git a/atest/testdata/running/continue_on_failure_tag.robot b/atest/testdata/running/continue_on_failure_tag.robot new file mode 100644 index 00000000000..6143bbc6de3 --- /dev/null +++ b/atest/testdata/running/continue_on_failure_tag.robot @@ -0,0 +1,263 @@ +*** Variables *** +${HEADER} Several failures occurred: + +*** Test Cases *** +Continue in test with tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) 1\n\n + ... 2) 2 + [Tags] robot:continue-on-failure + Fail 1 + Fail 2 + Log This should be executed + +Continue in test with Set Tags + [Documentation] FAIL ${HEADER}\n\n + ... 1) 1\n\n + ... 2) 2 + Set Tags robot:continue-on-failure + Fail 1 + Fail 2 + Log This should be executed + +Continue in user keyword with tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) kw1a\n\n + ... 2) kw1b + Failure in user keyword with tag + Fail This should not be executed + +Continue in test with tag and UK without tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) kw2a\n\n + ... 2) This should be executed + [Tags] robot:continue-on-failure + Failure in user keyword without tag + Fail This should be executed + +Continue in test with tag and nested UK with and without tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) kw1a\n\n + ... 2) kw1b\n\n + ... 3) kw2a\n\n + ... 4) This should be executed + [Tags] robot:continue-on-failure + Failure in user keyword with tag run_kw=Failure in user keyword without tag + Fail This should be executed + +Continue in test with tag and two nested UK with tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) kw1a\n\n + ... 2) kw1b\n\n + ... 3) kw1a\n\n + ... 4) kw1b\n\n + ... 5) This should be executed + [Tags] robot:continue-on-failure + Failure in user keyword with tag run_kw=Failure in user keyword with tag + Fail This should be executed + +Continue in FOR loop with tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) loop-1\n\n + ... 2) loop-2\n\n + ... 3) loop-3 + [Tags] robot:continue-on-failure + FOR ${val} IN 1 2 3 + Fail loop-${val} + END + +Continue in FOR loop with Set Tags + [Documentation] FAIL ${HEADER}\n\n + ... 1) loop-1\n\n + ... 2) loop-2\n\n + ... 3) loop-3 + FOR ${val} IN 1 2 3 + Set Tags robot:continue-on-failure + Fail loop-${val} + END + +No continue in FOR loop without tag + [Documentation] FAIL loop-1 + FOR ${val} IN 1 2 3 + Fail loop-${val} + END + +Continue in FOR loop in UK with tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) kw-loop-1\n\n + ... 2) kw-loop-2\n\n + ... 3) kw-loop-3 + FOR loop in in user keyword with tag + +Continue in FOR loop in UK without tag + [Documentation] FAIL kw-loop-1 + FOR loop in in user keyword without tag + +Continue in IF with tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) 1\n\n + ... 2) 2\n\n + ... 3) 3\n\n + ... 4) 4 + [Tags] robot:continue-on-failure + IF 1==1 + Fail 1 + Fail 2 + END + IF 1==2 + No Operation + ELSE + Fail 3 + Fail 4 + END + +Continue in IF with set and remove tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) 1\n\n + ... 2) 2\n\n + ... 3) 3 + Set Tags robot:continue-on-failure + IF 1==1 + Fail 1 + Fail 2 + END + Remove Tags robot:continue-on-failure + IF 1==2 + No Operation + ELSE + Fail 3 + Fail this is not executed + END + +No continue in IF without tag + [Documentation] FAIL 1 + IF 1==1 + Fail 1 + Fail This should not be executed + END + +Continue in IF in UK with tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) kw1a\n\n + ... 2) kw1b\n\n + ... 3) kw1c\n\n + ... 4) kw1d + IF in user keyword with tag + +No continue in IF in UK without tag + [Documentation] FAIL kw1a + IF in user keyword without tag + +Continue in Run Keywords with tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) 1\n\n + ... 2) 2 + [Tags] robot:continue-on-failure + Run Keywords Fail 1 AND Fail 2 + +Recursive continue in test with tag and two nested UK without tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) kw2a\n\n + ... 2) kw2b\n\n + ... 3) kw2a\n\n + ... 4) kw2b\n\n + ... 5) This should be executed + [Tags] robot:continue-on-failure-recursive + Failure in user keyword without tag run_kw=Failure in user keyword without tag + Fail This should be executed + +Recursive continue in test with Set Tags and two nested UK without tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) kw2a\n\n + ... 2) kw2b\n\n + ... 3) kw2a\n\n + ... 4) kw2b\n\n + ... 5) This should be executed + Set Tags robot:continue-on-failure-recursive + Failure in user keyword without tag run_kw=Failure in user keyword without tag + Fail This should be executed + +Recursive continue in test with tag and two nested UK with and without tag + [Documentation] FAIL ${HEADER}\n\n + ... 1) kw1a\n\n + ... 2) kw1b\n\n + ... 3) kw2a\n\n + ... 4) kw2b\n\n + ... 5) This should be executed + [Tags] robot:continue-on-failure-recursive + Failure in user keyword with tag run_kw=Failure in user keyword without tag + Fail This should be executed + +Recursive continue in user keyword + [Documentation] FAIL ${HEADER}\n\n + ... 1) kw1a\n\n + ... 2) kw1b\n\n + ... 3) kw2a\n\n + ... 4) kw2b + Failure in user keyword with recursive tag run_kw=Failure in user keyword without tag + Fail This should not be executed + +No recursive continue in user keyword + [Documentation] FAIL kw2a + Failure in user keyword without tag run_kw=Failure in user keyword with recursive tag + Fail This should not be executed + +*** Keywords *** + +Failure in user keyword with tag + [Arguments] ${run_kw}=No Operation + [Tags] robot:continue-on-failure + Fail kw1a + Fail kw1b + Log This should be executed + Run Keyword ${run_kw} + +Failure in user keyword without tag + [Arguments] ${run_kw}=No Operation + Fail kw2a + Fail kw2b + Run Keyword ${run_kw} + +Failure in user keyword with recursive tag + [Arguments] ${run_kw}=No Operation + [Tags] robot:continue-on-failure-recursive + Fail kw1a + Fail kw1b + Log This should be executed + Run Keyword ${run_kw} + +FOR loop in in user keyword with tag + [Tags] robot:continue-on-failure + FOR ${val} IN 1 2 3 + Fail kw-loop-${val} + END + +FOR loop in in user keyword without tag + FOR ${val} IN 1 2 3 + Fail kw-loop-${val} + END + +IF in user keyword with tag + [Tags] robot:continue-on-failure + IF 1==1 + Fail kw1a + Fail kw1b + END + IF 1==2 + No Operation + ELSE + Fail kw1c + Fail kw1d + END + +IF in user keyword without tag + IF 1==1 + Fail kw1a + Fail kw1b + END + IF 1==2 + No Operation + ELSE + Fail kw1c + Fail kw1d + END diff --git a/atest/testdata/tags/include_and_exclude.robot b/atest/testdata/tags/include_and_exclude.robot index 9ec3fedd1f2..20c624b0721 100644 --- a/atest/testdata/tags/include_and_exclude.robot +++ b/atest/testdata/tags/include_and_exclude.robot @@ -1,5 +1,5 @@ *** Settings *** -Force Tags force +Force Tags force robot:just-an-example *** Test Cases *** Incl-1 diff --git a/doc/userguide/src/CreatingTestData/CreatingTestCases.rst b/doc/userguide/src/CreatingTestData/CreatingTestCases.rst index bebdc927606..9e8d9432ebd 100644 --- a/doc/userguide/src/CreatingTestData/CreatingTestCases.rst +++ b/doc/userguide/src/CreatingTestData/CreatingTestCases.rst @@ -687,9 +687,14 @@ tag with this prefixes unless actually activating the special functionality. At the time of writing, the only special tags are `robot:exit`, that is automatically added to tests when `stopping test execution gracefully`_, -and `robot:no-dry-run`, that can be used to disable the `dry run`_ mode. +and `robot:no-dry-run`, that can be used to disable the `dry run`_ mode as +well as `robot:continue-on-failure` which controls continuable execution. More usages are likely to be added in the future. +As of RobotFramework 4.1, reserved tags are suppressed by default in the +test suite's tag statistics. They will be shown when they are explicitly +included via the `--tagstatinclude 'robot:*'` command line option. + Test setup and teardown ----------------------- diff --git a/doc/userguide/src/ExecutingTestCases/TestExecution.rst b/doc/userguide/src/ExecutingTestCases/TestExecution.rst index 214d8da5d55..8ca16cc885a 100644 --- a/doc/userguide/src/ExecutingTestCases/TestExecution.rst +++ b/doc/userguide/src/ExecutingTestCases/TestExecution.rst @@ -364,6 +364,86 @@ converting any failure into a continuable failure. These failures are handled by the framework exactly the same way as continuable failures originating from library keywords. +Controlling continue on failure using reserved tags +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All keywords executed as part of test cases or user keywords which are +tagged with the reserved tag `robot:continue-on-failure` are considered continuable +by default. + +Thus, the following two test cases :name:`Test 1` and :name:`Test 2` behave identically: + +.. sourcecode:: robotframework + + *** Test Cases *** + Test 1 + Run Keyword and Continue on Failure Should be Equal 1 2 + User Keyword 1 + + Test 2 + [Tags] robot:continue-on-failure + Should be Equal 1 2 + User Keyword 2 + + *** Keywords *** + User Keyword 1 + Run Keyword and Continue on Failure Should be Equal 3 4 + Log this message is logged + + User Keyword 2 + [Tags] robot:continue-on-failure + Should be Equal 3 4 + Log this message is logged + + +These tags also influence continue-on-failure in FOR loops and +within IF/ELSE branches. +The below test case will execute the test 10 times, no matter if +the "Perform some test keyword" failed or not. + +.. sourcecode:: robotframework + + *** Test Cases *** + Test Case + [Tags] robot:continue-on-failure + FOR ${index} IN RANGE 10 + Perform some test + END + + +Setting `robot:continue-on-failure` within a test case will not +propagate the continue on failure behaviour into user keywords +executed from within this test case (same is true for user keywords +executed from within a user keyword with the reserved tag set). + +To support use cases where the behaviour should propagate from +test cases into user keywords (and/or from user keywords into other +user keywords), the reserved tag `robot:continue-on-failure-recursive` +can be used. The below examples executes all the keywords listed. + +.. sourcecode:: robotframework + + *** Test Cases *** + Test + [Tags] robot:continue-on-failure-recursive + Should be Equal 1 2 + User Keyword 1 + Log log from test case + + *** Keywords *** + User Keyword 1 + Should be Equal 3 4 + Log log from keyword 1 + User Keyword 2 + + User Keyword 2 + Should be Equal 5 6 + Log log from keyword 2 + + +The `robot:continue-on-failure` and `robot:continue-on-failure-recursive` +tags are new in Robot Framework 4.1. + Execution continues on teardowns automatically ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/robot/errors.py b/src/robot/errors.py index 6c922cc8bb9..d5387b0e658 100644 --- a/src/robot/errors.py +++ b/src/robot/errors.py @@ -140,8 +140,8 @@ def continue_on_failure(self, continue_on_failure): if child is not self: child.continue_on_failure = continue_on_failure - def can_continue(self, teardown=False, templated=False, dry_run=False): - if dry_run: + def can_continue(self, context, templated=False): + if context.dry_run: return True if self.syntax or self.exit or self.skip or self.test_timeout: return False @@ -149,7 +149,7 @@ def can_continue(self, teardown=False, templated=False, dry_run=False): return True if self.keyword_timeout: return False - if teardown: + if context.in_teardown or context.continue_on_failure: return True return self.continue_on_failure diff --git a/src/robot/libraries/BuiltIn.py b/src/robot/libraries/BuiltIn.py index 6dd92b8a6ba..bc980128e68 100644 --- a/src/robot/libraries/BuiltIn.py +++ b/src/robot/libraries/BuiltIn.py @@ -1815,7 +1815,7 @@ def _run_keywords(self, iterable): raise err except ExecutionFailed as err: errors.extend(err.get_errors()) - if not err.can_continue(self._context.in_teardown): + if not err.can_continue(self._context): break if errors: raise ExecutionFailures(errors) diff --git a/src/robot/model/tagstatistics.py b/src/robot/model/tagstatistics.py index 427163d5593..bdecaa8ff05 100644 --- a/src/robot/model/tagstatistics.py +++ b/src/robot/model/tagstatistics.py @@ -56,7 +56,7 @@ def add_test(self, test): def _add_tags_to_statistics(self, test): for tag in test.tags: - if self._is_included(tag): + if self._is_included(tag) and not self._suppress_reserved(tag): if tag not in self.stats.tags: self.stats.tags[tag] = self._info.get_stat(tag) self.stats.tags[tag].add_test(test) @@ -71,6 +71,10 @@ def _add_to_combined_statistics(self, test): if stat.match(test.tags): stat.add_test(test) + def _suppress_reserved(self, tag): + # don't suppress reserved tags if the user explicitly included them + return tag.startswith('robot:') and not self._included.match(tag) + class TagStatInfo(object): diff --git a/src/robot/running/bodyrunner.py b/src/robot/running/bodyrunner.py index 08cf4ab784e..901a6d4de5b 100644 --- a/src/robot/running/bodyrunner.py +++ b/src/robot/running/bodyrunner.py @@ -45,9 +45,8 @@ def run(self, body): raise exception except ExecutionFailed as exception: errors.extend(exception.get_errors()) - self._run = exception.can_continue(self._context.in_teardown, - self._templated, - self._context.dry_run) + self._run = exception.can_continue(self._context, + self._templated) if errors: raise ExecutionFailures(errors) @@ -175,9 +174,8 @@ def _run_loop(self, data, result): raise exception except ExecutionFailed as exception: errors.extend(exception.get_errors()) - if not exception.can_continue(self._context.in_teardown, - self._templated, - self._context.dry_run): + if not exception.can_continue(self._context, + self._templated): break if errors: raise ExecutionFailures(errors) diff --git a/src/robot/running/context.py b/src/robot/running/context.py index b4a17dac2d3..aac1bfce9ac 100644 --- a/src/robot/running/context.py +++ b/src/robot/running/context.py @@ -67,6 +67,7 @@ def __init__(self, suite, namespace, output, dry_run=False): self.in_keyword_teardown = 0 self._started_keywords = 0 self.timeout_occurred = False + self.user_keywords = [] @contextmanager def suite_teardown(self): @@ -97,14 +98,15 @@ def keyword_teardown(self, error): finally: self.in_keyword_teardown -= 1 - @property @contextmanager - def user_keyword(self): + def user_keyword(self, handler): + self.user_keywords.append(handler) self.namespace.start_user_keyword() try: yield finally: self.namespace.end_user_keyword() + self.user_keywords.pop() @contextmanager def timeout(self, timeout): @@ -124,6 +126,15 @@ def in_teardown(self): def variables(self): return self.namespace.variables + @property + def continue_on_failure(self): + parents = ([self.test] if self.test else []) + self.user_keywords + if not parents: + return False + if 'robot:continue-on-failure' in parents[-1].tags: + return True + return any('robot:continue-on-failure-recursive' in p.tags for p in parents) + def end_suite(self, suite): for name in ['${PREV_TEST_NAME}', '${PREV_TEST_STATUS}', diff --git a/src/robot/running/userkeywordrunner.py b/src/robot/running/userkeywordrunner.py index 161ec113dc8..d760a5b8a4b 100644 --- a/src/robot/running/userkeywordrunner.py +++ b/src/robot/running/userkeywordrunner.py @@ -75,14 +75,14 @@ def _get_result(self, kw, assignment, variables): def _run(self, context, args, result): variables = context.variables args = self._resolve_arguments(args, variables) - with context.user_keyword: + with context.user_keyword(self._handler): self._set_arguments(args, context) timeout = self._get_timeout(variables) if timeout is not None: result.timeout = str(timeout) with context.timeout(timeout): exception, return_ = self._execute(context) - if exception and not exception.can_continue(context.in_teardown): + if exception and not exception.can_continue(context): raise exception return_value = self._get_return_value(variables, return_) if exception: @@ -213,7 +213,7 @@ def dry_run(self, kw, context): def _dry_run(self, context, args, result): self._resolve_arguments(args) - with context.user_keyword: + with context.user_keyword(self._handler): timeout = self._get_timeout() if timeout: result.timeout = str(timeout) diff --git a/src/robot/variables/assigner.py b/src/robot/variables/assigner.py index 63b9d6c01fa..a99fe4f9b15 100644 --- a/src/robot/variables/assigner.py +++ b/src/robot/variables/assigner.py @@ -95,7 +95,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): if exc_val is None: return failure = self._get_failure(exc_type, exc_val, exc_tb) - if failure.can_continue(self._context.in_teardown): + if failure.can_continue(self._context): self.assign(failure.return_value) def _get_failure(self, exc_type, exc_val, exc_tb):