Skip to content

Commit 000d391

Browse files
pekkaklarckyanne
pekkaklarck
authored and
yanne
committed
Handle corner cases with skip. robotframework#3622
Changes: - More tests for skip in setup. - More tests and implementation for skip with teardown including running teardown after skip, using skip in teardown, skip in teardown after failure in body, and failure in teardown after skip in body. - Tests and support for continuable failure before skip. - Tests and support for pass execution in combination with skip. - Fixes for skip in teardown.
1 parent 3a9a0c4 commit 000d391

File tree

5 files changed

+171
-46
lines changed

5 files changed

+171
-46
lines changed

‎atest/robot/running/skip.robot

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,42 @@ Skip If Keyword with False Condition
2121
Skip Keyword with Custom Message
2222
Check Test Case ${TEST NAME}
2323

24-
Skipped in Setup
25-
Check Test Case ${TEST NAME} 1
26-
Check Test Case ${TEST NAME} 2
24+
Skip in Setup
25+
Check Test Case ${TEST NAME}
26+
27+
Remaining setup keywords aren't run after skip
28+
Check Test Case ${TEST NAME}
29+
30+
Skip in Teardown
31+
Check Test Case ${TEST NAME}
32+
33+
Remaining teardown keywords aren't run after skip
34+
Check Test Case ${TEST NAME}
35+
36+
Skip in Teardown After Failure In Body
37+
Check Test Case ${TEST NAME}
38+
39+
Teardown is executed after skip
40+
${tc} = Check Test Case ${TEST NAME}
41+
Check log message ${tc.teardown.msgs[0]} Teardown is executed!
42+
43+
Fail in Teardown After Skip In Body
44+
Check Test Case ${TEST NAME}
45+
46+
Skip in Teardown After Skip In Body
47+
Check Test Case ${TEST NAME}
48+
49+
Skip with Continuable Failures
50+
Check Test Case ${TEST NAME}
51+
52+
Skip with Pass Execution in Teardown
53+
Check Test Case ${TEST NAME}
54+
55+
Skip in Teardown with Pass Execution in Body
56+
Check Test Case ${TEST NAME}
57+
58+
Skip in Teardown After Continuable Failures
59+
Check Test Case ${TEST NAME}
2760

2861
Skip in Suite Setup
2962
Check Test Case ${TEST NAME}

‎atest/testdata/running/skip/skip.robot

Lines changed: 67 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,73 @@ Skip Keyword with Custom Message
3333
Skip Skipped due to reasons
3434
Fail Should not be executed!
3535

36-
Skipped in Setup 1
36+
Skip in Setup
3737
[Documentation] SKIP Setup skip
3838
[Setup] Skip Setup skip
3939
Fail Should not be executed!
4040

41-
Skipped in Setup 2
42-
[Documentation] SKIP Setup skip
43-
[Setup] Run Keywords
44-
... No Operation AND
45-
... Skip Setup skip AND
46-
... Fail Should not be executed!
41+
Remaining setup keywords aren't run after skip
42+
[Documentation] SKIP Skip between keywords
43+
[Setup] Skip with keywords before and after
4744
Fail Should not be executed!
4845

46+
Skip in Teardown
47+
[Documentation] SKIP Teardown skip
48+
No operation
49+
[Teardown] Skip Teardown skip
50+
51+
Remaining teardown keywords aren't run after skip
52+
[Documentation] SKIP Skip between keywords
53+
No operation
54+
[Teardown] Skip with keywords before and after
55+
56+
Skip in Teardown After Failure In Body
57+
[Documentation] SKIP Skipped in teardown:\nTeardown skip\n\nEarlier message:\nFailure in body!
58+
Fail Failure in body!
59+
[Teardown] Skip Teardown skip
60+
61+
Teardown is executed after skip
62+
[Documentation] SKIP Skip in body
63+
Skip Skip in body
64+
[Teardown] Log Teardown is executed!
65+
66+
Fail in Teardown After Skip In Body
67+
[Documentation] SKIP Skip in body\n\nAlso teardown failed:\nTeardown failed!
68+
Skip Skip in body
69+
[Teardown] Fail Teardown failed!
70+
71+
Skip in Teardown After Skip In Body
72+
[Documentation] SKIP Skipped in teardown:\nTeardown skip\n\nEarlier message:\nSkip in body
73+
Skip Skip in body
74+
[Teardown] Skip Teardown skip
75+
76+
Skip with Continuable Failures
77+
[Documentation] SKIP Skipping should stop execution but test should still fail\n\nAlso failures occurred\n\n1) We can continue!\n\n2) We can continue again!
78+
Run Keyword And Continue On Failure
79+
... Fail We can continue!
80+
Run Keyword And Continue On Failure
81+
... Fail We can continue again!
82+
Skip Skipping should stop execution but test should still fail
83+
Fail Should not be executed!
84+
85+
Skip in Teardown After Continuable Failures
86+
[Documentation] SKIP Skipped in teardown:\nTeardown skip\n\nEarlier message:\nSeveral failures occurred:\n\n1) We can continue!\n\n2) We can continue again!
87+
Run Keyword And Continue On Failure
88+
... Fail We can continue!
89+
Run Keyword And Continue On Failure
90+
... Fail We can continue again!
91+
[Teardown] Skip Teardown skip
92+
93+
Skip with Pass Execution in Teardown
94+
[Documentation] SKIP Skip in body
95+
[Teardown] Pass Execution Thou shall pass
96+
Skip Skip in body
97+
98+
Skip in Teardown with Pass Execution in Body
99+
[Documentation] SKIP Then we skip
100+
[Teardown] Skip Then we skip
101+
Pass Execution First we pass
102+
49103
Skipped with --skip
50104
[Tags] skip-this
51105
[Documentation] SKIP ${TEST_OR_TASK} skipped with --skip command line option.
@@ -72,3 +126,9 @@ Failing Test
72126

73127
Passing Test
74128
No Operation
129+
130+
*** Keywords ***
131+
Skip with keywords before and after
132+
No Operation
133+
Skip Skip between keywords
134+
Fail Should not be executed!

‎src/robot/errors.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ def continue_on_failure(self, continue_on_failure):
143143
def can_continue(self, teardown=False, templated=False, dry_run=False):
144144
if dry_run:
145145
return True
146-
if self.syntax or self.exit or self.test_timeout:
146+
if self.syntax or self.exit or self.skip or self.test_timeout:
147147
return False
148148
if templated:
149149
return True
@@ -190,17 +190,23 @@ def _get(self, error, attr):
190190
class ExecutionFailures(ExecutionFailed):
191191

192192
def __init__(self, errors, message=None):
193-
message = message or self._format_message([e.message for e in errors])
193+
message = message or self._format_message(errors)
194194
ExecutionFailed.__init__(self, message, **self._get_attrs(errors))
195195
self._errors = errors
196196

197-
def _format_message(self, messages):
197+
def _format_message(self, errors):
198+
messages = [e.message for e in errors]
198199
if len(messages) == 1:
199200
return messages[0]
200201
prefix = 'Several failures occurred:'
201202
if any(msg.startswith('*HTML*') for msg in messages):
202203
prefix = '*HTML* ' + prefix
203204
messages = self._format_html_messages(messages)
205+
if any(e.skip for e in errors):
206+
skip_idx = errors.index([e for e in errors if e.skip][0])
207+
skip_msg = messages[skip_idx]
208+
messages = messages[:skip_idx] + messages[skip_idx+1:]
209+
prefix = '%s\n\nAlso failures occurred' % skip_msg
204210
return '\n\n'.join(
205211
[prefix] +
206212
['%d) %s' % (i, m) for i, m in enumerate(messages, start=1)]
@@ -221,7 +227,7 @@ def _get_attrs(self, errors):
221227
'syntax': any(e.syntax for e in errors),
222228
'exit': any(e.exit for e in errors),
223229
'continue_on_failure': all(e.continue_on_failure for e in errors),
224-
'skip': all(e.skip for e in errors)
230+
'skip': any(e.skip for e in errors)
225231
}
226232

227233
def get_errors(self):

‎src/robot/running/runner.py

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -155,10 +155,7 @@ def visit_test(self, test):
155155
else:
156156
result.message = exception.message
157157
except ExecutionStatus as err:
158-
if err.status == 'SKIP':
159-
status.test_skipped(err)
160-
else:
161-
status.test_failed(err)
158+
status.test_failed(err)
162159
result.status = status.status
163160
result.message = status.message or result.message
164161
if status.teardown_allowed:
@@ -199,11 +196,12 @@ def _run_teardown(self, teardown, status, result=None):
199196
if status.teardown_allowed:
200197
exception = self._run_setup_or_teardown(teardown)
201198
status.teardown_executed(exception)
202-
failed = (exception
203-
and not isinstance(exception, PassExecution)
204-
and not exception.skip)
199+
failed = exception and not isinstance(exception, PassExecution)
205200
if result and exception:
206-
result.message = status.message if failed else exception.message
201+
if failed or not result.passed:
202+
result.message = status.message
203+
else:
204+
result.message = exception.message
207205
return exception if failed else None
208206

209207
def _run_setup_or_teardown(self, data):

‎src/robot/running/status.py

Lines changed: 52 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,15 @@ def __init__(self):
2525
self.setup = None
2626
self.test = None
2727
self.teardown = None
28+
self.setup_skipped = None
29+
self.test_skipped = None
30+
self.teardown_skipped = None
2831

2932
def __nonzero__(self):
30-
return bool(self.setup or self.test or self.teardown)
33+
return bool(
34+
self.setup or self.test or self.teardown or
35+
self.setup_skipped or self.test_skipped or self.teardown_skipped
36+
)
3137

3238

3339
@py2to3
@@ -74,16 +80,24 @@ def __init__(self, parent=None, *exit_modes):
7480

7581
def setup_executed(self, failure=None):
7682
if failure and not isinstance(failure, PassExecution):
77-
self.failure.setup = unic(failure)
83+
if failure.skip:
84+
self.failure.setup_skipped = unic(failure)
85+
self.skipped = True
86+
else:
87+
self.failure.setup = unic(failure)
7888
self.exit.failure_occurred(failure)
79-
self.skipped = failure.skip
89+
8090
self._teardown_allowed = True
8191

8292
def teardown_executed(self, failure=None):
8393
if failure and not isinstance(failure, PassExecution):
84-
self.failure.teardown = unic(failure)
94+
if failure.skip:
95+
self.failure.teardown_skipped = unic(failure)
96+
# Keep the Skip status in case the teardown failed
97+
self.skipped = self.skipped or failure.skip
98+
else:
99+
self.failure.teardown = unic(failure)
85100
self.exit.failure_occurred(failure)
86-
# TODO: Handle skip in teardown!
87101

88102
def failure_occurred(self):
89103
self.exit.failure_occurred()
@@ -102,7 +116,7 @@ def failed(self):
102116

103117
@property
104118
def status(self):
105-
if self.skipped:
119+
if self.skipped or (self.parent and self.parent.skipped):
106120
return 'SKIP'
107121
if self.failed:
108122
return 'FAIL'
@@ -147,7 +161,9 @@ def __init__(self, parent, test, skip_on_failure=None, critical_tags=None,
147161
self._rpa = rpa
148162

149163
def test_failed(self, failure):
150-
if self._skip_on_failure:
164+
if hasattr(failure, 'skip') and failure.skip:
165+
self.test_skipped(failure)
166+
elif self._skip_on_failure:
151167
msg = ("%s skipped with --SkipOnFailure, original error:\n%s"
152168
% (test_or_task('{Test}', self._rpa) ,unic(failure)))
153169
self.failure.test = msg
@@ -156,6 +172,10 @@ def test_failed(self, failure):
156172
self.failure.test = unic(failure)
157173
self.exit.failure_occurred(failure)
158174

175+
def test_skipped(self, reason):
176+
self.skipped = True
177+
self.failure.test_skipped = unic(reason)
178+
159179
def _should_skip_on_failure(self, test, skip_on_failure_tags,
160180
critical_tags):
161181
critical_pattern = TagPatterns(critical_tags)
@@ -164,17 +184,14 @@ def _should_skip_on_failure(self, test, skip_on_failure_tags,
164184
skip_on_fail_pattern = TagPatterns(skip_on_failure_tags)
165185
return skip_on_fail_pattern and skip_on_fail_pattern.match(test.tags)
166186

167-
def test_skipped(self, reason):
168-
self.skipped = True
169-
self.failure.test = unic(reason)
170-
171187
def _my_message(self):
172188
return TestMessage(self).message
173189

174190

175191
class _Message(object):
176192
setup_message = NotImplemented
177-
skipped_message = NotImplemented
193+
setup_skipped_message = NotImplemented
194+
teardown_skipped_message = NotImplemented
178195
teardown_message = NotImplemented
179196
also_teardown_message = NotImplemented
180197

@@ -188,11 +205,13 @@ def message(self):
188205
return self._get_message_after_teardown(message)
189206

190207
def _get_message_before_teardown(self):
208+
if self.failure.setup_skipped:
209+
return self._format_setup_or_teardown_message(
210+
self.setup_skipped_message, self.failure.setup_skipped)
191211
if self.failure.setup:
192-
msg = self.setup_message if not self.skipped else self.skipped_message
193-
return self._format_setup_or_teardown_message(msg,
194-
self.failure.setup)
195-
return self.failure.test or ''
212+
return self._format_setup_or_teardown_message(
213+
self.setup_message, self.failure.setup)
214+
return self.failure.test_skipped or self.failure.test or ''
196215

197216
def _format_setup_or_teardown_message(self, prefix, message):
198217
if message.startswith('*HTML*'):
@@ -201,29 +220,36 @@ def _format_setup_or_teardown_message(self, prefix, message):
201220
return prefix % message
202221

203222
def _get_message_after_teardown(self, message):
204-
if not self.failure.teardown:
223+
if not (self.failure.teardown or self.failure.teardown_skipped):
205224
return message
206225
if not message:
207-
return self._format_setup_or_teardown_message(self.teardown_message,
208-
self.failure.teardown)
226+
if self.failure.teardown:
227+
prefix, msg = self.teardown_message, self.failure.teardown
228+
else:
229+
prefix, msg = self.teardown_skipped_message, self.failure.teardown_skipped
230+
return self._format_setup_or_teardown_message(prefix, msg)
209231
return self._format_message_with_teardown_message(message)
210232

211233
def _format_message_with_teardown_message(self, message):
212-
teardown = self.failure.teardown
234+
teardown = self.failure.teardown or self.failure.teardown_skipped
213235
if teardown.startswith('*HTML*'):
214236
teardown = teardown[6:].lstrip()
215237
if not message.startswith('*HTML*'):
216238
message = '*HTML* ' + html_escape(message)
217239
elif message.startswith('*HTML*'):
218240
teardown = html_escape(teardown)
219-
return self.also_teardown_message % (message, teardown)
241+
if self.failure.teardown:
242+
return self.also_teardown_message % (message, teardown)
243+
return self.also_teardown_skip_message % (teardown, message)
220244

221245

222246
class TestMessage(_Message):
223247
setup_message = 'Setup failed:\n%s'
224248
teardown_message = 'Teardown failed:\n%s'
225-
skipped_message = '%s'
249+
setup_skipped_message = '%s'
250+
teardown_skipped_message = '%s'
226251
also_teardown_message = '%s\n\nAlso teardown failed:\n%s'
252+
also_teardown_skip_message = 'Skipped in teardown:\n%s\n\nEarlier message:\n%s'
227253
exit_on_fatal_message = 'Test execution stopped due to a fatal error.'
228254
exit_on_failure_message = \
229255
'Failure occurred and exit-on-failure mode is in use.'
@@ -250,14 +276,16 @@ def message(self):
250276
class SuiteMessage(_Message):
251277
setup_message = 'Suite setup failed:\n%s'
252278
# TODO: wording
253-
skipped_message = 'Skipped in suite setup:\n%s'
279+
setup_skipped_message = 'Skipped in suite setup:\n%s'
280+
teardown_skipped_message = 'Skipped in suite teardown:\n%s'
254281
teardown_message = 'Suite teardown failed:\n%s'
255282
also_teardown_message = '%s\n\nAlso suite teardown failed:\n%s'
256283

257284

258285
class ParentMessage(SuiteMessage):
259286
setup_message = 'Parent suite setup failed:\n%s'
260-
skipped_message = 'Skipped in parent suite setup:\n%s'
287+
setup_skipped_message = 'Skipped in parent suite setup:\n%s'
288+
teardown_skipped_message = 'Skipped in parent suite teardown:\n%s'
261289
teardown_message = 'Parent suite teardown failed:\n%s'
262290
also_teardown_message = '%s\n\nAlso parent suite teardown failed:\n%s'
263291

0 commit comments

Comments
 (0)