Skip to content

Custom Test Settings a.k.a. Test Metadata #5281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions atest/robot/parsing/test_metadata.robot
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
*** Settings ***
Documentation Tests for --metadata are located in robot/cli/runner and
... for other suite settings in suite_settings.robot.
Suite Setup Run Tests --variable meta_value_from_cli:my_metadata parsing/test_metadata.robot
Test Template Validate metadata
Resource atest_resource.robot

*** Test Cases ***
Metadata
Name Value
name Value
NAME Value

Metadata In Multiple Columns
Multiple columns Value in${SPACE*4}multiple${SPACE*4}columns

Metadata In Multiple Lines
Multiple lines Metadata in multiple lines
... is parsed using
... same semantics${SPACE*4}as${SPACE*4}documentation.
... | table |
... |${SPACE*3}!${SPACE*3}|

Metadata With Variables
Variables Version: 1.2

Metadata With Variable From Resource
Variable from resource Variable from a resource file

Metadata With Variable From Commandline
Value from CLI my_metadata

Using Same Name Twice
Overridden This overrides first value

Unescaping Metadata In Setting Table
Escaping Three backslashes \\\\\\ & \${version}

*** Keywords ***
Validate metadata
[Arguments] ${name} @{lines}
${value} = Catenate SEPARATOR=\n @{lines}
Should be Equal ${TEST.metadata['${name}']} ${value}
24 changes: 24 additions & 0 deletions atest/testdata/parsing/test_metadata.robot
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
*** Settings ***
Resource ../core/resources.robot



*** Variables ***
${version} 1.2

*** Test Cases ***
Test Case
[Metadata] Name Value
[Metadata] Multiple columns Value in multiple columns
[Metadata] multiple lines Metadata in multiple lines
... is parsed using
... same semantics as documentation.
... | table |
... | ! |
[Metadata] variables Version: ${version}
[Metadata] Variable from resource ${resource_file_var}
[Metadata] Value from CLI ${META_VALUE_FROM_CLI}
[Metadata] Escaping Three backslashes \\\\\\ & \${version}
[Metadata] Overridden first value
[Metadata] over ridden This overrides first value
No Operation
15 changes: 14 additions & 1 deletion src/robot/conf/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
from datetime import datetime
from pathlib import Path

import toml

from robot.errors import DataError, FrameworkError
from robot.output import LOGGER, LogLevel
from robot.result.keywordremover import KeywordRemover
Expand Down Expand Up @@ -141,6 +143,8 @@ def _process_value(self, name, value):
self._validate_expandkeywords(value)
if name == 'Extension':
return tuple('.' + ext.lower().lstrip('.') for ext in value.split(':'))
if name == 'Customsetting':
return self._process_custom_settings(value)
return value

def _process_doc(self, value):
Expand Down Expand Up @@ -359,6 +363,9 @@ def _validate_expandkeywords(self, values):

def _raise_invalid(self, option, error):
raise DataError(f"Invalid value for option '--{option.lower()}': {error}")

def _process_custom_settings(self, name):
return name

def __contains__(self, setting):
return setting in self._opts
Expand Down Expand Up @@ -496,7 +503,8 @@ class RobotSettings(_BaseSettings):
'ConsoleWidth' : ('consolewidth', 78),
'ConsoleMarkers' : ('consolemarkers', 'AUTO'),
'DebugFile' : ('debugfile', None),
'Language' : ('language', [])}
'Language' : ('language', []),
'CustomSettings' : ('customsettings', None)}
_languages = None

def get_rebot_settings(self):
Expand Down Expand Up @@ -549,6 +557,7 @@ def suite_config(self):
'randomize_suites': self.randomize_suites,
'randomize_tests': self.randomize_tests,
'randomize_seed': self.randomize_seed,
'custom_settings': self.custom_settings
}

@property
Expand Down Expand Up @@ -672,6 +681,10 @@ def variable_files(self):
@property
def extension(self):
return self['Extension']

@property
def custom_settings(self):
return self['CustomSettings']


class RebotSettings(_BaseSettings):
Expand Down
18 changes: 18 additions & 0 deletions src/robot/htmldata/rebot/log.html
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,12 @@ <h2>{{= testOrTask('{Test}')}} Execution Errors</h2>
<td class="doc">{{html $value[1]}}</td>
</tr>
{{/each}}
{{each custom_setting}}
<tr>
<th>{{html $value[0]}}:</th>
<td class="doc">{{html $value[1]}}</td>
</tr>
{{/each}}
{{if source}}
<tr>
<th>Source:</th>
Expand Down Expand Up @@ -280,6 +286,18 @@ <h2>{{= testOrTask('{Test}')}} Execution Errors</h2>
<td>{{html timeout}}</td>
</tr>
{{/if}}
{{each metadata}}
<tr>
<th>{{html $value[0]}}:</th>
<td class="doc">{{html $value[1]}}</td>
</tr>
{{/each}}
{{each custom_setting}}
<tr>
<th>{{html $value[0]}}:</th>
<td class="doc">{{html $value[1]}}</td>
</tr>
{{/each}}
<tr>
<th>Start / End / Elapsed:</th>
<td>${times.startTime} / ${times.endTime} / ${times.elapsedTime}</td>
Expand Down
1 change: 1 addition & 0 deletions src/robot/htmldata/rebot/model.js
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ window.model = (function () {
return test.keywords();
};
test.tags = data.tags;
test.metadata = data.metadata;
test.matchesTagPattern = function (pattern) {
return containsTagPattern(test.tags, pattern);
};
Expand Down
3 changes: 2 additions & 1 deletion src/robot/htmldata/rebot/testdata.js
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,8 @@ window.testdata = function () {
},
times: model.Times(times(status)),
tags: tags(element[3], strings),
isChildrenLoaded: typeof(element[5]) !== 'number'
isChildrenLoaded: typeof(element[5]) !== 'number',
metadata: parseMetadata(element[6], strings)
});
lazyPopulateKeywordsFromFile(test, element[5], strings);
return test;
Expand Down
187 changes: 94 additions & 93 deletions src/robot/model/configurer.py
Original file line number Diff line number Diff line change
@@ -1,93 +1,94 @@
# Copyright 2008-2015 Nokia Networks
# Copyright 2016- Robot Framework Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from robot.utils import seq2str
from robot.errors import DataError

from .visitor import SuiteVisitor


class SuiteConfigurer(SuiteVisitor):

def __init__(self, name=None, doc=None, metadata=None, set_tags=None,
include_tags=None, exclude_tags=None, include_suites=None,
include_tests=None, empty_suite_ok=False):
self.name = name
self.doc = doc
self.metadata = metadata
self.set_tags = set_tags or []
self.include_tags = include_tags
self.exclude_tags = exclude_tags
self.include_suites = include_suites
self.include_tests = include_tests
self.empty_suite_ok = empty_suite_ok

@property
def add_tags(self):
return [t for t in self.set_tags if not t.startswith('-')]

@property
def remove_tags(self):
return [t[1:] for t in self.set_tags if t.startswith('-')]

def visit_suite(self, suite):
self._set_suite_attributes(suite)
self._filter(suite)
suite.set_tags(self.add_tags, self.remove_tags)

def _set_suite_attributes(self, suite):
if self.name:
suite.name = self.name
if self.doc:
suite.doc = self.doc
if self.metadata:
suite.metadata.update(self.metadata)

def _filter(self, suite):
name = suite.name
suite.filter(self.include_suites, self.include_tests,
self.include_tags, self.exclude_tags)
if not (suite.has_tests or self.empty_suite_ok):
self._raise_no_tests_or_tasks_error(name, suite.rpa)

def _raise_no_tests_or_tasks_error(self, name, rpa):
parts = [{False: 'tests', True: 'tasks', None: 'tests or tasks'}[rpa],
self._get_test_selector_msgs(),
self._get_suite_selector_msg()]
raise DataError(f"Suite '{name}' contains no "
f"{' '.join(p for p in parts if p)}.")

def _get_test_selector_msgs(self):
parts = []
for separator, explanation, selectors in [
(None, 'matching name', self.include_tests),
('and', 'matching tags', self.include_tags),
('and', 'not matching tags', self.exclude_tags)
]:
if selectors:
if parts:
parts.append(separator)
parts.append(self._format_selector_msg(explanation, selectors))
return ' '.join(parts)

def _format_selector_msg(self, explanation, selectors):
if len(selectors) == 1 and explanation[-1] == 's':
explanation = explanation[:-1]
return f"{explanation} {seq2str(selectors, lastsep=' or ')}"

def _get_suite_selector_msg(self):
if not self.include_suites:
return ''
return self._format_selector_msg('in suites', self.include_suites)
# Copyright 2008-2015 Nokia Networks
# Copyright 2016- Robot Framework Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from robot.utils import seq2str
from robot.errors import DataError

from .visitor import SuiteVisitor


class SuiteConfigurer(SuiteVisitor):

def __init__(self, name=None, doc=None, metadata=None, set_tags=None,
include_tags=None, exclude_tags=None, include_suites=None,
include_tests=None, empty_suite_ok=False, custom_settings=None):
self.name = name
self.doc = doc
self.metadata = metadata
self.set_tags = set_tags or []
self.include_tags = include_tags
self.exclude_tags = exclude_tags
self.include_suites = include_suites
self.include_tests = include_tests
self.empty_suite_ok = empty_suite_ok
self.custom_settings = custom_settings

@property
def add_tags(self):
return [t for t in self.set_tags if not t.startswith('-')]

@property
def remove_tags(self):
return [t[1:] for t in self.set_tags if t.startswith('-')]

def visit_suite(self, suite):
self._set_suite_attributes(suite)
self._filter(suite)
suite.set_tags(self.add_tags, self.remove_tags)

def _set_suite_attributes(self, suite):
if self.name:
suite.name = self.name
if self.doc:
suite.doc = self.doc
if self.metadata:
suite.metadata.update(self.metadata)

def _filter(self, suite):
name = suite.name
suite.filter(self.include_suites, self.include_tests,
self.include_tags, self.exclude_tags)
if not (suite.has_tests or self.empty_suite_ok):
self._raise_no_tests_or_tasks_error(name, suite.rpa)

def _raise_no_tests_or_tasks_error(self, name, rpa):
parts = [{False: 'tests', True: 'tasks', None: 'tests or tasks'}[rpa],
self._get_test_selector_msgs(),
self._get_suite_selector_msg()]
raise DataError(f"Suite '{name}' contains no "
f"{' '.join(p for p in parts if p)}.")

def _get_test_selector_msgs(self):
parts = []
for separator, explanation, selectors in [
(None, 'matching name', self.include_tests),
('and', 'matching tags', self.include_tags),
('and', 'not matching tags', self.exclude_tags)
]:
if selectors:
if parts:
parts.append(separator)
parts.append(self._format_selector_msg(explanation, selectors))
return ' '.join(parts)

def _format_selector_msg(self, explanation, selectors):
if len(selectors) == 1 and explanation[-1] == 's':
explanation = explanation[:-1]
return f"{explanation} {seq2str(selectors, lastsep=' or ')}"

def _get_suite_selector_msg(self):
if not self.include_suites:
return ''
return self._format_selector_msg('in suites', self.include_suites)
Loading