Skip to content

Commit e10e769

Browse files
tiurindominikschubert
tiurin
and
dominikschubert
authored
Record validated tests duration (#12638)
Co-authored-by: Dominik Schubert <dominik.schubert91@gmail.com>
1 parent b90f172 commit e10e769

File tree

4 files changed

+84
-47
lines changed

4 files changed

+84
-47
lines changed

‎docs/testing/parity-testing/README.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,3 +248,12 @@ localstack.testing.snapshots.transformer: Replacing regex '000000000000' with '1
248248
localstack.testing.snapshots.transformer: Replacing regex 'us-east-1' with '<region>'
249249
localstack.testing.snapshots.transformer: Replacing '1ad533b5-ac54-4354-a273-3ea885f0d59d' in snapshot with '<uuid:1>'
250250
```
251+
252+
### Test duration recording
253+
254+
When a test runs successfully against AWS, its last validation date and duration are recorded in a corresponding ***.validation.json** file.
255+
The validation date is recorded precisely, while test durations can vary between runs.
256+
For example, test setup time may differ depending on whether a test runs in isolation or as part of a class test suite with class-level fixtures.
257+
The recorded durations should be treated as approximate indicators of test execution time rather than precise measurements.
258+
The goal of duration recording is to give _an idea_ about execution times.
259+
If no duration is present in the validation file, it means the test has not been re-validated against AWS since duration recording was implemented.

‎localstack-core/localstack/testing/pytest/validation_tracking.py

Lines changed: 67 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,28 @@
99
import json
1010
import os
1111
from pathlib import Path
12-
from typing import Optional
12+
from typing import Dict, Optional
1313

14-
import pluggy
1514
import pytest
15+
from pluggy import Result
16+
from pytest import StashKey, TestReport
1617

1718
from localstack.testing.aws.util import is_aws_cloud
1819

20+
durations_key = StashKey[Dict[str, float]]()
21+
"""
22+
Stores phase durations on the test node between execution phases.
23+
See https://docs.pytest.org/en/latest/reference/reference.html#pytest.Stash
24+
"""
25+
test_failed_key = StashKey[bool]()
26+
"""
27+
Stores information from call execution phase about whether the test failed.
28+
"""
29+
1930

20-
def find_snapshot_for_item(item: pytest.Item) -> Optional[dict]:
31+
def find_validation_data_for_item(item: pytest.Item) -> Optional[dict]:
2132
base_path = os.path.join(item.fspath.dirname, item.fspath.purebasename)
22-
snapshot_path = f"{base_path}.snapshot.json"
33+
snapshot_path = f"{base_path}.validation.json"
2334

2435
if not os.path.exists(snapshot_path):
2536
return None
@@ -29,65 +40,76 @@ def find_snapshot_for_item(item: pytest.Item) -> Optional[dict]:
2940
return file_content.get(item.nodeid)
3041

3142

32-
def find_validation_data_for_item(item: pytest.Item) -> Optional[dict]:
33-
base_path = os.path.join(item.fspath.dirname, item.fspath.purebasename)
34-
snapshot_path = f"{base_path}.validation.json"
43+
@pytest.hookimpl(hookwrapper=True)
44+
def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo):
45+
"""
46+
This hook is called after each test execution phase (setup, call, teardown).
47+
"""
48+
result: Result = yield
49+
report: TestReport = result.get_result()
3550

36-
if not os.path.exists(snapshot_path):
37-
return None
51+
if call.when == "setup":
52+
_makereport_setup(item, call)
53+
elif call.when == "call":
54+
_makereport_call(item, call)
55+
elif call.when == "teardown":
56+
_makereport_teardown(item, call)
3857

39-
with open(snapshot_path, "r") as fd:
40-
file_content = json.load(fd)
41-
return file_content.get(item.nodeid)
58+
return report
4259

4360

44-
def record_passed_validation(item: pytest.Item, timestamp: Optional[datetime.datetime] = None):
61+
def _stash_phase_duration(call, item):
62+
durations_by_phase = item.stash.setdefault(durations_key, {})
63+
durations_by_phase[call.when] = round(call.duration, 2)
64+
65+
66+
def _makereport_setup(item: pytest.Item, call: pytest.CallInfo):
67+
_stash_phase_duration(call, item)
68+
69+
70+
def _makereport_call(item: pytest.Item, call: pytest.CallInfo):
71+
_stash_phase_duration(call, item)
72+
item.stash[test_failed_key] = call.excinfo is not None
73+
74+
75+
def _makereport_teardown(item: pytest.Item, call: pytest.CallInfo):
76+
_stash_phase_duration(call, item)
77+
78+
# only update the file when running against AWS and the test finishes successfully
79+
if not is_aws_cloud() or item.stash.get(test_failed_key, True):
80+
return
81+
4582
base_path = os.path.join(item.fspath.dirname, item.fspath.purebasename)
4683
file_path = Path(f"{base_path}.validation.json")
4784
file_path.touch()
4885
with file_path.open(mode="r+") as fd:
4986
# read existing state from file
5087
try:
5188
content = json.load(fd)
52-
except json.JSONDecodeError: # expected on first try (empty file)
89+
except json.JSONDecodeError: # expected on the first try (empty file)
5390
content = {}
5491

55-
# update for this pytest node
56-
if not timestamp:
57-
timestamp = datetime.datetime.now(tz=datetime.timezone.utc)
58-
content[item.nodeid] = {"last_validated_date": timestamp.isoformat(timespec="seconds")}
92+
test_execution_data = content.setdefault(item.nodeid, {})
5993

60-
# save updates
61-
fd.seek(0)
62-
json.dump(content, fd, indent=2, sort_keys=True)
63-
fd.write("\n") # add trailing newline for linter and Git compliance
94+
timestamp = datetime.datetime.now(tz=datetime.timezone.utc)
95+
test_execution_data["last_validated_date"] = timestamp.isoformat(timespec="seconds")
6496

97+
durations_by_phase = item.stash[durations_key]
98+
test_execution_data["durations_in_seconds"] = durations_by_phase
6599

66-
# TODO: we should skip if we're updating snapshots
67-
# make sure this is *AFTER* snapshot comparison => tryfirst=True
68-
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
69-
def pytest_runtest_call(item: pytest.Item):
70-
outcome: pluggy.Result = yield
100+
total_duration = sum(durations_by_phase.values())
101+
durations_by_phase["total"] = round(total_duration, 2)
71102

72-
# we only want to track passed runs against AWS
73-
if not is_aws_cloud() or outcome.excinfo:
74-
return
103+
# For json.dump sorted test entries enable consistent diffs.
104+
# But test execution data is more readable in insert order for each step (setup, call, teardown).
105+
# Hence, not using global sort_keys=True for json.dump but rather additionally sorting top-level dict only.
106+
content = dict(sorted(content.items()))
75107

76-
record_passed_validation(item)
77-
78-
79-
# this is a sort of utility used for retroactively creating validation files in accordance with existing snapshot files
80-
# it takes the recorded date from a snapshot and sets it to the last validated date
81-
# @pytest.hookimpl(trylast=True)
82-
# def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config, items: list[pytest.Item]):
83-
# for item in items:
84-
# snapshot_entry = find_snapshot_for_item(item)
85-
# if not snapshot_entry:
86-
# continue
87-
#
88-
# snapshot_update_timestamp = datetime.datetime.strptime(snapshot_entry["recorded-date"], "%d-%m-%Y, %H:%M:%S").astimezone(tz=datetime.timezone.utc)
89-
#
90-
# record_passed_validation(item, snapshot_update_timestamp)
108+
# save updates
109+
fd.truncate(0) # clear existing content
110+
fd.seek(0)
111+
json.dump(content, fd, indent=2)
112+
fd.write("\n") # add trailing newline for linter and Git compliance
91113

92114

93115
@pytest.hookimpl

‎tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.snapshot.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.py::test_adding_tags": {
3-
"recorded-date": "06-11-2024, 11:55:29",
3+
"recorded-date": "19-05-2025, 09:32:18",
44
"recorded-content": {
55
"event-source-mapping-tags": {
66
"Tags": {

‎tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.validation.json

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
{
22
"tests/aws/services/lambda_/event_source_mapping/test_cfn_resource.py::test_adding_tags": {
3-
"last_validated_date": "2024-11-06T11:55:29+00:00"
3+
"last_validated_date": "2025-05-19T09:33:12+00:00",
4+
"durations_in_seconds": {
5+
"setup": 0.54,
6+
"call": 69.88,
7+
"teardown": 54.76,
8+
"total": 125.18
9+
}
410
}
511
}

0 commit comments

Comments
 (0)