Skip to content

Commit ff55625

Browse files
author
Jon Wayne Parrott
committed
Deflaking tests by refactoring the model fixture.
1 parent 76807fa commit ff55625

23 files changed

+251
-164
lines changed

‎.travis.yml

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,22 @@ env:
66
before_install:
77
- openssl aes-256-cbc -K $encrypted_b4c8e1c51f6e_key -iv $encrypted_b4c8e1c51f6e_iv -in secrets.tar.enc -out secrets.tar -d
88
- tar xvf secrets.tar
9+
install:
910
- pip install nox-automation tox
1011
script:
1112
- nox --session reqcheck
1213
- cd $TRAVIS_BUILD_DIR/1-hello-world && tox
1314
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/2-structured-data && cd $TRAVIS_BUILD_DIR/2-structured-data
14-
&& tox
15+
&& tox -e lint
1516
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/3-binary-data && cd $TRAVIS_BUILD_DIR/3-binary-data
16-
&& tox
17+
&& tox -e lint
1718
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/4-auth && cd $TRAVIS_BUILD_DIR/4-auth
18-
&& tox
19+
&& tox -e lint
1920
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/5-logging && cd $TRAVIS_BUILD_DIR/5-logging
20-
&& tox
21+
&& tox -e lint
2122
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/6-pubsub && cd $TRAVIS_BUILD_DIR/6-pubsub
22-
&& tox
23+
&& tox -e lint
2324
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/7-gce && cd $TRAVIS_BUILD_DIR/7-gce
24-
&& tox
25+
&& tox -e lint
26+
after_script:
27+
- kill $DATASTORE_EMULATOR

‎2-structured-data/requirements-dev.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ flake8==2.5.4
33
flaky==3.1.0
44
pytest==2.8.7
55
pytest-cov==2.2.1
6+
retrying==1.3.3

‎2-structured-data/tests/conftest.py

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# Copyright 2015 Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""conftest.py is used to define common test fixtures for pytest."""
16+
17+
import bookshelf
18+
import config
19+
from gcloud.exceptions import ServiceUnavailable
20+
from oauth2client.client import HttpAccessTokenRefreshError
21+
import pytest
22+
from retrying import retry
23+
24+
25+
@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb'])
26+
def app(request):
27+
"""This fixtures provides a Flask app instance configured for testing.
28+
29+
Because it's parametric, it will cause every test that uses this fixture
30+
to run three times: one time for each backend (datastore, cloudsql, and
31+
mongodb).
32+
33+
It also ensures the tests run within a request context, allowing
34+
any calls to flask.request, flask.current_app, etc. to work."""
35+
app = bookshelf.create_app(
36+
config,
37+
testing=True,
38+
config_overrides={
39+
'DATA_BACKEND': request.param
40+
})
41+
42+
with app.test_request_context():
43+
yield app
44+
45+
46+
@pytest.yield_fixture
47+
def model(monkeypatch, app):
48+
"""This fixture provides a modified version of the app's model that tracks
49+
all created items and deletes them at the end of the test.
50+
51+
Any tests that directly or indirectly interact with the database should use
52+
this to ensure that resources are properly cleaned up.
53+
54+
Monkeypatch is provided by pytest and used to patch the model's create
55+
method.
56+
57+
The app fixture is needed to provide the configuration and context needed
58+
to get the proper model object.
59+
"""
60+
model = bookshelf.get_model()
61+
62+
# Ensure no books exist before running. This typically helps if tests
63+
# somehow left the database in a bad state.
64+
delete_all_books(model)
65+
66+
yield model
67+
68+
# Delete all books that we created during tests.
69+
delete_all_books(model)
70+
71+
72+
# The backend data stores can sometimes be flaky. It's useful to retry this
73+
# a few times before giving up.
74+
@retry(
75+
stop_max_attempt_number=3,
76+
wait_exponential_multiplier=100,
77+
wait_exponential_max=2000)
78+
def delete_all_books(model):
79+
while True:
80+
books, _ = model.list(limit=50)
81+
if not books:
82+
break
83+
for book in books:
84+
model.delete(book['id'])
85+
86+
87+
def flaky_filter(info, *args):
88+
"""Used by flaky to determine when to re-run a test case."""
89+
_, e, _ = info
90+
return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError))

‎2-structured-data/tests/test_crud.py

Lines changed: 3 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -12,81 +12,18 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
import bookshelf
16-
import config
15+
from conftest import flaky_filter
1716
from flaky import flaky
18-
from gcloud.exceptions import ServiceUnavailable
1917
import pytest
2018

2119

22-
@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb'])
23-
def app(request):
24-
"""This fixtures provides a Flask app instance configured for testing.
25-
26-
Because it's parametric, it will cause every test that uses this fixture
27-
to run three times: one time for each backend (datastore, cloudsql, and
28-
mongodb).
29-
30-
It also ensures the tests run within a request context, allowing
31-
any calls to flask.request, flask.current_app, etc. to work."""
32-
app = bookshelf.create_app(
33-
config,
34-
testing=True,
35-
config_overrides={
36-
'DATA_BACKEND': request.param
37-
})
38-
39-
with app.test_request_context():
40-
yield app
41-
42-
43-
@pytest.yield_fixture
44-
def model(monkeypatch, app):
45-
"""This fixture provides a modified version of the app's model that tracks
46-
all created items and deletes them at the end of the test.
47-
48-
Any tests that directly or indirectly interact with the database should use
49-
this to ensure that resources are properly cleaned up.
50-
51-
Monkeypatch is provided by pytest and used to patch the model's create
52-
method.
53-
54-
The app fixture is needed to provide the configuration and context needed
55-
to get the proper model object.
56-
"""
57-
model = bookshelf.get_model()
58-
59-
ids_to_delete = []
60-
61-
# Monkey-patch create so we can track the IDs of every item
62-
# created and delete them after the test case.
63-
original_create = model.create
64-
65-
def tracking_create(*args, **kwargs):
66-
res = original_create(*args, **kwargs)
67-
ids_to_delete.append(res['id'])
68-
return res
69-
70-
monkeypatch.setattr(model, 'create', tracking_create)
71-
72-
yield model
73-
74-
# Delete all items that we created during tests.
75-
list(map(model.delete, ids_to_delete))
76-
77-
78-
def flaky_filter(info, *args):
79-
"""Used by flaky to determine when to re-run a test case."""
80-
_, e, _ = info
81-
return isinstance(e, ServiceUnavailable)
82-
83-
8420
# Mark all test cases in this class as flaky, so that if errors occur they
8521
# can be retried. This is useful when databases are temporarily unavailable.
8622
@flaky(rerun_filter=flaky_filter)
8723
# Tell pytest to use both the app and model fixtures for all test cases.
8824
# This ensures that configuration is properly applied and that all database
89-
# resources created during tests are cleaned up.
25+
# resources created during tests are cleaned up. These fixtures are defined
26+
# in conftest.py
9027
@pytest.mark.usefixtures('app', 'model')
9128
class TestCrudActions(object):
9229

‎2-structured-data/tox.ini

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ deps =
99
-rrequirements-dev.txt
1010
commands =
1111
py.test --cov=bookshelf --no-success-flaky-report {posargs} tests
12-
passenv = GOOGLE_APPLICATION_CREDENTIALS
12+
passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST
1313
setenv = PYTHONPATH={toxinidir}
1414

1515
[testenv:py34]

‎3-binary-data/requirements-dev.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ flake8==2.5.4
33
flaky==3.1.0
44
pytest==2.8.7
55
pytest-cov==2.2.1
6+
retrying==1.3.3

‎3-binary-data/tests/conftest.py

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
import bookshelf
1818
import config
1919
from gcloud.exceptions import ServiceUnavailable
20+
from oauth2client.client import HttpAccessTokenRefreshError
2021
import pytest
22+
from retrying import retry
2123

2224

2325
@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb'])
@@ -57,26 +59,32 @@ def model(monkeypatch, app):
5759
"""
5860
model = bookshelf.get_model()
5961

60-
ids_to_delete = []
62+
# Ensure no books exist before running. This typically helps if tests
63+
# somehow left the database in a bad state.
64+
delete_all_books(model)
6165

62-
# Monkey-patch create so we can track the IDs of every item
63-
# created and delete them after the test case.
64-
original_create = model.create
65-
66-
def tracking_create(*args, **kwargs):
67-
res = original_create(*args, **kwargs)
68-
ids_to_delete.append(res['id'])
69-
return res
66+
yield model
7067

71-
monkeypatch.setattr(model, 'create', tracking_create)
68+
# Delete all books that we created during tests.
69+
delete_all_books(model)
7270

73-
yield model
7471

75-
# Delete all items that we created during tests.
76-
list(map(model.delete, ids_to_delete))
72+
# The backend data stores can sometimes be flaky. It's useful to retry this
73+
# a few times before giving up.
74+
@retry(
75+
stop_max_attempt_number=3,
76+
wait_exponential_multiplier=100,
77+
wait_exponential_max=2000)
78+
def delete_all_books(model):
79+
while True:
80+
books, _ = model.list(limit=50)
81+
if not books:
82+
break
83+
for book in books:
84+
model.delete(book['id'])
7785

7886

7987
def flaky_filter(info, *args):
8088
"""Used by flaky to determine when to re-run a test case."""
8189
_, e, _ = info
82-
return isinstance(e, ServiceUnavailable)
90+
return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError))

‎3-binary-data/tox.ini

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ deps =
99
-rrequirements-dev.txt
1010
commands =
1111
py.test --cov=bookshelf --no-success-flaky-report {posargs} tests
12-
passenv = GOOGLE_APPLICATION_CREDENTIALS
12+
passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST
1313
setenv = PYTHONPATH={toxinidir}
1414

1515
[testenv:py34]

‎4-auth/requirements-dev.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ flake8==2.5.4
33
flaky==3.1.0
44
pytest==2.8.7
55
pytest-cov==2.2.1
6+
retrying==1.3.3

‎4-auth/tests/conftest.py

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
import bookshelf
1818
import config
1919
from gcloud.exceptions import ServiceUnavailable
20+
from oauth2client.client import HttpAccessTokenRefreshError
2021
import pytest
22+
from retrying import retry
2123

2224

2325
@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb'])
@@ -57,26 +59,32 @@ def model(monkeypatch, app):
5759
"""
5860
model = bookshelf.get_model()
5961

60-
ids_to_delete = []
62+
# Ensure no books exist before running. This typically helps if tests
63+
# somehow left the database in a bad state.
64+
delete_all_books(model)
6165

62-
# Monkey-patch create so we can track the IDs of every item
63-
# created and delete them after the test case.
64-
original_create = model.create
65-
66-
def tracking_create(*args, **kwargs):
67-
res = original_create(*args, **kwargs)
68-
ids_to_delete.append(res['id'])
69-
return res
66+
yield model
7067

71-
monkeypatch.setattr(model, 'create', tracking_create)
68+
# Delete all books that we created during tests.
69+
delete_all_books(model)
7270

73-
yield model
7471

75-
# Delete all items that we created during tests.
76-
list(map(model.delete, ids_to_delete))
72+
# The backend data stores can sometimes be flaky. It's useful to retry this
73+
# a few times before giving up.
74+
@retry(
75+
stop_max_attempt_number=3,
76+
wait_exponential_multiplier=100,
77+
wait_exponential_max=2000)
78+
def delete_all_books(model):
79+
while True:
80+
books, _ = model.list(limit=50)
81+
if not books:
82+
break
83+
for book in books:
84+
model.delete(book['id'])
7785

7886

7987
def flaky_filter(info, *args):
8088
"""Used by flaky to determine when to re-run a test case."""
8189
_, e, _ = info
82-
return isinstance(e, ServiceUnavailable)
90+
return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError))

‎4-auth/tox.ini

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ deps =
99
-rrequirements-dev.txt
1010
commands =
1111
py.test --cov=bookshelf --no-success-flaky-report {posargs:tests}
12-
passenv = GOOGLE_APPLICATION_CREDENTIALS
12+
passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST
1313
setenv = PYTHONPATH={toxinidir}
1414

1515
[testenv:py34]

‎5-logging/requirements-dev.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ flake8==2.5.4
33
flaky==3.1.0
44
pytest==2.8.7
55
pytest-cov==2.2.1
6+
retrying==1.3.3

0 commit comments

Comments
 (0)