Skip to content

Deflaking tests by refactoring the model fixture. #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 3, 2016
Merged

Deflaking tests by refactoring the model fixture. #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,22 @@ env:
before_install:
- openssl aes-256-cbc -K $encrypted_b4c8e1c51f6e_key -iv $encrypted_b4c8e1c51f6e_iv -in secrets.tar.enc -out secrets.tar -d
- tar xvf secrets.tar
install:
- pip install nox-automation tox
script:
- nox --session reqcheck
- cd $TRAVIS_BUILD_DIR/1-hello-world && tox
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/2-structured-data && cd $TRAVIS_BUILD_DIR/2-structured-data
&& tox
&& tox -e lint
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/3-binary-data && cd $TRAVIS_BUILD_DIR/3-binary-data
&& tox
&& tox -e lint
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/4-auth && cd $TRAVIS_BUILD_DIR/4-auth
&& tox
&& tox -e lint
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/5-logging && cd $TRAVIS_BUILD_DIR/5-logging
&& tox
&& tox -e lint
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/6-pubsub && cd $TRAVIS_BUILD_DIR/6-pubsub
&& tox
&& tox -e lint
- cp $TRAVIS_BUILD_DIR/config.py $TRAVIS_BUILD_DIR/7-gce && cd $TRAVIS_BUILD_DIR/7-gce
&& tox
&& tox -e lint
after_script:
- kill $DATASTORE_EMULATOR
1 change: 1 addition & 0 deletions 2-structured-data/requirements-dev.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ flake8==2.5.4
flaky==3.1.0
pytest==2.8.7
pytest-cov==2.2.1
retrying==1.3.3
90 changes: 90 additions & 0 deletions 2-structured-data/tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rebase?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No this is right. This file didn't exist before.

#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""conftest.py is used to define common test fixtures for pytest."""

import bookshelf
import config
from gcloud.exceptions import ServiceUnavailable
from oauth2client.client import HttpAccessTokenRefreshError
import pytest
from retrying import retry


@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb'])
def app(request):
"""This fixtures provides a Flask app instance configured for testing.

Because it's parametric, it will cause every test that uses this fixture
to run three times: one time for each backend (datastore, cloudsql, and
mongodb).

It also ensures the tests run within a request context, allowing
any calls to flask.request, flask.current_app, etc. to work."""
app = bookshelf.create_app(
config,
testing=True,
config_overrides={
'DATA_BACKEND': request.param
})

with app.test_request_context():
yield app


@pytest.yield_fixture
def model(monkeypatch, app):
"""This fixture provides a modified version of the app's model that tracks
all created items and deletes them at the end of the test.

Any tests that directly or indirectly interact with the database should use
this to ensure that resources are properly cleaned up.

Monkeypatch is provided by pytest and used to patch the model's create
method.

The app fixture is needed to provide the configuration and context needed
to get the proper model object.
"""
model = bookshelf.get_model()

# Ensure no books exist before running. This typically helps if tests
# somehow left the database in a bad state.
delete_all_books(model)

yield model

# Delete all books that we created during tests.
delete_all_books(model)


# The backend data stores can sometimes be flaky. It's useful to retry this
# a few times before giving up.
@retry(
stop_max_attempt_number=3,
wait_exponential_multiplier=100,
wait_exponential_max=2000)
def delete_all_books(model):
while True:
books, _ = model.list(limit=50)
if not books:
break
for book in books:
model.delete(book['id'])


def flaky_filter(info, *args):
"""Used by flaky to determine when to re-run a test case."""
_, e, _ = info
return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError))
69 changes: 3 additions & 66 deletions 2-structured-data/tests/test_crud.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,81 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import bookshelf
import config
from conftest import flaky_filter
from flaky import flaky
from gcloud.exceptions import ServiceUnavailable
import pytest


@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb'])
def app(request):
"""This fixtures provides a Flask app instance configured for testing.

Because it's parametric, it will cause every test that uses this fixture
to run three times: one time for each backend (datastore, cloudsql, and
mongodb).

It also ensures the tests run within a request context, allowing
any calls to flask.request, flask.current_app, etc. to work."""
app = bookshelf.create_app(
config,
testing=True,
config_overrides={
'DATA_BACKEND': request.param
})

with app.test_request_context():
yield app


@pytest.yield_fixture
def model(monkeypatch, app):
"""This fixture provides a modified version of the app's model that tracks
all created items and deletes them at the end of the test.

Any tests that directly or indirectly interact with the database should use
this to ensure that resources are properly cleaned up.

Monkeypatch is provided by pytest and used to patch the model's create
method.

The app fixture is needed to provide the configuration and context needed
to get the proper model object.
"""
model = bookshelf.get_model()

ids_to_delete = []

# Monkey-patch create so we can track the IDs of every item
# created and delete them after the test case.
original_create = model.create

def tracking_create(*args, **kwargs):
res = original_create(*args, **kwargs)
ids_to_delete.append(res['id'])
return res

monkeypatch.setattr(model, 'create', tracking_create)

yield model

# Delete all items that we created during tests.
list(map(model.delete, ids_to_delete))


def flaky_filter(info, *args):
"""Used by flaky to determine when to re-run a test case."""
_, e, _ = info
return isinstance(e, ServiceUnavailable)


# Mark all test cases in this class as flaky, so that if errors occur they
# can be retried. This is useful when databases are temporarily unavailable.
@flaky(rerun_filter=flaky_filter)
# Tell pytest to use both the app and model fixtures for all test cases.
# This ensures that configuration is properly applied and that all database
# resources created during tests are cleaned up.
# resources created during tests are cleaned up. These fixtures are defined
# in conftest.py
@pytest.mark.usefixtures('app', 'model')
class TestCrudActions(object):

Expand Down
2 changes: 1 addition & 1 deletion 2-structured-data/tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ deps =
-rrequirements-dev.txt
commands =
py.test --cov=bookshelf --no-success-flaky-report {posargs} tests
passenv = GOOGLE_APPLICATION_CREDENTIALS
passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST
setenv = PYTHONPATH={toxinidir}

[testenv:py34]
Expand Down
1 change: 1 addition & 0 deletions 3-binary-data/requirements-dev.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ flake8==2.5.4
flaky==3.1.0
pytest==2.8.7
pytest-cov==2.2.1
retrying==1.3.3
36 changes: 22 additions & 14 deletions 3-binary-data/tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@
import bookshelf
import config
from gcloud.exceptions import ServiceUnavailable
from oauth2client.client import HttpAccessTokenRefreshError
import pytest
from retrying import retry


@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb'])
Expand Down Expand Up @@ -57,26 +59,32 @@ def model(monkeypatch, app):
"""
model = bookshelf.get_model()

ids_to_delete = []
# Ensure no books exist before running. This typically helps if tests
# somehow left the database in a bad state.
delete_all_books(model)

# Monkey-patch create so we can track the IDs of every item
# created and delete them after the test case.
original_create = model.create

def tracking_create(*args, **kwargs):
res = original_create(*args, **kwargs)
ids_to_delete.append(res['id'])
return res
yield model

monkeypatch.setattr(model, 'create', tracking_create)
# Delete all books that we created during tests.
delete_all_books(model)

yield model

# Delete all items that we created during tests.
list(map(model.delete, ids_to_delete))
# The backend data stores can sometimes be flaky. It's useful to retry this
# a few times before giving up.
@retry(
stop_max_attempt_number=3,
wait_exponential_multiplier=100,
wait_exponential_max=2000)
def delete_all_books(model):
while True:
books, _ = model.list(limit=50)
if not books:
break
for book in books:
model.delete(book['id'])


def flaky_filter(info, *args):
"""Used by flaky to determine when to re-run a test case."""
_, e, _ = info
return isinstance(e, ServiceUnavailable)
return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError))
2 changes: 1 addition & 1 deletion 3-binary-data/tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ deps =
-rrequirements-dev.txt
commands =
py.test --cov=bookshelf --no-success-flaky-report {posargs} tests
passenv = GOOGLE_APPLICATION_CREDENTIALS
passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST
setenv = PYTHONPATH={toxinidir}

[testenv:py34]
Expand Down
1 change: 1 addition & 0 deletions 4-auth/requirements-dev.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ flake8==2.5.4
flaky==3.1.0
pytest==2.8.7
pytest-cov==2.2.1
retrying==1.3.3
36 changes: 22 additions & 14 deletions 4-auth/tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@
import bookshelf
import config
from gcloud.exceptions import ServiceUnavailable
from oauth2client.client import HttpAccessTokenRefreshError
import pytest
from retrying import retry


@pytest.yield_fixture(params=['datastore', 'cloudsql', 'mongodb'])
Expand Down Expand Up @@ -57,26 +59,32 @@ def model(monkeypatch, app):
"""
model = bookshelf.get_model()

ids_to_delete = []
# Ensure no books exist before running. This typically helps if tests
# somehow left the database in a bad state.
delete_all_books(model)

# Monkey-patch create so we can track the IDs of every item
# created and delete them after the test case.
original_create = model.create

def tracking_create(*args, **kwargs):
res = original_create(*args, **kwargs)
ids_to_delete.append(res['id'])
return res
yield model

monkeypatch.setattr(model, 'create', tracking_create)
# Delete all books that we created during tests.
delete_all_books(model)

yield model

# Delete all items that we created during tests.
list(map(model.delete, ids_to_delete))
# The backend data stores can sometimes be flaky. It's useful to retry this
# a few times before giving up.
@retry(
stop_max_attempt_number=3,
wait_exponential_multiplier=100,
wait_exponential_max=2000)
def delete_all_books(model):
while True:
books, _ = model.list(limit=50)
if not books:
break
for book in books:
model.delete(book['id'])


def flaky_filter(info, *args):
"""Used by flaky to determine when to re-run a test case."""
_, e, _ = info
return isinstance(e, ServiceUnavailable)
return isinstance(e, (ServiceUnavailable, HttpAccessTokenRefreshError))
2 changes: 1 addition & 1 deletion 4-auth/tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ deps =
-rrequirements-dev.txt
commands =
py.test --cov=bookshelf --no-success-flaky-report {posargs:tests}
passenv = GOOGLE_APPLICATION_CREDENTIALS
passenv = GOOGLE_APPLICATION_CREDENTIALS DATASTORE_HOST
setenv = PYTHONPATH={toxinidir}

[testenv:py34]
Expand Down
1 change: 1 addition & 0 deletions 5-logging/requirements-dev.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ flake8==2.5.4
flaky==3.1.0
pytest==2.8.7
pytest-cov==2.2.1
retrying==1.3.3
Loading