From bfca2e24243c01262049ff63cbd69751e53fcc48 Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Fri, 3 Jun 2016 15:32:10 -0700 Subject: [PATCH 01/25] add newer django+python to tox+travis --- .travis.yml | 16 ++++++++++++++++ tox.ini | 22 ++++++++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1282ff8..d00bf3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ python: - "2.7" - "3.3" - "3.4" + - "3.5" addons: postgresql: "9.4" before_script: @@ -31,15 +32,30 @@ env: - DJANGO_SPEC="Django>=1.6,<1.7" - DJANGO_SPEC="Django>=1.7,<1.8" - DJANGO_SPEC="Django>=1.8,<1.9" + - DJANGO_SPEC="Django==1.9b1" matrix: exclude: - python: "2.6" env: DJANGO_SPEC="Django>=1.7,<1.8" - python: "2.6" env: DJANGO_SPEC="Django>=1.8,<1.9" + - python: "2.6" + env: DJANGO_SPEC="Django==1.9b1" - python: "3.3" env: DJANGO_SPEC="Django>=1.4,<1.5" + - python: "3.3" + env: DJANGO_SPEC="Django1.9b1" - python: "3.4" env: DJANGO_SPEC="Django>=1.4,<1.5" + - python: "3.5" + env: DJANGO_SPEC="Django>=1.4,<1.5" + - python: "3.5" + env: DJANGO_SPEC="Django>=1.5,<1.6" + - python: "3.5" + env: DJANGO_SPEC="Django>=1.6,<1.7" + - python: "3.5" + env: DJANGO_SPEC="Django>=1.7,<1.8" + - python: "3.5" + env: DJANGO_SPEC="Django>=1.8,<1.9" # Adding sudo: False tells Travis to use their container-based infrastructure, which is somewhat faster. sudo: False diff --git a/tox.ini b/tox.ini index d82f261..6e76039 100644 --- a/tox.ini +++ b/tox.ini @@ -5,22 +5,36 @@ [tox] envlist = - py26-dj{14,15,16} - py27-dj{14,15,16,17,18} - py{33,34}-dj{15,16,17,18} + dj14-py{26,27} + dj{15,16}-py{26,27,33,34} + dj{17,18}-py{27,33,34} + dj19-py{27,34,35} + dj110-py{27,34,35} +# py26-dj{14,15,16} +# py27-dj{14,15,16,17,18} +# py{33,34}-dj{15,16,17,18} +# py{27,34,35}-dj19 py{27,34}-flake8 docs +[testenv:py33] +basepython = if [ -x $(which pythonz) ]; then pythonz locate 3.3.6; else which python3.3; fi + +[testenv:py35] +basepython = if [ -x $(which pythonz) ]; then pythonz locate 3.5.1; else which python3.5; fi + [testenv] commands = {envpython} run_tests.py deps = py{26,27}: -rrequirements/py2.txt - py{33,34}: -rrequirements/py3.txt + py{33,34,35}: -rrequirements/py3.txt dj14: Django>=1.4,<1.5 dj15: Django>=1.5,<1.6 dj16: Django>=1.6,<1.7 dj17: Django>=1.7,<1.8 dj18: Django>=1.8,<1.9 + dj19: Django>=1.9,<1.10 + dj110: Django>=1.10a1,<1.11 [testenv:docs] basepython = python2.7 From 004261f4c0f53a124a41f40ef8ee884ca8e22648 Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Fri, 3 Jun 2016 15:47:10 -0700 Subject: [PATCH 02/25] convert assigned lambda funcs to def statements --- run_tests.py | 2 +- tests/test_cache.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/run_tests.py b/run_tests.py index 6bea3fb..5cdbb26 100644 --- a/run_tests.py +++ b/run_tests.py @@ -15,7 +15,7 @@ # Python 2.6 doesn't have check_output. Note this will not raise a CalledProcessError # like check_output does, but it should work for our purposes. import subprocess - check_output = lambda x: subprocess.Popen(x, stdout=subprocess.PIPE).communicate()[0] + def check_output(x): return subprocess.Popen(x, stdout=subprocess.PIPE).communicate()[0] NAME = os.path.basename(os.path.dirname(__file__)) ROOT = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/test_cache.py b/tests/test_cache.py index 408cd58..953e717 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -308,7 +308,7 @@ def expensive(): return counter.call_count a = Addon.objects.get(id=1) - f = lambda: base.cached_with(a, expensive, 'key') + def f(): return base.cached_with(a, expensive, 'key') # Only gets called once. eq_(f(), 1) @@ -328,7 +328,7 @@ def expensive(): counter.reset_mock() q = Addon.objects.filter(id=1) - f = lambda: base.cached_with(q, expensive, 'key') + def f(): return base.cached_with(q, expensive, 'key') # Only gets called once. eq_(f(), 1) @@ -355,7 +355,7 @@ def test_cached_with_unicode(self): obj = mock.Mock() obj.query_key.return_value = 'xxx' obj.flush_key.return_value = 'key' - f = lambda: 1 + def f(): return 1 eq_(base.cached_with(obj, f, 'adf:%s' % u), 1) def test_cached_method(self): From 23b28c62fc8a123e0d65f1b37a5ce61538d945fe Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Fri, 3 Jun 2016 15:52:41 -0700 Subject: [PATCH 03/25] fix flake8 errors --- caching/base.py | 2 +- tests/test_cache.py | 24 +++++++++++++----------- tests/testapp/models.py | 3 +-- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/caching/base.py b/caching/base.py index 9e0e324..0e7fd8f 100644 --- a/caching/base.py +++ b/caching/base.py @@ -388,7 +388,7 @@ def __init__(self, obj, func): self.cache = {} def __call__(self, *args, **kwargs): - k = lambda o: o.cache_key if hasattr(o, 'cache_key') else o + def k(o): return o.cache_key if hasattr(o, 'cache_key') else o arg_keys = list(map(k, args)) kwarg_keys = [(key, k(val)) for key, val in list(kwargs.items())] key_parts = ('m', self.obj.cache_key, self.func.__name__, diff --git a/tests/test_cache.py b/tests/test_cache.py index 69832ee..5a86588 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,29 +1,28 @@ from __future__ import unicode_literals +import django +import jinja2 import logging import pickle import sys +from django.conf import settings +from django.test import TestCase, TransactionTestCase +from django.utils import translation, encoding + +from caching import base, invalidation, config, compat + +from .testapp.models import Addon, User + if sys.version_info < (2, 7): import unittest2 as unittest else: import unittest -import django -from django.conf import settings -from django.test import TestCase, TransactionTestCase -from django.utils import translation, encoding - if sys.version_info >= (3, ): from unittest import mock else: import mock -import jinja2 - -from caching import base, invalidation, config, compat - -from .testapp.models import Addon, User - cache = invalidation.cache log = logging.getLogger(__name__) @@ -314,6 +313,7 @@ def expensive(): return counter.call_count a = Addon.objects.get(id=1) + def f(): return base.cached_with(a, expensive, 'key') # Only gets called once. @@ -334,6 +334,7 @@ def f(): return base.cached_with(a, expensive, 'key') counter.reset_mock() q = Addon.objects.filter(id=1) + def f(): return base.cached_with(q, expensive, 'key') # Only gets called once. @@ -361,6 +362,7 @@ def test_cached_with_unicode(self): obj = mock.Mock() obj.query_key.return_value = 'xxx' obj.flush_key.return_value = 'key' + def f(): return 1 self.assertEqual(base.cached_with(obj, f, 'adf:%s' % u), 1) diff --git a/tests/testapp/models.py b/tests/testapp/models.py index 37fbdc6..1df219d 100644 --- a/tests/testapp/models.py +++ b/tests/testapp/models.py @@ -2,14 +2,13 @@ from django.db import models from django.utils import six +from caching.base import CachingMixin, CachingManager, cached_method if six.PY3: from unittest import mock else: import mock -from caching.base import CachingMixin, CachingManager, cached_method - # This global call counter will be shared among all instances of an Addon. call_counter = mock.Mock() From f12a36c055f009a1d303599fdc8a4e4cce32c3b4 Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Fri, 3 Jun 2016 16:25:27 -0700 Subject: [PATCH 04/25] make travis use tox --- .travis.yml | 61 ++++++++++++++++++++++++----------------------------- tox.ini | 16 ++++++-------- 2 files changed, 35 insertions(+), 42 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd828a2..2d92858 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,43 +19,38 @@ before_script: install: - pip install -U pip # make sure we have the latest version - pip install -e . - - pip install -r requirements/py`echo $TRAVIS_PYTHON_VERSION|cut -d'.' -f1`.txt "$DJANGO_SPEC" + - pip install tox - pip install coveralls script: - - python run_tests.py --with-coverage - - flake8 --ignore=E731,E402 . + - tox -e $TOX_ENV + - flake8 after_success: - coveralls env: - - DJANGO_SPEC="Django>=1.4,<1.5" - - DJANGO_SPEC="Django>=1.5,<1.6" - - DJANGO_SPEC="Django>=1.6,<1.7" - - DJANGO_SPEC="Django>=1.7,<1.8" - - DJANGO_SPEC="Django>=1.8,<1.9" - - DJANGO_SPEC="Django==1.9b1" -matrix: - exclude: - - python: "2.6" - env: DJANGO_SPEC="Django>=1.7,<1.8" - - python: "2.6" - env: DJANGO_SPEC="Django>=1.8,<1.9" - - python: "2.6" - env: DJANGO_SPEC="Django==1.9b1" - - python: "3.3" - env: DJANGO_SPEC="Django>=1.4,<1.5" - - python: "3.3" - env: DJANGO_SPEC="Django1.9b1" - - python: "3.4" - env: DJANGO_SPEC="Django>=1.4,<1.5" - - python: "3.5" - env: DJANGO_SPEC="Django>=1.4,<1.5" - - python: "3.5" - env: DJANGO_SPEC="Django>=1.5,<1.6" - - python: "3.5" - env: DJANGO_SPEC="Django>=1.6,<1.7" - - python: "3.5" - env: DJANGO_SPEC="Django>=1.7,<1.8" - - python: "3.5" - env: DJANGO_SPEC="Django>=1.8,<1.9" + - TOX_ENV="dj14-py26" + - TOX_ENV="dj14-py27" + - TOX_ENV="dj15-py26" + - TOX_ENV="dj15-py27" + - TOX_ENV="dj15-py33" + - TOX_ENV="dj15-py34" + - TOX_ENV="dj16-py26" + - TOX_ENV="dj16-py27" + - TOX_ENV="dj16-py33" + - TOX_ENV="dj16-py34" + - TOX_ENV="dj17-py27" + - TOX_ENV="dj17-py33" + - TOX_ENV="dj17-py34" + - TOX_ENV="dj18-py27" + - TOX_ENV="dj18-py33" + - TOX_ENV="dj18-py34" + - TOX_ENV="dj19-py27" + - TOX_ENV="dj19-py34" + - TOX_ENV="dj19-py35" + - TOX_ENV="dj110-py27" + - TOX_ENV="dj110-py34" + - TOX_ENV="dj110-py35" + - TOX_ENV="py27-flake8" + - TOX_ENV="py35-flake8" + - TOX_ENV="docs" # Adding sudo: False tells Travis to use their container-based infrastructure, which is somewhat faster. sudo: False diff --git a/tox.ini b/tox.ini index 6e76039..83b85ae 100644 --- a/tox.ini +++ b/tox.ini @@ -8,20 +8,18 @@ envlist = dj14-py{26,27} dj{15,16}-py{26,27,33,34} dj{17,18}-py{27,33,34} - dj19-py{27,34,35} - dj110-py{27,34,35} -# py26-dj{14,15,16} -# py27-dj{14,15,16,17,18} -# py{33,34}-dj{15,16,17,18} -# py{27,34,35}-dj19 - py{27,34}-flake8 + dj{19,110}-py{27,34,35} + py{27,35}-flake8 docs [testenv:py33] -basepython = if [ -x $(which pythonz) ]; then pythonz locate 3.3.6; else which python3.3; fi +basepython = "if [ -x $(which pythonz) ]; then pythonz locate 3.3.6; else which python3.3; fi" + +[testenv:py34] +basepython = "if [ -x $(which pythonz) ]; then pythonz locate 3.4.3; else which python3.4; fi" [testenv:py35] -basepython = if [ -x $(which pythonz) ]; then pythonz locate 3.5.1; else which python3.5; fi +basepython = "if [ -x $(which pythonz) ]; then pythonz locate 3.5.1; else which python3.5; fi" [testenv] commands = {envpython} run_tests.py From d6c527e5fed078765064eb345a039135423a05fd Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Fri, 3 Jun 2016 16:30:15 -0700 Subject: [PATCH 05/25] specify only one python since tox does that now --- .travis.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2d92858..edc93d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,14 +2,8 @@ language: python services: - memcached - redis-server -# Use Travis' build matrix and exclude functions rather than running tox -# directly so that we can run the builds in parallel and get coverage reports -# for each Python/Django version combo python: - - "2.6" - - "2.7" - - "3.3" - - "3.4" +# python selected by tox, so specify only one version here - "3.5" addons: postgresql: "9.4" From aed2defd22884e009cd63723aca03344eb999d04 Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Fri, 3 Jun 2016 16:42:47 -0700 Subject: [PATCH 06/25] pass TRAVIS environment variable into tox environment --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 83b85ae..47d3c1f 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,7 @@ basepython = "if [ -x $(which pythonz) ]; then pythonz locate 3.5.1; else which [testenv] commands = {envpython} run_tests.py +passenv = TRAVIS deps = py{26,27}: -rrequirements/py2.txt py{33,34,35}: -rrequirements/py3.txt From 615585bca9f8c03d0a48ab98bb8f889b06993a1d Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Fri, 3 Jun 2016 17:07:12 -0700 Subject: [PATCH 07/25] don't run flake8 as part of travis, use tox --- .travis.yml | 1 - tox.ini | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index edc93d5..41323bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,6 @@ install: - pip install coveralls script: - tox -e $TOX_ENV - - flake8 after_success: - coveralls env: diff --git a/tox.ini b/tox.ini index 47d3c1f..a3b524d 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,7 @@ basepython = python2.7 deps = flake8 commands = flake8 -[testenv:py34-flake8] -basepython = python3.4 +[testenv:py35-flake8] +basepython = python3.5 deps = flake8 commands = flake8 From dcc81ab73ff08d3a823d4488845a5c43228f5c16 Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 6 Oct 2017 21:45:31 -0400 Subject: [PATCH 08/25] Update tox and travis --- .travis.yml | 33 +++++++-------------------------- docs/conf.py | 3 ++- run_tests.py | 10 +--------- tox.ini | 38 +++++++++++++------------------------- 4 files changed, 23 insertions(+), 61 deletions(-) diff --git a/.travis.yml b/.travis.yml index 41323bf..f31fff0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,9 +4,9 @@ services: - redis-server python: # python selected by tox, so specify only one version here - - "3.5" + - "3.6" addons: - postgresql: "9.4" + postgresql: "9.5" before_script: - psql -c 'create database travis_ci_test;' -U postgres - psql -c 'create database travis_ci_test2;' -U postgres @@ -20,30 +20,11 @@ script: after_success: - coveralls env: - - TOX_ENV="dj14-py26" - - TOX_ENV="dj14-py27" - - TOX_ENV="dj15-py26" - - TOX_ENV="dj15-py27" - - TOX_ENV="dj15-py33" - - TOX_ENV="dj15-py34" - - TOX_ENV="dj16-py26" - - TOX_ENV="dj16-py27" - - TOX_ENV="dj16-py33" - - TOX_ENV="dj16-py34" - - TOX_ENV="dj17-py27" - - TOX_ENV="dj17-py33" - - TOX_ENV="dj17-py34" - - TOX_ENV="dj18-py27" - - TOX_ENV="dj18-py33" - - TOX_ENV="dj18-py34" - - TOX_ENV="dj19-py27" - - TOX_ENV="dj19-py34" - - TOX_ENV="dj19-py35" - - TOX_ENV="dj110-py27" - - TOX_ENV="dj110-py34" - - TOX_ENV="dj110-py35" - - TOX_ENV="py27-flake8" - - TOX_ENV="py35-flake8" + - TOX_ENV="dj18-py27,dj18-py34,dj18-py35" + # - TOX_ENV="dj19-py27,dj19-py34,dj19-py35,dj19-py36" + # - TOX_ENV="dj110-py27,dj110-py34,dj110-py35,dj110-py36" + # - TOX_ENV="dj111-py27,dj111-py34,dj111-py35,dj111-py36" + # - TOX_ENV="py27-flake8,py36-flake8" - TOX_ENV="docs" # Adding sudo: False tells Travis to use their container-based infrastructure, which is somewhat faster. sudo: False diff --git a/docs/conf.py b/docs/conf.py index 71f3df3..3341ac5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,9 +1,10 @@ import os import sys +import caching + sys.path.append(os.path.abspath('..')) -import caching # The suffix of source filenames. source_suffix = '.rst' diff --git a/run_tests.py b/run_tests.py index 5cdbb26..d33f08f 100644 --- a/run_tests.py +++ b/run_tests.py @@ -7,15 +7,7 @@ import os import sys import argparse - -from subprocess import call -try: - from subprocess import check_output -except ImportError: - # Python 2.6 doesn't have check_output. Note this will not raise a CalledProcessError - # like check_output does, but it should work for our purposes. - import subprocess - def check_output(x): return subprocess.Popen(x, stdout=subprocess.PIPE).communicate()[0] +from subprocess import call, check_output NAME = os.path.basename(os.path.dirname(__file__)) ROOT = os.path.abspath(os.path.dirname(__file__)) diff --git a/tox.ini b/tox.ini index a3b524d..0035638 100644 --- a/tox.ini +++ b/tox.ini @@ -5,38 +5,28 @@ [tox] envlist = - dj14-py{26,27} - dj{15,16}-py{26,27,33,34} - dj{17,18}-py{27,33,34} - dj{19,110}-py{27,34,35} - py{27,35}-flake8 + dj{18}-py{27,34,35} + dj{19,110,111}-py{27,34,35,36} + py{27,36}-flake8 docs -[testenv:py33] -basepython = "if [ -x $(which pythonz) ]; then pythonz locate 3.3.6; else which python3.3; fi" - -[testenv:py34] -basepython = "if [ -x $(which pythonz) ]; then pythonz locate 3.4.3; else which python3.4; fi" - -[testenv:py35] -basepython = "if [ -x $(which pythonz) ]; then pythonz locate 3.5.1; else which python3.5; fi" - [testenv] +basepython = + py27: python2.7 + py34: python3.4 + py35: python3.5 + py36: python3.6 commands = {envpython} run_tests.py -passenv = TRAVIS deps = py{26,27}: -rrequirements/py2.txt - py{33,34,35}: -rrequirements/py3.txt - dj14: Django>=1.4,<1.5 - dj15: Django>=1.5,<1.6 - dj16: Django>=1.6,<1.7 - dj17: Django>=1.7,<1.8 + py{34,35,36}: -rrequirements/py3.txt dj18: Django>=1.8,<1.9 dj19: Django>=1.9,<1.10 - dj110: Django>=1.10a1,<1.11 + dj110: Django>=1.10,<1.11 + dj111: Django>=1.11,<2.0 [testenv:docs] -basepython = python2.7 +basepython = python3.6 deps = Sphinx Django @@ -47,11 +37,9 @@ changedir = docs commands = /usr/bin/make html [testenv:py27-flake8] -basepython = python2.7 deps = flake8 commands = flake8 -[testenv:py35-flake8] -basepython = python3.5 +[testenv:py36-flake8] deps = flake8 commands = flake8 From 26680940e4bace73121fd8e98ba3ada5ce14e28a Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 6 Oct 2017 21:48:53 -0400 Subject: [PATCH 09/25] Include flake8 on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f31fff0..898ba66 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ env: # - TOX_ENV="dj19-py27,dj19-py34,dj19-py35,dj19-py36" # - TOX_ENV="dj110-py27,dj110-py34,dj110-py35,dj110-py36" # - TOX_ENV="dj111-py27,dj111-py34,dj111-py35,dj111-py36" - # - TOX_ENV="py27-flake8,py36-flake8" + - TOX_ENV="py27-flake8,py36-flake8" - TOX_ENV="docs" # Adding sudo: False tells Travis to use their container-based infrastructure, which is somewhat faster. sudo: False From d450cffb6f4ecf18fdd9e80b548e7b3a3f43dbed Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 6 Oct 2017 22:13:04 -0400 Subject: [PATCH 10/25] Workaround Travis bug --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 898ba66..03dca3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,9 @@ python: - "3.6" addons: postgresql: "9.5" +before_install: + # work around https://github.com/travis-ci/travis-ci/issues/8363 + - pyenv global system 3.5 before_script: - psql -c 'create database travis_ci_test;' -U postgres - psql -c 'create database travis_ci_test2;' -U postgres From 77d21e862f46a392baedb87acde7ffc7d3d73d49 Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 6 Oct 2017 22:29:20 -0400 Subject: [PATCH 11/25] Try to get coverage working --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0035638..56232f4 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ basepython = py34: python3.4 py35: python3.5 py36: python3.6 -commands = {envpython} run_tests.py +commands = {envpython} run_tests.py --with-coverage deps = py{26,27}: -rrequirements/py2.txt py{34,35,36}: -rrequirements/py3.txt From 79281be526d333f71839c1156d04f6dc192bf347 Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 6 Oct 2017 23:04:42 -0400 Subject: [PATCH 12/25] Remove support for Python < 2.7, Django < 1.8 --- README.rst | 6 +-- caching/backends/__init__.py | 0 caching/backends/locmem.py | 43 ----------------- caching/backends/memcached.py | 32 ------------- caching/base.py | 6 +-- caching/compat.py | 14 ------ caching/invalidation.py | 13 ++---- docs/index.rst | 56 ++--------------------- examples/cache_machine/custom_backend.py | 2 +- examples/cache_machine/locmem_settings.py | 2 +- examples/cache_machine/settings.py | 2 +- requirements/base.txt | 1 - setup.py | 4 +- tests/test_cache.py | 29 ++---------- 14 files changed, 19 insertions(+), 191 deletions(-) delete mode 100644 caching/backends/__init__.py delete mode 100644 caching/backends/locmem.py delete mode 100644 caching/backends/memcached.py delete mode 100644 caching/compat.py diff --git a/README.rst b/README.rst index e78b4f8..8a61283 100644 --- a/README.rst +++ b/README.rst @@ -17,7 +17,7 @@ For full docs, see https://cache-machine.readthedocs.org/en/latest/. Requirements ------------ -Cache Machine works with Django 1.4-1.8 and Python 2.6, 2.7, 3.3 and 3.4. +Cache Machine works with Django 1.8 and Python 2.7, 3.4, 3.5 and 3.6. Installation @@ -27,10 +27,6 @@ Get it from `pypi `_:: pip install django-cache-machine -or `github `_:: - - pip install -e git://github.com/django-cache-machine/django-cache-machine.git#egg=django-cache-machine - Running Tests ------------- diff --git a/caching/backends/__init__.py b/caching/backends/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/caching/backends/locmem.py b/caching/backends/locmem.py deleted file mode 100644 index 5276c93..0000000 --- a/caching/backends/locmem.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import unicode_literals - -import django -from django.core.cache.backends import locmem - -from caching.compat import DEFAULT_TIMEOUT, FOREVER - - -if django.VERSION[:2] >= (1, 6): - Infinity = FOREVER -else: - class _Infinity(object): - """Always compares greater than numbers.""" - - def __radd__(self, _): - return self - - def __cmp__(self, o): - return 0 if self is o else 1 - - def __repr__(self): - return 'Infinity' - - Infinity = _Infinity() - del _Infinity - - -# Add infinite timeout support to the locmem backend. Useful for testing. -class InfinityMixin(object): - - def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): - if timeout == FOREVER: - timeout = Infinity - return super(InfinityMixin, self).add(key, value, timeout, version) - - def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): - if timeout == FOREVER: - timeout = Infinity - return super(InfinityMixin, self).set(key, value, timeout, version) - - -class LocMemCache(InfinityMixin, locmem.LocMemCache): - pass diff --git a/caching/backends/memcached.py b/caching/backends/memcached.py deleted file mode 100644 index f08f9bf..0000000 --- a/caching/backends/memcached.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import unicode_literals - -import django -from django.core.cache.backends import memcached - -from caching.compat import DEFAULT_TIMEOUT - - -# Add infinite timeout support to the memcached backend, if needed. -class InfinityMixin(object): - - if django.VERSION[:2] < (1, 6): - # Django 1.6 and later do it the right way already - def _get_memcache_timeout(self, timeout): - if timeout == 0: - return timeout - else: - return super(InfinityMixin, self)._get_memcache_timeout(timeout) - - def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): - return super(InfinityMixin, self).add(key, value, timeout, version) - - def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None): - return super(InfinityMixin, self).set(key, value, timeout, version) - - -class MemcachedCache(InfinityMixin, memcached.MemcachedCache): - pass - - -class PyLibMCCache(InfinityMixin, memcached.PyLibMCCache): - pass diff --git a/caching/base.py b/caching/base.py index 0e7fd8f..57853dc 100644 --- a/caching/base.py +++ b/caching/base.py @@ -3,14 +3,13 @@ import functools import logging -import django +from django.core.cache.backends.base import DEFAULT_TIMEOUT from django.db import models from django.db.models import signals from django.db.models.sql import query, EmptyResultSet from django.utils import encoding from caching import config -from .compat import DEFAULT_TIMEOUT from .invalidation import invalidator, flush_key, make_key, byid, cache @@ -32,9 +31,6 @@ class CachingManager(models.Manager): def get_queryset(self): return CachingQuerySet(self.model, using=self._db) - if django.VERSION < (1, 6): - get_query_set = get_queryset - def contribute_to_class(self, cls, name): signals.post_save.connect(self.post_save, sender=cls) signals.post_delete.connect(self.post_delete, sender=cls) diff --git a/caching/compat.py b/caching/compat.py deleted file mode 100644 index cd1731a..0000000 --- a/caching/compat.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import unicode_literals - -import django - -__all__ = ['DEFAULT_TIMEOUT', 'FOREVER'] - - -if django.VERSION[:2] >= (1, 6): - from django.core.cache.backends.base import DEFAULT_TIMEOUT as DJANGO_DEFAULT_TIMEOUT - DEFAULT_TIMEOUT = DJANGO_DEFAULT_TIMEOUT - FOREVER = None -else: - DEFAULT_TIMEOUT = None - FOREVER = 0 diff --git a/caching/invalidation.py b/caching/invalidation.py index 6c75000..360c80e 100644 --- a/caching/invalidation.py +++ b/caching/invalidation.py @@ -6,13 +6,15 @@ import logging import socket -import django from django.conf import settings from django.core.cache import cache as default_cache +from django.core.cache import caches from django.core.cache.backends.base import InvalidCacheBackendError from django.utils import encoding, translation, six from django.utils.six.moves.urllib.parse import parse_qsl +from caching import config + try: import redis as redislib except ImportError: @@ -20,17 +22,10 @@ # Look for an own cache first before falling back to the default cache try: - if django.VERSION[:2] >= (1, 7): - from django.core.cache import caches - cache = caches['cache_machine'] - else: - from django.core.cache import get_cache - cache = get_cache('cache_machine') + cache = caches['cache_machine'] except (InvalidCacheBackendError, ValueError): cache = default_cache -from caching import config - log = logging.getLogger('caching.invalidation') diff --git a/docs/index.rst b/docs/index.rst index 28eb041..aff85a7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,59 +14,9 @@ affect you, please see the :ref:`release-notes`. Settings -------- -Before we start, you'll have to update your ``settings.py`` to use one of the -caching backends provided by Cache Machine. Prior to Django 1.6, Django's -built-in caching backends did not allow for infinite cache timeouts, -which are critical for doing invalidation (see below). Cache Machine extends -the ``locmem`` and ``memcached`` backends provided by Django to enable -indefinite caching when a timeout of ``caching.base.FOREVER`` is -passed. If you were already using one of these backends, you can probably go -on using them just as you were. - -With Django 1.4 or higher, you should use the ``CACHES`` setting:: - - CACHES = { - 'default': { - 'BACKEND': 'caching.backends.memcached.MemcachedCache', - 'LOCATION': [ - 'server-1:11211', - 'server-2:11211', - ], - 'KEY_PREFIX': 'weee:', - }, - } - -Note that we have to specify the class, not the module, for the ``BACKEND`` -property, and that the ``KEY_PREFIX`` is optional. The ``LOCATION`` may be a -string, instead of a list, if you only have one server. - -If you require the default cache backend to be a different type of -cache backend or want Cache Machine to use specific cache server -options simply define a separate ``cache_machine`` entry for the -``CACHES`` setting, e.g.:: - - CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': 'server-1:11211', - }, - 'cache_machine': { - 'BACKEND': 'caching.backends.memcached.MemcachedCache', - 'LOCATION': [ - 'server-1:11211', - 'server-2:11211', - ], - 'KEY_PREFIX': 'weee:', - }, - } - -.. note:: - - Cache Machine also supports the other memcache backend support by - Django >= 1.4 based on pylibmc_: - ``caching.backends.memcached.PyLibMCCache``. - -.. _pylibmc: http://sendapatch.se/projects/pylibmc/ +Older versions of Cache Machine required you to use customized cache backends. As of Django 1.6, +these are no longer needed and they have been removed from Cache Machine. Use the standard Django +cache backends. COUNT queries ^^^^^^^^^^^^^ diff --git a/examples/cache_machine/custom_backend.py b/examples/cache_machine/custom_backend.py index 53e2789..c9030d0 100644 --- a/examples/cache_machine/custom_backend.py +++ b/examples/cache_machine/custom_backend.py @@ -5,7 +5,7 @@ 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', }, 'cache_machine': { - 'BACKEND': 'caching.backends.memcached.MemcachedCache', + 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': 'localhost:11211', }, } diff --git a/examples/cache_machine/locmem_settings.py b/examples/cache_machine/locmem_settings.py index bfe4c62..8ed4fba 100644 --- a/examples/cache_machine/locmem_settings.py +++ b/examples/cache_machine/locmem_settings.py @@ -2,6 +2,6 @@ CACHES = { 'default': { - 'BACKEND': 'caching.backends.locmem.LocMemCache', + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', }, } diff --git a/examples/cache_machine/settings.py b/examples/cache_machine/settings.py index 4cba276..2b6027f 100644 --- a/examples/cache_machine/settings.py +++ b/examples/cache_machine/settings.py @@ -2,7 +2,7 @@ CACHES = { 'default': { - 'BACKEND': 'caching.backends.memcached.MemcachedCache', + 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': 'localhost:11211', }, } diff --git a/requirements/base.txt b/requirements/base.txt index 8fc8122..f9a98f4 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,4 @@ # These are the reqs to build docs and run tests. ---no-binary :all: # workaround for https://bitbucket.org/ned/coveragepy/issue/382/pip-install-coverage-uses-slower-pytracer sphinx django-nose jinja2 diff --git a/setup.py b/setup.py index d0b6afa..f6559e8 100644 --- a/setup.py +++ b/setup.py @@ -26,11 +26,11 @@ 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Software Development :: Libraries :: Python Modules', ] ) diff --git a/tests/test_cache.py b/tests/test_cache.py index 5a86588..d59173a 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,22 +1,19 @@ from __future__ import unicode_literals -import django import jinja2 import logging import pickle import sys +import unittest from django.conf import settings +from django.core.cache.backends.base import DEFAULT_TIMEOUT from django.test import TestCase, TransactionTestCase from django.utils import translation, encoding -from caching import base, invalidation, config, compat +from caching import base, invalidation, config from .testapp.models import Addon, User -if sys.version_info < (2, 7): - import unittest2 as unittest -else: - import unittest if sys.version_info >= (3, ): from unittest import mock @@ -27,22 +24,6 @@ cache = invalidation.cache log = logging.getLogger(__name__) -if django.get_version().startswith('1.3'): - class settings_patch(object): - def __init__(self, **kwargs): - self.options = kwargs - - def __enter__(self): - self._old_settings = dict((k, getattr(settings, k, None)) for k in self.options) - for k, v in list(self.options.items()): - setattr(settings, k, v) - - def __exit__(self, *args): - for k in self.options: - setattr(settings, k, self._old_settings[k]) - - TestCase.settings = settings_patch - class CachingTestCase(TestCase): fixtures = ['tests/testapp/fixtures/testapp/test_cache.json'] @@ -423,7 +404,7 @@ def test_infinite_timeout(self, mock_set): """ Test that memcached infinite timeouts work with all Django versions. """ - cache.set('foo', 'bar', timeout=compat.FOREVER) + cache.set('foo', 'bar', timeout=None) # for memcached, 0 timeout means store forever mock_set.assert_called_with(':1:foo', 'bar', 0) @@ -546,7 +527,7 @@ def test_pickle_queryset(self): # pickled/unpickled on/from different Python processes which may have different # underlying values for DEFAULT_TIMEOUT: q1 = Addon.objects.all() - self.assertEqual(q1.timeout, compat.DEFAULT_TIMEOUT) + self.assertEqual(q1.timeout, DEFAULT_TIMEOUT) pickled = pickle.dumps(q1) new_timeout = object() with mock.patch('caching.base.DEFAULT_TIMEOUT', new_timeout): From 36589e98655289c15ecd71180daaa0f2b09e402e Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 6 Oct 2017 23:08:25 -0400 Subject: [PATCH 13/25] Remove obsolete package --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f6559e8..519a5b7 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ author_email='jbalogh@mozilla.com', url='http://github.com/django-cache-machine/django-cache-machine', license='BSD', - packages=['caching', 'caching.backends'], + packages=['caching'], include_package_data=True, zip_safe=False, classifiers=[ From f6839e4274e56f3cce892faa017f8c33b40a1ad0 Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 6 Oct 2017 23:25:27 -0400 Subject: [PATCH 14/25] Django 1.9 support --- .travis.yml | 2 +- README.rst | 2 +- caching/base.py | 32 +++++++++++++++++++++++--------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 03dca3a..ff230c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ after_success: - coveralls env: - TOX_ENV="dj18-py27,dj18-py34,dj18-py35" - # - TOX_ENV="dj19-py27,dj19-py34,dj19-py35,dj19-py36" + - TOX_ENV="dj19-py27,dj19-py34,dj19-py35,dj19-py36" # - TOX_ENV="dj110-py27,dj110-py34,dj110-py35,dj110-py36" # - TOX_ENV="dj111-py27,dj111-py34,dj111-py35,dj111-py36" - TOX_ENV="py27-flake8,py36-flake8" diff --git a/README.rst b/README.rst index 8a61283..3113605 100644 --- a/README.rst +++ b/README.rst @@ -17,7 +17,7 @@ For full docs, see https://cache-machine.readthedocs.org/en/latest/. Requirements ------------ -Cache Machine works with Django 1.8 and Python 2.7, 3.4, 3.5 and 3.6. +Cache Machine works with Django 1.8-1.9 and Python 2.7, 3.4, 3.5 and 3.6. Installation diff --git a/caching/base.py b/caching/base.py index 57853dc..4d44e05 100644 --- a/caching/base.py +++ b/caching/base.py @@ -9,6 +9,13 @@ from django.db.models.sql import query, EmptyResultSet from django.utils import encoding +try: + from django.db.models.query import ValuesListIterable +except ImportError: + # ValuesListIterable is defined in Django 1.9+, and if it's present, we + # need to workaround a possible infinite recursion. See CachingQuerySet.iterator() + ValuesListIterable = None + from caching import config from .invalidation import invalidator, flush_key, make_key, byid, cache @@ -162,15 +169,22 @@ def iterator(self): iterator = super(CachingQuerySet, self).iterator if self.timeout == config.NO_CACHE: return iter(iterator()) - else: - try: - # Work-around for Django #12717. - query_string = self.query_key() - except query.EmptyResultSet: - return iterator() - if config.FETCH_BY_ID: - iterator = self.fetch_by_id - return iter(CacheMachine(self.model, query_string, iterator, self.timeout, db=self.db)) + + try: + # Work-around for Django #12717. + query_string = self.query_key() + except query.EmptyResultSet: + return iterator() + if config.FETCH_BY_ID: + # fetch_by_id uses a ValuesList to get a list of pks. If we are + # currently about to run that query, we DON'T want to use the + # fetch_by_id iterator or else we will run into an infinite + # recursion. So, if we are about to run that query, use the + # standard iterator. + if ValuesListIterable and self._iterable_class == ValuesListIterable: + return iter(iterator()) + iterator = self.fetch_by_id + return iter(CacheMachine(self.model, query_string, iterator, self.timeout, db=self.db)) def fetch_by_id(self): """ From 3b562d4358e15ff6c7e7d75817653bda901ac625 Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 6 Oct 2017 23:33:35 -0400 Subject: [PATCH 15/25] Django 1.10 support --- README.rst | 2 +- caching/ext.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3113605..4837e8f 100644 --- a/README.rst +++ b/README.rst @@ -17,7 +17,7 @@ For full docs, see https://cache-machine.readthedocs.org/en/latest/. Requirements ------------ -Cache Machine works with Django 1.8-1.9 and Python 2.7, 3.4, 3.5 and 3.6. +Cache Machine works with Django 1.8-1.10 and Python 2.7, 3.4, 3.5 and 3.6. Installation diff --git a/caching/ext.py b/caching/ext.py index acdc226..c5b5e71 100644 --- a/caching/ext.py +++ b/caching/ext.py @@ -69,7 +69,7 @@ def process_cache_arguments(self, args): def _cache_support(self, name, obj, timeout, extra, caller): """Cache helper callback.""" - if settings.TEMPLATE_DEBUG: + if settings.DEBUG: return caller() extra = ':'.join(map(encoding.smart_str, extra)) key = 'fragment:%s:%s' % (name, extra) From b7295ff8409b0aeb0b8c7f3c592656507b855a6b Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 6 Oct 2017 23:34:40 -0400 Subject: [PATCH 16/25] Tell Travis to test 1.10 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ff230c2..4690a28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ after_success: env: - TOX_ENV="dj18-py27,dj18-py34,dj18-py35" - TOX_ENV="dj19-py27,dj19-py34,dj19-py35,dj19-py36" - # - TOX_ENV="dj110-py27,dj110-py34,dj110-py35,dj110-py36" + - TOX_ENV="dj110-py27,dj110-py34,dj110-py35,dj110-py36" # - TOX_ENV="dj111-py27,dj111-py34,dj111-py35,dj111-py36" - TOX_ENV="py27-flake8,py36-flake8" - TOX_ENV="docs" From 6a547d8e4295985b2a84d4e115b1192cff540dbc Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Sat, 7 Oct 2017 10:03:09 -0400 Subject: [PATCH 17/25] Django 1.11 support --- .travis.yml | 2 +- README.rst | 2 +- caching/base.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4690a28..603232c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ env: - TOX_ENV="dj18-py27,dj18-py34,dj18-py35" - TOX_ENV="dj19-py27,dj19-py34,dj19-py35,dj19-py36" - TOX_ENV="dj110-py27,dj110-py34,dj110-py35,dj110-py36" - # - TOX_ENV="dj111-py27,dj111-py34,dj111-py35,dj111-py36" + - TOX_ENV="dj111-py27,dj111-py34,dj111-py35,dj111-py36" - TOX_ENV="py27-flake8,py36-flake8" - TOX_ENV="docs" # Adding sudo: False tells Travis to use their container-based infrastructure, which is somewhat faster. diff --git a/README.rst b/README.rst index 4837e8f..1475cce 100644 --- a/README.rst +++ b/README.rst @@ -17,7 +17,7 @@ For full docs, see https://cache-machine.readthedocs.org/en/latest/. Requirements ------------ -Cache Machine works with Django 1.8-1.10 and Python 2.7, 3.4, 3.5 and 3.6. +Cache Machine works with Django 1.8-1.11 and Python 2.7, 3.4, 3.5 and 3.6. Installation diff --git a/caching/base.py b/caching/base.py index 4d44e05..f4ad1ed 100644 --- a/caching/base.py +++ b/caching/base.py @@ -157,6 +157,18 @@ def __setstate__(self, state): if self.timeout == self._default_timeout_pickle_key: self.timeout = DEFAULT_TIMEOUT + def _fetch_all(self): + """ + Django 1.11 changed _fetch_all to use self._iterable_class() rather than + self.iterator(). That bypasses our iterator, so override Queryset._fetch_all + to use our iterator. + + https://github.com/django/django/commit/f3b7c059367a4e82bbfc7e4f0d42b10975e79f0c#diff-5b0dda5eb9a242c15879dc9cd2121379 + """ + if self._result_cache is None: + self._result_cache = list(self.iterator()) + super(CachingQuerySet, self)._fetch_all() + def flush_key(self): return flush_key(self.query_key()) From 7bd4f770990f379155f97bd94c70cda9b205e78e Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Sat, 7 Oct 2017 10:03:22 -0400 Subject: [PATCH 18/25] Cleanup and docs updates --- caching/base.py | 10 +--------- docs/index.rst | 5 ++--- docs/releases.rst | 21 ++++++++++++++++++++- requirements/base.txt | 1 + requirements/py2.txt | 2 -- requirements/py3.txt | 1 - 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/caching/base.py b/caching/base.py index f4ad1ed..bccb853 100644 --- a/caching/base.py +++ b/caching/base.py @@ -17,17 +17,9 @@ ValuesListIterable = None from caching import config -from .invalidation import invalidator, flush_key, make_key, byid, cache - - -class NullHandler(logging.Handler): - - def emit(self, record): - pass - +from caching.invalidation import invalidator, flush_key, make_key, byid, cache log = logging.getLogger('caching') -log.addHandler(NullHandler()) class CachingManager(models.Manager): diff --git a/docs/index.rst b/docs/index.rst index aff85a7..655f464 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,9 +14,8 @@ affect you, please see the :ref:`release-notes`. Settings -------- -Older versions of Cache Machine required you to use customized cache backends. As of Django 1.6, -these are no longer needed and they have been removed from Cache Machine. Use the standard Django -cache backends. +Older versions of Cache Machine required you to use customized cache backends. These are no longer +needed and they have been removed from Cache Machine. Use the standard Django cache backends. COUNT queries ^^^^^^^^^^^^^ diff --git a/docs/releases.rst b/docs/releases.rst index a047f62..6e8f64b 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -3,8 +3,27 @@ Release Notes ================== +v1.0.0dev (TBD) +--------------- + +- Update Travis and Tox configurations +- Drop support for Python < 2.7 +- Add support for Python 3.5 and 3.6 +- Drop support for Django < 1.8 +- Add support for Django 1.9, 1.10, and 1.11 +- Removed all custom cache backends. +- Flake8 fixes + +Backwards Incompatible Changes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Cache Machine previously included custom backends for LocMem, Memcached and PyLibMemcached. These + were necessary because the core backends in old versions of Django did not support infinte + timeouts. They now do, so Cache Machine's custom backends are no longer necessary. They have been + removed, so you should revert to using the core Django backends. + v0.9.1 (2015-10-22) ------------------ +------------------- - Fix bug that prevented objects retrieved via cache machine from being re-cached by application code (see PR #103) diff --git a/requirements/base.txt b/requirements/base.txt index f9a98f4..ed304fb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,3 +6,4 @@ redis flake8 coverage psycopg2 +python-memcached>=1.58 diff --git a/requirements/py2.txt b/requirements/py2.txt index 3bf2f67..032a559 100644 --- a/requirements/py2.txt +++ b/requirements/py2.txt @@ -1,4 +1,2 @@ -r base.txt -python-memcached mock==1.0.1 -unittest2 diff --git a/requirements/py3.txt b/requirements/py3.txt index ced3eed..a3e81b8 100644 --- a/requirements/py3.txt +++ b/requirements/py3.txt @@ -1,2 +1 @@ -r base.txt -python3-memcached From 24c977923ab5fee5525fb5fea0df6b4859e53316 Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Sat, 7 Oct 2017 23:29:12 -0400 Subject: [PATCH 19/25] Add tests for failure when running .values() or .values_list() --- caching/base.py | 19 ++++++++----------- tests/test_cache.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/caching/base.py b/caching/base.py index bccb853..8415fc2 100644 --- a/caching/base.py +++ b/caching/base.py @@ -10,11 +10,11 @@ from django.utils import encoding try: - from django.db.models.query import ValuesListIterable + from django.db.models.query import ModelIterable except ImportError: - # ValuesListIterable is defined in Django 1.9+, and if it's present, we + # ModelIterable is defined in Django 1.9+, and if it's present, we # need to workaround a possible infinite recursion. See CachingQuerySet.iterator() - ValuesListIterable = None + ModelIterable = None from caching import config from caching.invalidation import invalidator, flush_key, make_key, byid, cache @@ -173,20 +173,17 @@ def iterator(self): iterator = super(CachingQuerySet, self).iterator if self.timeout == config.NO_CACHE: return iter(iterator()) - + # ModelIterable and _iterable_class are introduced in Django 1.9. We only cache + # ModelIterable querysets (because we mark each instance as being cached with a `from_cache` + # attribute, and can't do so with dictionaries or tuples) + if getattr(self, '_iterable_class', None) != ModelIterable: + return iter(iterator()) try: # Work-around for Django #12717. query_string = self.query_key() except query.EmptyResultSet: return iterator() if config.FETCH_BY_ID: - # fetch_by_id uses a ValuesList to get a list of pks. If we are - # currently about to run that query, we DON'T want to use the - # fetch_by_id iterator or else we will run into an infinite - # recursion. So, if we are about to run that query, use the - # standard iterator. - if ValuesListIterable and self._iterable_class == ValuesListIterable: - return iter(iterator()) iterator = self.fetch_by_id return iter(CacheMachine(self.model, query_string, iterator, self.timeout, db=self.db)) diff --git a/tests/test_cache.py b/tests/test_cache.py index d59173a..1186a13 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -63,6 +63,16 @@ def test_slice_cache(self): self.assertIs(Addon.objects.filter(id=1)[:1][0].from_cache, False) self.assertIs(Addon.objects.filter(id=1)[:1][0].from_cache, True) + def test_should_not_cache_values1(self): + with self.assertNumQueries(2): + Addon.objects.values('id')[0] + Addon.objects.values('id')[0] + + def test_should_not_cache_values_list(self): + with self.assertNumQueries(2): + Addon.objects.values_list('id')[0] + Addon.objects.values_list('id')[0] + def test_invalidation(self): self.assertIs(Addon.objects.get(id=1).from_cache, False) a = [x for x in Addon.objects.all() if x.id == 1][0] From f98e32fd9f8e838fd088b41be21dd65860cada42 Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Mon, 9 Oct 2017 08:31:17 -0400 Subject: [PATCH 20/25] Typo --- tests/test_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cache.py b/tests/test_cache.py index 1186a13..8a87246 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -63,7 +63,7 @@ def test_slice_cache(self): self.assertIs(Addon.objects.filter(id=1)[:1][0].from_cache, False) self.assertIs(Addon.objects.filter(id=1)[:1][0].from_cache, True) - def test_should_not_cache_values1(self): + def test_should_not_cache_values(self): with self.assertNumQueries(2): Addon.objects.values('id')[0] Addon.objects.values('id')[0] From 4f1fb8e2b0c75e9d19641bc46cc681fb49c64d54 Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Wed, 11 Oct 2017 15:19:25 -0400 Subject: [PATCH 21/25] Attempt to tease out Django 1.8 differences --- caching/base.py | 146 ++++++++++++++++++++++++-------------------- tests/test_cache.py | 12 ++-- 2 files changed, 86 insertions(+), 72 deletions(-) diff --git a/caching/base.py b/caching/base.py index 8415fc2..c9a4d9b 100644 --- a/caching/base.py +++ b/caching/base.py @@ -9,16 +9,16 @@ from django.db.models.sql import query, EmptyResultSet from django.utils import encoding +from caching import config +from caching.invalidation import invalidator, flush_key, make_key, byid, cache + try: from django.db.models.query import ModelIterable except ImportError: # ModelIterable is defined in Django 1.9+, and if it's present, we - # need to workaround a possible infinite recursion. See CachingQuerySet.iterator() + # use it iterate over our results. If not, we fall back to a Django 1.8 compatible way. ModelIterable = None -from caching import config -from caching.invalidation import invalidator, flush_key, make_key, byid, cache - log = logging.getLogger('caching') @@ -57,21 +57,11 @@ def no_cache(self): return self.cache(config.NO_CACHE) -class CacheMachine(object): +class CacheInternalCommonMixin(object): """ - Handles all the cache management for a QuerySet. - - Takes the string representation of a query and a function that can be - called to get an iterator over some database results. + A set of methods common to our Django 1.8 and Django 1.9+ iterators. """ - def __init__(self, model, query_string, iter_function, timeout=DEFAULT_TIMEOUT, db='default'): - self.model = model - self.query_string = query_string - self.iter_function = iter_function - self.timeout = timeout - self.db = db - def query_key(self): """ Generate the cache key for this query. @@ -81,16 +71,39 @@ def query_key(self): master), throwing a Django ValueError in the process. Django prevents cross DB model saving among related objects. """ - query_db_string = 'qs:%s::db:%s' % (self.query_string, self.db) + query_db_string = 'qs:%s::db:%s' % (self.queryset.query_key(), self.db) return make_key(query_db_string, with_locale=False) + def cache_objects(self, objects, query_key): + """Cache query_key => objects, then update the flush lists.""" + log.debug('query_key: %s' % query_key) + query_flush = flush_key(self.queryset.query_key()) + log.debug('query_flush: %s' % query_flush) + cache.add(query_key, objects, timeout=self.timeout) + invalidator.cache_objects(self.queryset.model, objects, query_key, query_flush) + def __iter__(self): + if hasattr(super(CacheInternalCommonMixin, self), '__iter__'): + # This is the Django 1.9+ class, so we'll use super().__iter__ + # which is a ModelIterable iterator. + iterator = super(CacheInternalCommonMixin, self).__iter__ + else: + # This is Django 1.8. Use the function passed into the class + # constructor. + iterator = self.iter_function + + if self.timeout == config.NO_CACHE: + # no cache, just iterate and return the results + for obj in iterator(): + yield obj + return + + # Try to fetch from the cache. try: query_key = self.query_key() except query.EmptyResultSet: raise StopIteration - # Try to fetch from the cache. cached = cache.get(query_key) if cached is not None: log.debug('cache hit: %s' % query_key) @@ -99,28 +112,47 @@ def __iter__(self): yield obj return - # Do the database query, cache it once we have all the objects. - iterator = self.iter_function() + # Use the special FETCH_BY_ID iterator if configured. + if config.FETCH_BY_ID and hasattr(self.queryset, 'fetch_by_id'): + iterator = self.queryset.fetch_by_id + # No cached results. Do the database query, and cache it once we have + # all the objects. to_cache = [] - try: - while True: - obj = next(iterator) - obj.from_cache = False - to_cache.append(obj) - yield obj - except StopIteration: - if to_cache or config.CACHE_EMPTY_QUERYSETS: - self.cache_objects(to_cache, query_key) - raise + for obj in iterator(): + obj.from_cache = False + to_cache.append(obj) + yield obj + if to_cache or config.CACHE_EMPTY_QUERYSETS: + self.cache_objects(to_cache, query_key) - def cache_objects(self, objects, query_key): - """Cache query_key => objects, then update the flush lists.""" - log.debug('query_key: %s' % query_key) - query_flush = flush_key(self.query_string) - log.debug('query_flush: %s' % query_flush) - cache.add(query_key, objects, timeout=self.timeout) - invalidator.cache_objects(self.model, objects, query_key, query_flush) + +class CacheMachine(CacheInternalCommonMixin): + """ + Handles all the cache management for a QuerySet. + + Takes the string representation of a query and a function that can be + called to get an iterator over some database results. + """ + + def __init__(self, queryset, iter_function=None, timeout=DEFAULT_TIMEOUT, db='default'): + self.queryset = queryset + self.iter_function = iter_function + self.timeout = timeout + self.db = db + + +if ModelIterable: + class CachingModelIterable(CacheInternalCommonMixin, ModelIterable): + """ + A version of Django's ModelIterable that first tries to get results from the cache. + """ + + def __init__(self, *args, **kwargs): + super(CachingModelIterable, self).__init__(*args, **kwargs) + # copy timeout and db from queryset to allow CacheInternalCommonMixin to be DRYer + self.timeout = self.queryset.timeout + self.db = self.queryset.db class CachingQuerySet(models.query.QuerySet): @@ -130,6 +162,9 @@ class CachingQuerySet(models.query.QuerySet): def __init__(self, *args, **kw): super(CachingQuerySet, self).__init__(*args, **kw) self.timeout = DEFAULT_TIMEOUT + if ModelIterable: + # Django 1.9+ + self._iterable_class = CachingModelIterable def __getstate__(self): """ @@ -149,18 +184,6 @@ def __setstate__(self, state): if self.timeout == self._default_timeout_pickle_key: self.timeout = DEFAULT_TIMEOUT - def _fetch_all(self): - """ - Django 1.11 changed _fetch_all to use self._iterable_class() rather than - self.iterator(). That bypasses our iterator, so override Queryset._fetch_all - to use our iterator. - - https://github.com/django/django/commit/f3b7c059367a4e82bbfc7e4f0d42b10975e79f0c#diff-5b0dda5eb9a242c15879dc9cd2121379 - """ - if self._result_cache is None: - self._result_cache = list(self.iterator()) - super(CachingQuerySet, self)._fetch_all() - def flush_key(self): return flush_key(self.query_key()) @@ -170,22 +193,11 @@ def query_key(self): return sql % params def iterator(self): + if ModelIterable: + # Django 1.9+ + return self._iterable_class(self) iterator = super(CachingQuerySet, self).iterator - if self.timeout == config.NO_CACHE: - return iter(iterator()) - # ModelIterable and _iterable_class are introduced in Django 1.9. We only cache - # ModelIterable querysets (because we mark each instance as being cached with a `from_cache` - # attribute, and can't do so with dictionaries or tuples) - if getattr(self, '_iterable_class', None) != ModelIterable: - return iter(iterator()) - try: - # Work-around for Django #12717. - query_string = self.query_key() - except query.EmptyResultSet: - return iterator() - if config.FETCH_BY_ID: - iterator = self.fetch_by_id - return iter(CacheMachine(self.model, query_string, iterator, self.timeout, db=self.db)) + return iter(CacheMachine(self, iterator, self.timeout, db=self.db)) def fetch_by_id(self): """ @@ -320,11 +332,13 @@ def __iter__(self): while True: yield next(iterator) else: - sql = self.raw_query % tuple(self.params) - for obj in CacheMachine(self.model, sql, iterator, timeout=self.timeout): + for obj in CacheMachine(self, iterator, timeout=self.timeout): yield obj raise StopIteration + def query_key(self): + return self.raw_query % tuple(self.params) + def _function_cache_key(key): return make_key('f:%s' % key, with_locale=True) diff --git a/tests/test_cache.py b/tests/test_cache.py index 8a87246..25f40cb 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -378,19 +378,19 @@ def test_cached_method(self): # Make sure we're updating the wrapper's docstring. self.assertEqual(b.calls.__doc__, Addon.calls.__doc__) - @mock.patch('caching.base.CacheMachine') - def test_no_cache_from_manager(self, CacheMachine): + @mock.patch('caching.base.cache.get') + def test_no_cache_from_manager(self, mock_cache): a = Addon.objects.no_cache().get(id=1) self.assertEqual(a.id, 1) self.assertFalse(hasattr(a, 'from_cache')) - self.assertFalse(CacheMachine.called) + self.assertFalse(mock_cache.called) - @mock.patch('caching.base.CacheMachine') - def test_no_cache_from_queryset(self, CacheMachine): + @mock.patch('caching.base.cache.get') + def test_no_cache_from_queryset(self, mock_cache): a = Addon.objects.all().no_cache().get(id=1) self.assertEqual(a.id, 1) self.assertFalse(hasattr(a, 'from_cache')) - self.assertFalse(CacheMachine.called) + self.assertFalse(mock_cache.called) def test_timeout_from_manager(self): q = Addon.objects.cache(12).filter(id=1) From 085bf3552c66a017464f2806837484b9f47e8295 Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Thu, 12 Oct 2017 18:57:44 -0400 Subject: [PATCH 22/25] enable --keepdb for faster test runs --- run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.py b/run_tests.py index d33f08f..c4f7d33 100644 --- a/run_tests.py +++ b/run_tests.py @@ -47,7 +47,7 @@ def main(): test_cmd = ['coverage', 'run'] else: test_cmd = [] - test_cmd += [django_admin, 'test'] + test_cmd += [django_admin, 'test', '--keepdb'] results.append(call(test_cmd)) if args.with_coverage: results.append(call(['coverage', 'report', '-m', '--fail-under', '70'])) From 52d4ea6abc79946f737cc9a711ba95ca291653ab Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Thu, 12 Oct 2017 19:03:12 -0400 Subject: [PATCH 23/25] try to simplify Django 1.8/Django 1.11 compatibility --- caching/base.py | 81 +++++++++++++++++---------------------------- tests/test_cache.py | 6 ++-- 2 files changed, 33 insertions(+), 54 deletions(-) diff --git a/caching/base.py b/caching/base.py index c9a4d9b..a1a87ea 100644 --- a/caching/base.py +++ b/caching/base.py @@ -13,11 +13,17 @@ from caching.invalidation import invalidator, flush_key, make_key, byid, cache try: + # ModelIterable is defined in Django 1.9+, and if it's present, we use it + # iterate over our results. from django.db.models.query import ModelIterable except ImportError: - # ModelIterable is defined in Django 1.9+, and if it's present, we - # use it iterate over our results. If not, we fall back to a Django 1.8 compatible way. - ModelIterable = None + # If not, define a Django 1.8-compatible stub we can use instead. + class ModelIterable(object): + def __init__(self, queryset): + self.queryset = queryset + + def __iter__(self): + return super(CachingQuerySet, self.queryset).iterator() log = logging.getLogger('caching') @@ -57,11 +63,20 @@ def no_cache(self): return self.cache(config.NO_CACHE) -class CacheInternalCommonMixin(object): +class CachingModelIterable(ModelIterable): """ - A set of methods common to our Django 1.8 and Django 1.9+ iterators. + Handles all the cache management for a QuerySet. + + Takes the string representation of a query and a function that can be + called to get an iterator over some database results. """ + def __init__(self, queryset, *args, **kwargs): + self.iter_function = kwargs.pop('iter_function', None) + self.timeout = kwargs.pop('timeout', queryset.timeout) + self.db = kwargs.pop('db', queryset.db) + super(CachingModelIterable, self).__init__(queryset, *args, **kwargs) + def query_key(self): """ Generate the cache key for this query. @@ -83,14 +98,13 @@ def cache_objects(self, objects, query_key): invalidator.cache_objects(self.queryset.model, objects, query_key, query_flush) def __iter__(self): - if hasattr(super(CacheInternalCommonMixin, self), '__iter__'): - # This is the Django 1.9+ class, so we'll use super().__iter__ - # which is a ModelIterable iterator. - iterator = super(CacheInternalCommonMixin, self).__iter__ - else: - # This is Django 1.8. Use the function passed into the class - # constructor. + if self.iter_function is not None: + # This a RawQuerySet. Use the function passed into + # the class constructor. iterator = self.iter_function + else: + # Otherwise, use super().__iter__. + iterator = super(CachingModelIterable, self).__iter__ if self.timeout == config.NO_CACHE: # no cache, just iterate and return the results @@ -102,7 +116,7 @@ def __iter__(self): try: query_key = self.query_key() except query.EmptyResultSet: - raise StopIteration + return cached = cache.get(query_key) if cached is not None: @@ -127,34 +141,6 @@ def __iter__(self): self.cache_objects(to_cache, query_key) -class CacheMachine(CacheInternalCommonMixin): - """ - Handles all the cache management for a QuerySet. - - Takes the string representation of a query and a function that can be - called to get an iterator over some database results. - """ - - def __init__(self, queryset, iter_function=None, timeout=DEFAULT_TIMEOUT, db='default'): - self.queryset = queryset - self.iter_function = iter_function - self.timeout = timeout - self.db = db - - -if ModelIterable: - class CachingModelIterable(CacheInternalCommonMixin, ModelIterable): - """ - A version of Django's ModelIterable that first tries to get results from the cache. - """ - - def __init__(self, *args, **kwargs): - super(CachingModelIterable, self).__init__(*args, **kwargs) - # copy timeout and db from queryset to allow CacheInternalCommonMixin to be DRYer - self.timeout = self.queryset.timeout - self.db = self.queryset.db - - class CachingQuerySet(models.query.QuerySet): _default_timeout_pickle_key = '__DEFAULT_TIMEOUT__' @@ -162,9 +148,7 @@ class CachingQuerySet(models.query.QuerySet): def __init__(self, *args, **kw): super(CachingQuerySet, self).__init__(*args, **kw) self.timeout = DEFAULT_TIMEOUT - if ModelIterable: - # Django 1.9+ - self._iterable_class = CachingModelIterable + self._iterable_class = CachingModelIterable def __getstate__(self): """ @@ -193,11 +177,7 @@ def query_key(self): return sql % params def iterator(self): - if ModelIterable: - # Django 1.9+ - return self._iterable_class(self) - iterator = super(CachingQuerySet, self).iterator - return iter(CacheMachine(self, iterator, self.timeout, db=self.db)) + return self._iterable_class(self) def fetch_by_id(self): """ @@ -332,9 +312,8 @@ def __iter__(self): while True: yield next(iterator) else: - for obj in CacheMachine(self, iterator, timeout=self.timeout): + for obj in CachingModelIterable(self, iter_function=iterator, timeout=self.timeout): yield obj - raise StopIteration def query_key(self): return self.raw_query % tuple(self.params) diff --git a/tests/test_cache.py b/tests/test_cache.py index 25f40cb..50fd773 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -165,15 +165,15 @@ def test_raw_cache_params(self): raw2 = list(Addon.objects.raw(sql, [2]))[0] self.assertEqual(raw2.id, 2) - @mock.patch('caching.base.CacheMachine') - def test_raw_nocache(self, CacheMachine): + @mock.patch('caching.base.CachingModelIterable') + def test_raw_nocache(self, CachingModelIterable): base.TIMEOUT = 60 sql = 'SELECT * FROM %s WHERE id = 1' % Addon._meta.db_table raw = list(Addon.objects.raw(sql, timeout=config.NO_CACHE)) self.assertEqual(len(raw), 1) raw_addon = raw[0] self.assertFalse(hasattr(raw_addon, 'from_cache')) - self.assertFalse(CacheMachine.called) + self.assertFalse(CachingModelIterable.called) @mock.patch('caching.base.cache') def test_count_cache(self, cache_mock): From 096f2b2c7b9e8604b728eb53cddc0e2ac6f4f8e5 Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 13 Oct 2017 11:43:53 -0400 Subject: [PATCH 24/25] Minor doc updates --- caching/base.py | 5 +++-- docs/index.rst | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/caching/base.py b/caching/base.py index a1a87ea..0b78ef8 100644 --- a/caching/base.py +++ b/caching/base.py @@ -67,8 +67,9 @@ class CachingModelIterable(ModelIterable): """ Handles all the cache management for a QuerySet. - Takes the string representation of a query and a function that can be - called to get an iterator over some database results. + Takes a queryset, and optionally takes a function that can be called to + get an iterator over some database results. The function is only needed + for RawQuerySets currently. """ def __init__(self, queryset, *args, **kwargs): diff --git a/docs/index.rst b/docs/index.rst index 655f464..960c210 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -202,7 +202,7 @@ file, replacing ``localhost`` with the hostname of your Redis server:: Classes That May Interest You ----------------------------- -.. autoclass:: caching.base.CacheMachine +.. autoclass:: caching.base.CachingModelIterable .. autoclass:: caching.base.CachingManager :members: From 251e5e6bfef49771c988f73a923c132a6ecc453d Mon Sep 17 00:00:00 2001 From: Vinod Kurup Date: Fri, 13 Oct 2017 12:44:15 -0400 Subject: [PATCH 25/25] Bump version for release --- caching/__init__.py | 2 +- docs/releases.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/caching/__init__.py b/caching/__init__.py index 88bdfc2..3d38e44 100644 --- a/caching/__init__.py +++ b/caching/__init__.py @@ -1,4 +1,4 @@ from __future__ import unicode_literals -VERSION = ('0', '9', '1') +VERSION = ('1', '0', '0') __version__ = '.'.join(VERSION) diff --git a/docs/releases.rst b/docs/releases.rst index 6e8f64b..2094e79 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -3,8 +3,8 @@ Release Notes ================== -v1.0.0dev (TBD) ---------------- +v1.0.0 (2017-10-13) +------------------- - Update Travis and Tox configurations - Drop support for Python < 2.7