From 56b9f5b063e3c5df29863b16f1f5da68c4ffcf13 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 3 Jun 2020 10:30:08 +0200 Subject: [PATCH 01/88] [3.0.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index e2082dfde8bc..b567a794347a 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 7, 'final', 0) +VERSION = (3, 0, 8, 'alpha', 0) __version__ = get_version(VERSION) From c1dc423f10e99a13ac9c3a7faf0434c93f1948c9 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 3 Jun 2020 10:54:29 +0200 Subject: [PATCH 02/88] [3.0.x] Added stub release notes for 3.0.8. Backport of 7ec2658e1e24149f0f3244c08c361348f6ebc0e4 from master --- docs/releases/3.0.8.txt | 12 ++++++++++++ docs/releases/index.txt | 1 + 2 files changed, 13 insertions(+) create mode 100644 docs/releases/3.0.8.txt diff --git a/docs/releases/3.0.8.txt b/docs/releases/3.0.8.txt new file mode 100644 index 000000000000..99247a43557c --- /dev/null +++ b/docs/releases/3.0.8.txt @@ -0,0 +1,12 @@ +========================== +Django 3.0.8 release notes +========================== + +*Expected July 1, 2020* + +Django 3.0.8 fixes several bugs in 3.0.7. + +Bugfixes +======== + +* ... diff --git a/docs/releases/index.txt b/docs/releases/index.txt index a9581634e265..3e2074ff820d 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.0.8 3.0.7 3.0.6 3.0.5 From 683d01b0ef830065c4db59ada752b04c34114188 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 3 Jun 2020 11:42:42 +0200 Subject: [PATCH 03/88] [3.0.x] Made default_role_error use logger. This prevents raising errors for translated docs, see https://github.com/django/djangoproject.com/issues/997 Follow up to 1cdfe8d91215eefaa18c398069dd9c6879a9511d. Backport of 36a2e9607e3069e875eedfd814d4473d9b6811cb from master --- docs/_ext/djangodocs.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/_ext/djangodocs.py b/docs/_ext/djangodocs.py index e9f5edc34078..1194c1ab5add 100644 --- a/docs/_ext/djangodocs.py +++ b/docs/_ext/djangodocs.py @@ -11,11 +11,10 @@ from sphinx import addnodes from sphinx.builders.html import StandaloneHTMLBuilder from sphinx.directives.code import CodeBlock -from sphinx.errors import SphinxError from sphinx.domains.std import Cmdoption from sphinx.errors import ExtensionError from sphinx.util import logging -from sphinx.util.console import bold, red +from sphinx.util.console import bold from sphinx.writers.html import HTMLTranslator logger = logging.getLogger(__name__) @@ -379,8 +378,9 @@ def default_role_error( name, rawtext, text, lineno, inliner, options=None, content=None ): msg = ( - "Default role used (`single backticks`) at line %s: %s. Did you mean " - "to use two backticks for ``code``, or miss an underscore for a " - "`link`_ ?" % (lineno, rawtext) + "Default role used (`single backticks`): %s. Did you mean to use two " + "backticks for ``code``, or miss an underscore for a `link`_ ?" + % rawtext ) - raise SphinxError(red(msg)) + logger.warning(msg, location=(inliner.document.current_source, lineno)) + return [nodes.Text(text)], [] From fafbcc57dbfae1810a0a91255780bcf6f4fe09cb Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 3 Jun 2020 12:03:27 +0200 Subject: [PATCH 04/88] [3.0.x] Added CVE-2020-13254 and CVE-2020-13596 to security archive. Backport of 54975780ee2e4017844ecad94835fdce43d97377 from master --- docs/releases/security.txt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index 340aba041b02..d896974e72e8 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -1082,3 +1082,27 @@ Versions affected * Django 3.0 :commit:`(patch) <26a5cf834526e291db00385dd33d319b8271fc4c>` * Django 2.2 :commit:`(patch) ` * Django 1.11 :commit:`(patch) <02d97f3c9a88adc890047996e5606180bd1c6166>` + +June 3, 2020 - :cve:`2020-13254` +-------------------------------- + +Potential data leakage via malformed memcached keys. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.0 :commit:`(patch) <84b2da5552e100ae3294f564f6c862fef8d0e693>` +* Django 2.2 :commit:`(patch) <07e59caa02831c4569bbebb9eb773bdd9cb4b206>` + +June 3, 2020 - :cve:`2020-13596` +-------------------------------- + +Possible XSS via admin ``ForeignKeyRawIdWidget``. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.0 :commit:`(patch) <1f2dd37f6fcefdd10ed44cb233b2e62b520afb38>` +* Django 2.2 :commit:`(patch) <6d61860b22875f358fac83d903dc629897934815>` From 88dc69fcec39ca950a8b031560d508250b28328b Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 3 Jun 2020 12:37:37 +0200 Subject: [PATCH 05/88] [3.0.x] Refs CVE-2020-13254 -- Fixed cache.tests when KEY_PREFIX is defined. Follow up to 2c82414914ae6476be5a166be9ff49c24d0d9069. Backport of 229c9c6653356a0bc23846d83b2d4b5d0438a145 from master --- tests/cache/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cache/tests.py b/tests/cache/tests.py index a30e4ceeb89d..4a705642dc5f 100644 --- a/tests/cache/tests.py +++ b/tests/cache/tests.py @@ -1261,7 +1261,7 @@ def _perform_invalid_key_test(self, key, expected_warning): Whilst other backends merely warn, memcached should raise for an invalid key. """ - msg = expected_warning.replace(key, ':1:%s' % key) + msg = expected_warning.replace(key, cache.make_key(key)) with self.assertRaisesMessage(InvalidCacheKey, msg): cache.set(key, 'value') From 8ec607be8482da4dd95fb585ea94ef95ce6e023e Mon Sep 17 00:00:00 2001 From: David Smith <39445562+smithdc1@users.noreply.github.com> Date: Wed, 3 Jun 2020 15:28:10 +0100 Subject: [PATCH 06/88] [3.0.x] Updated How to install Django on Windows guide. Backport of 25352dc019d175dfa6e62a50de76cb3614955077 from master --- docs/howto/windows.txt | 24 +++++++++++------------- docs/intro/contributing.txt | 3 +-- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/docs/howto/windows.txt b/docs/howto/windows.txt index fecdbf3fba68..9149509604a1 100644 --- a/docs/howto/windows.txt +++ b/docs/howto/windows.txt @@ -4,27 +4,28 @@ How to install Django on Windows .. highlight:: doscon -This document will guide you through installing Python 3.7 and Django on +This document will guide you through installing Python 3.8 and Django on Windows. It also provides instructions for setting up a virtual environment, which makes it easier to work on Python projects. This is meant as a beginner's guide for users working on Django projects and does not reflect how Django should be installed when developing patches for Django itself. -The steps in this guide have been tested with Windows 7, 8, and 10. In other +The steps in this guide have been tested with Windows 10. In other versions, the steps would be similar. You will need to be familiar with using the Windows command prompt. +.. _install_python_windows: + Install Python ============== Django is a Python web framework, thus requiring Python to be installed on your -machine. At the time of writing, Python 3.7 is the latest version. +machine. At the time of writing, Python 3.8 is the latest version. To install Python on your machine go to https://python.org/downloads/. The website should offer you a download button for the latest Python version. -Download the executable installer and run it. Check the boxes next to ``Install -launcher for all users (recommended)`` and ``Add Python 3.7 to PATH`` then -click ``Install Now``. +Download the executable installer and run it. Check the boxes next to "Install +launcher for all users (recommended)" then click "Install Now". After installation, open the command prompt and check that the Python version matches the version you installed by executing:: @@ -38,13 +39,10 @@ matches the version you installed by executing:: About ``pip`` ============= -`pip`_ is a package manage for Python. It makes installing and uninstalling -Python packages (such as Django!) very easy. For the rest of the installation, -we'll use ``pip`` to install Python packages from the command line. - -To install pip on your machine, go to -https://pip.pypa.io/en/latest/installing/, and follow the ``Installing with -get-pip.py`` instructions. +`pip`_ is a package manager for Python and is included by default with the +Python installer. It helps to install and uninstall Python packages +(such as Django!). For the rest of the installation, we'll use ``pip`` to +install Python packages from the command line. .. _pip: https://pypi.org/project/pip/ diff --git a/docs/intro/contributing.txt b/docs/intro/contributing.txt index 72a1d2a414a3..1ed21f9ba467 100644 --- a/docs/intro/contributing.txt +++ b/docs/intro/contributing.txt @@ -77,8 +77,7 @@ probably got the answers. .. admonition:: For Windows users - When installing Python on Windows, make sure you check the option "Add - python.exe to Path", so that it is always available on the command line. + See :ref:`install_python_windows` on Windows docs for additional guidance. Code of Conduct =============== From b80c5baf2936e7b5edfc8055e144e8b2566b047d Mon Sep 17 00:00:00 2001 From: Hasan Ramezani Date: Thu, 4 Jun 2020 07:36:12 +0200 Subject: [PATCH 07/88] [3.0.x] Fixed #31650 -- Added note uniqueness of constraints names in docs. Backport of 55556e51fb152c864ddc782d9b56e8b785ccf8bc from master --- docs/ref/models/constraints.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/ref/models/constraints.txt b/docs/ref/models/constraints.txt index f6d1842c14c5..1cb23e808039 100644 --- a/docs/ref/models/constraints.txt +++ b/docs/ref/models/constraints.txt @@ -65,7 +65,8 @@ ensures the age field is never less than 18. .. attribute:: CheckConstraint.name -The name of the constraint. +The name of the constraint. You must always specify a unique name for the +constraint. .. versionchanged:: 3.0 @@ -95,7 +96,8 @@ date. .. attribute:: UniqueConstraint.name -The name of the constraint. +The name of the constraint. You must always specify a unique name for the +constraint. .. versionchanged:: 3.0 From e8723af44be82968f573a00277f40016b049f08e Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Fri, 5 Jun 2020 07:21:52 +0200 Subject: [PATCH 08/88] [3.0.x] Fixed #31654 -- Fixed cache key validation messages. Backport of 926148ef019abcac3a9988c78734d9336d69f24e from master --- django/core/cache/backends/base.py | 2 +- docs/releases/2.2.14.txt | 13 +++++++++++++ docs/releases/3.0.8.txt | 3 ++- docs/releases/index.txt | 1 + tests/cache/tests.py | 6 ++++-- 5 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 docs/releases/2.2.14.txt diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py index 86a7aca5757a..875689bfaaad 100644 --- a/django/core/cache/backends/base.py +++ b/django/core/cache/backends/base.py @@ -286,6 +286,6 @@ def memcache_key_warnings(key): if ord(char) < 33 or ord(char) == 127: yield ( 'Cache key contains characters that will cause errors if ' - 'used with memcached: %r' % key, CacheKeyWarning + 'used with memcached: %r' % key ) break diff --git a/docs/releases/2.2.14.txt b/docs/releases/2.2.14.txt new file mode 100644 index 000000000000..0c7232fe08b7 --- /dev/null +++ b/docs/releases/2.2.14.txt @@ -0,0 +1,13 @@ +=========================== +Django 2.2.14 release notes +=========================== + +*Expected July 1, 2020* + +Django 2.2.14 fixes a bug in 2.2.13. + +Bugfixes +======== + +* Fixed messages of ``InvalidCacheKey`` exceptions and ``CacheKeyWarning`` + warnings raised by cache key validation (:ticket:`31654`). diff --git a/docs/releases/3.0.8.txt b/docs/releases/3.0.8.txt index 99247a43557c..e355f0a0ffae 100644 --- a/docs/releases/3.0.8.txt +++ b/docs/releases/3.0.8.txt @@ -9,4 +9,5 @@ Django 3.0.8 fixes several bugs in 3.0.7. Bugfixes ======== -* ... +* Fixed messages of ``InvalidCacheKey`` exceptions and ``CacheKeyWarning`` + warnings raised by cache key validation (:ticket:`31654`). diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 3e2074ff820d..af020844e8fe 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -40,6 +40,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 2.2.14 2.2.13 2.2.12 2.2.11 diff --git a/tests/cache/tests.py b/tests/cache/tests.py index 4a705642dc5f..d8ea67687577 100644 --- a/tests/cache/tests.py +++ b/tests/cache/tests.py @@ -624,8 +624,9 @@ def func(key, *args): cache.key_func = func try: - with self.assertWarnsMessage(CacheKeyWarning, expected_warning): + with self.assertWarns(CacheKeyWarning) as cm: cache.set(key, 'value') + self.assertEqual(str(cm.warning), expected_warning) finally: cache.key_func = old_func @@ -1262,8 +1263,9 @@ def _perform_invalid_key_test(self, key, expected_warning): invalid key. """ msg = expected_warning.replace(key, cache.make_key(key)) - with self.assertRaisesMessage(InvalidCacheKey, msg): + with self.assertRaises(InvalidCacheKey) as cm: cache.set(key, 'value') + self.assertEqual(str(cm.exception), msg) def test_default_never_expiring_timeout(self): # Regression test for #22845 From be7a295141337189b9eceea506489bdfe07f199e Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 8 Jun 2020 07:21:54 +0200 Subject: [PATCH 09/88] [3.0.x] Fixed #31660 -- Fixed queryset crash when grouping by m2o relation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regression in 3a941230c85b2702a5e1cd97e17251ce21057efa. Thanks Tomasz Szymański for the report. Backport of 78ad4b4b0201003792bfdbf1a7781cbc9ee03539 from master --- django/db/models/expressions.py | 4 +++- docs/releases/3.0.8.txt | 3 +++ tests/annotations/tests.py | 27 ++++++++++++++++++++++++--- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py index 7b68e1108ead..29649313e5a6 100644 --- a/django/db/models/expressions.py +++ b/django/db/models/expressions.py @@ -380,7 +380,9 @@ def select_format(self, compiler, sql, params): Custom format for select clauses. For example, EXISTS expressions need to be wrapped in CASE WHEN on Oracle. """ - return self.output_field.select_format(compiler, sql, params) + if hasattr(self.output_field, 'select_format'): + return self.output_field.select_format(compiler, sql, params) + return sql, params @cached_property def identity(self): diff --git a/docs/releases/3.0.8.txt b/docs/releases/3.0.8.txt index e355f0a0ffae..d21eac37c8e8 100644 --- a/docs/releases/3.0.8.txt +++ b/docs/releases/3.0.8.txt @@ -11,3 +11,6 @@ Bugfixes * Fixed messages of ``InvalidCacheKey`` exceptions and ``CacheKeyWarning`` warnings raised by cache key validation (:ticket:`31654`). + +* Fixed a regression in Django 3.0.7 that caused a queryset crash when grouping + by a many-to-one relationship (:ticket:`31660`). diff --git a/tests/annotations/tests.py b/tests/annotations/tests.py index 0064b14ed07e..0c6381c77367 100644 --- a/tests/annotations/tests.py +++ b/tests/annotations/tests.py @@ -1,11 +1,13 @@ import datetime from decimal import Decimal +from unittest import skipIf from django.core.exceptions import FieldDoesNotExist, FieldError +from django.db import connection from django.db.models import ( - BooleanField, CharField, Count, DateTimeField, Exists, ExpressionWrapper, - F, Func, IntegerField, Max, NullBooleanField, OuterRef, Q, Subquery, Sum, - Value, + BooleanField, Case, CharField, Count, DateTimeField, Exists, + ExpressionWrapper, F, Func, IntegerField, Max, NullBooleanField, OuterRef, + Q, Subquery, Sum, Value, When, ) from django.db.models.expressions import RawSQL from django.db.models.functions import Length, Lower @@ -632,3 +634,22 @@ def test_annotation_exists_aggregate_values_chaining(self): datetime.date(2008, 6, 23), datetime.date(2008, 11, 3), ]) + + @skipIf( + connection.vendor == 'mysql' and 'ONLY_FULL_GROUP_BY' in connection.sql_mode, + 'GROUP BY optimization does not work properly when ONLY_FULL_GROUP_BY ' + 'mode is enabled on MySQL, see #31331.', + ) + def test_annotation_aggregate_with_m2o(self): + qs = Author.objects.filter(age__lt=30).annotate( + max_pages=Case( + When(book_contact_set__isnull=True, then=Value(0)), + default=Max(F('book__pages')), + output_field=IntegerField(), + ), + ).values('name', 'max_pages') + self.assertCountEqual(qs, [ + {'name': 'James Bennett', 'max_pages': 300}, + {'name': 'Paul Bissex', 'max_pages': 0}, + {'name': 'Wesley J. Chun', 'max_pages': 0}, + ]) From 2b2500021b502318866e605f6c031f122703fe4b Mon Sep 17 00:00:00 2001 From: Nicolas Baccelli Date: Fri, 5 Jun 2020 21:01:39 +0200 Subject: [PATCH 10/88] [3.0.x] Fixed #31664 -- Reallowed using non-expressions having filterable attribute as rhs in queryset filters. Regression in 4edad1ddf6203326e0be4bdb105beecb0fe454c4. Backport of b38d44229ff185ad156bcb443d6db0db7ae3eb98 from master --- django/db/models/sql/query.py | 5 ++++- docs/releases/3.0.8.txt | 4 ++++ tests/queries/models.py | 1 + tests/queries/tests.py | 12 +++++++++--- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index f550d5e28bbf..a4f079f988eb 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -1127,7 +1127,10 @@ def check_related_objects(self, field, value, opts): def check_filterable(self, expression): """Raise an error if expression cannot be used in a WHERE clause.""" - if not getattr(expression, 'filterable', 'True'): + if ( + hasattr(expression, 'resolve_expression') and + not getattr(expression, 'filterable', True) + ): raise NotSupportedError( expression.__class__.__name__ + ' is disallowed in the filter ' 'clause.' diff --git a/docs/releases/3.0.8.txt b/docs/releases/3.0.8.txt index d21eac37c8e8..6bb3e39de741 100644 --- a/docs/releases/3.0.8.txt +++ b/docs/releases/3.0.8.txt @@ -14,3 +14,7 @@ Bugfixes * Fixed a regression in Django 3.0.7 that caused a queryset crash when grouping by a many-to-one relationship (:ticket:`31660`). + +* Reallowed, following a regression in Django 3.0, non-expressions having a + ``filterable`` attribute to be used as the right-hand side in queryset + filters (:ticket:`31664`). diff --git a/tests/queries/models.py b/tests/queries/models.py index e9eec5718dde..6bed09c9bfdd 100644 --- a/tests/queries/models.py +++ b/tests/queries/models.py @@ -77,6 +77,7 @@ class ExtraInfo(models.Model): note = models.ForeignKey(Note, models.CASCADE, null=True) value = models.IntegerField(null=True) date = models.ForeignKey(DateTimePK, models.SET_NULL, null=True) + filterable = models.BooleanField(default=True) class Meta: ordering = ['info'] diff --git a/tests/queries/tests.py b/tests/queries/tests.py index 081fb89d5466..83a2875f2aec 100644 --- a/tests/queries/tests.py +++ b/tests/queries/tests.py @@ -53,12 +53,12 @@ def setUpTestData(cls): # Create these out of order so that sorting by 'id' will be different to sorting # by 'info'. Helps detect some problems later. - cls.e2 = ExtraInfo.objects.create(info='e2', note=cls.n2, value=41) + cls.e2 = ExtraInfo.objects.create(info='e2', note=cls.n2, value=41, filterable=False) e1 = ExtraInfo.objects.create(info='e1', note=cls.n1, value=42) cls.a1 = Author.objects.create(name='a1', num=1001, extra=e1) cls.a2 = Author.objects.create(name='a2', num=2002, extra=e1) - a3 = Author.objects.create(name='a3', num=3003, extra=cls.e2) + cls.a3 = Author.objects.create(name='a3', num=3003, extra=cls.e2) cls.a4 = Author.objects.create(name='a4', num=4004, extra=cls.e2) cls.time1 = datetime.datetime(2007, 12, 19, 22, 25, 0) @@ -74,7 +74,7 @@ def setUpTestData(cls): i4.tags.set([t4]) cls.r1 = Report.objects.create(name='r1', creator=cls.a1) - Report.objects.create(name='r2', creator=a3) + Report.objects.create(name='r2', creator=cls.a3) Report.objects.create(name='r3') # Ordering by 'rank' gives us rank2, rank1, rank3. Ordering by the Meta.ordering @@ -1185,6 +1185,12 @@ def test_excluded_intermediary_m2m_table_joined(self): [], ) + def test_field_with_filterable(self): + self.assertSequenceEqual( + Author.objects.filter(extra=self.e2), + [self.a3, self.a4], + ) + class Queries2Tests(TestCase): @classmethod From 9ec6eca136cbc906f55e5844732cde6ea55b2152 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 10 Jun 2020 06:53:32 +0200 Subject: [PATCH 11/88] [3.0.x] Refs #31682 -- Doc'd minimal sqlparse version in Django 2.2. Support for sqlparse < 0.2.2 was broken in 40b0a58f5ff949fba1072627e4ad11ef98aa7f36 because is_whitespace property was added in sqlparse 0.2.2. Backport of 4339f2aff272bceabd67e452c65bcfe0700b3f09 from master --- docs/releases/2.2.txt | 2 +- docs/releases/3.0.txt | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/releases/2.2.txt b/docs/releases/2.2.txt index 195665b158e0..b3a815fbb14a 100644 --- a/docs/releases/2.2.txt +++ b/docs/releases/2.2.txt @@ -331,7 +331,7 @@ change shouldn't have an impact on your tests unless you've customized ``sqlparse`` is required dependency ----------------------------------- -To simplify a few parts of Django's database handling, `sqlparse +To simplify a few parts of Django's database handling, `sqlparse 0.2.2+ `_ is now a required dependency. It's automatically installed along with Django. diff --git a/docs/releases/3.0.txt b/docs/releases/3.0.txt index 51ca584ecb5d..2bfd7d4719c8 100644 --- a/docs/releases/3.0.txt +++ b/docs/releases/3.0.txt @@ -571,8 +571,6 @@ Miscellaneous * ``alias=None`` is added to the signature of :meth:`.Expression.get_group_by_cols`. -* Support for ``sqlparse`` < 0.2.2 is removed. - * ``RegexPattern``, used by :func:`~django.urls.re_path`, no longer returns keyword arguments with ``None`` values to be passed to the view for the optional named groups that are missing. From e2cdbc585a89e130d103307420acf34f232ba699 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Thu, 11 Jun 2020 10:36:24 +0200 Subject: [PATCH 12/88] [3.0.x] Refs #31660 -- Fixed annotations.tests crash on MySQL. Follow up to be7a295141337189b9eceea506489bdfe07f199e. --- tests/annotations/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/annotations/tests.py b/tests/annotations/tests.py index 0c6381c77367..bbd35fbd4fae 100644 --- a/tests/annotations/tests.py +++ b/tests/annotations/tests.py @@ -636,7 +636,7 @@ def test_annotation_exists_aggregate_values_chaining(self): ]) @skipIf( - connection.vendor == 'mysql' and 'ONLY_FULL_GROUP_BY' in connection.sql_mode, + connection.vendor == 'mysql', 'GROUP BY optimization does not work properly when ONLY_FULL_GROUP_BY ' 'mode is enabled on MySQL, see #31331.', ) From 33767d5ab6d6642b324b180a308e62312bd6c685 Mon Sep 17 00:00:00 2001 From: sebashwa Date: Fri, 12 Jun 2020 15:29:13 +0200 Subject: [PATCH 13/88] [3.0.x] Fixed #31690 -- Added note about fuzzy entries in translation docs. Backport of 3d664a158de18acf72fc8e0671f0f390cbca4b2e from master --- docs/ref/django-admin.txt | 4 +++- docs/topics/i18n/translation.txt | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index b66ca5f5510f..2094c60ac9c6 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -156,7 +156,7 @@ are excluded. .. django-admin-option:: --use-fuzzy, -f -Includes fuzzy translations into compiled files. +Includes `fuzzy translations`_ into compiled files. Example usage:: @@ -169,6 +169,8 @@ Example usage:: django-admin compilemessages -x pt_BR django-admin compilemessages -x pt_BR -x fr +.. _fuzzy translations: https://www.gnu.org/software/gettext/manual/html_node/Fuzzy-Entries.html + .. django-admin-option:: --ignore PATTERN, -i PATTERN .. versionadded:: 3.0 diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index f29da625de34..53f2f9920500 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1606,6 +1606,13 @@ otherwise, they'll be tacked together without whitespace! files are created). This means that everybody will be using the same encoding, which is important when Django processes the PO files. +.. admonition:: Fuzzy entries + + :djadmin:`makemessages` sometimes generates translation entries marked as + fuzzy, e.g. when translations are inferred from previously translated + strings. By default, fuzzy entries are **not** processed by + :djadmin:`compilemessages`. + To reexamine all source code and templates for new translation strings and update all message files for **all** languages, run this:: From b61af177eea2530235f1f1e702001f9faedb7f55 Mon Sep 17 00:00:00 2001 From: Hasan Ramezani Date: Mon, 15 Jun 2020 09:44:08 +0200 Subject: [PATCH 14/88] [3.0.x] Fixed #31696 -- Updated OWASP links in docs. Backport of a16080810bee8b3baf9ae7ac7b8433cb7b293e00 from master --- docs/ref/class-based-views/mixins-single-object.txt | 2 +- docs/ref/request-response.txt | 2 +- docs/ref/settings.txt | 2 +- docs/releases/1.3.txt | 2 +- docs/topics/security.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/ref/class-based-views/mixins-single-object.txt b/docs/ref/class-based-views/mixins-single-object.txt index 4fb33f6ce8a7..1378e10823df 100644 --- a/docs/ref/class-based-views/mixins-single-object.txt +++ b/docs/ref/class-based-views/mixins-single-object.txt @@ -66,7 +66,7 @@ Single object mixins non-sequential arguments. Using a unique slug may serve the same purpose, but this scheme allows you to have non-unique slugs. - .. _insecure direct object reference: https://www.owasp.org/index.php/Top_10_2013-A4-Insecure_Direct_Object_References + .. _insecure direct object reference: https://wiki.owasp.org/index.php/Top_10_2013-A4-Insecure_Direct_Object_References .. method:: get_object(queryset=None) diff --git a/docs/ref/request-response.txt b/docs/ref/request-response.txt index 7b20f324a0c4..e03bc56f8979 100644 --- a/docs/ref/request-response.txt +++ b/docs/ref/request-response.txt @@ -836,7 +836,7 @@ Methods isn't supported by all browsers, so it's not a replacement for Django's CSRF protection, but rather a defense in depth measure. - .. _HttpOnly: https://www.owasp.org/index.php/HttpOnly + .. _HttpOnly: https://owasp.org/www-community/HttpOnly .. _SameSite: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite .. warning:: diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 42c0600b1c8e..6396a9b3eee4 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -3081,7 +3081,7 @@ vulnerability into full hijacking of a user's session. There aren't many good reasons for turning this off. Your code shouldn't read session cookies from JavaScript. -.. _HttpOnly: https://www.owasp.org/index.php/HttpOnly +.. _HttpOnly: https://owasp.org/www-community/HttpOnly .. setting:: SESSION_COOKIE_NAME diff --git a/docs/releases/1.3.txt b/docs/releases/1.3.txt index 7e02bcd0a52c..37b3cc7311ae 100644 --- a/docs/releases/1.3.txt +++ b/docs/releases/1.3.txt @@ -315,7 +315,7 @@ requests. These include: * Support for combining :class:`F expressions ` with ``timedelta`` values when retrieving or updating database values. -.. _HttpOnly: https://www.owasp.org/index.php/HttpOnly +.. _HttpOnly: https://owasp.org/www-community/HttpOnly .. _backwards-incompatible-changes-1.3: diff --git a/docs/topics/security.txt b/docs/topics/security.txt index ba73f2089913..426c33d03516 100644 --- a/docs/topics/security.txt +++ b/docs/topics/security.txt @@ -294,5 +294,5 @@ security protection of the Web server, operating system and other components. pages also include security principles that apply to any system. .. _LimitRequestBody: https://httpd.apache.org/docs/2.4/mod/core.html#limitrequestbody -.. _Top 10 list: https://www.owasp.org/index.php/Top_10-2017_Top_10 +.. _Top 10 list: https://owasp.org/www-project-top-ten/OWASP_Top_Ten_2017/ .. _web security: https://infosec.mozilla.org/guidelines/web_security.html From 0afe8ba2d6712f3766f1ef11bbd14734cb3b2d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kr=C3=B6nke?= Date: Fri, 12 Jun 2020 14:54:23 +0200 Subject: [PATCH 15/88] [3.0.x] Fixed #31689 -- Doc'd caveat about using bulk_create()'s ignore_conflicts on MariDB and MySQL. Backport of 69e0d9c553bb55dde8d7d1d479a78bfa7093f406 from master --- docs/ref/models/querysets.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index a64d986dbb40..4d41d0c390f1 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -2100,6 +2100,17 @@ that fail constraints such as duplicate unique values. Enabling this parameter disables setting the primary key on each model instance (if the database normally supports it). +.. warning:: + + On MySQL and MariaDB, setting the ``ignore_conflicts`` parameter to + ``True`` turns certain types of errors, other than duplicate key, into + warnings. Even with Strict Mode. For example: invalid values or + non-nullable violations. See the `MySQL documentation`_ and + `MariaDB documentation`_ for more details. + +.. _MySQL documentation: https://dev.mysql.com/doc/refman/en/sql-mode.html#ignore-strict-comparison +.. _MariaDB documentation: https://mariadb.com/kb/en/ignore/ + .. versionchanged:: 2.2 The ``ignore_conflicts`` parameter was added. From f22f660a33f507d3a3b3aa2bff7734df1dfea6f8 Mon Sep 17 00:00:00 2001 From: David Smith Date: Wed, 10 Jun 2020 20:56:18 +0100 Subject: [PATCH 16/88] [3.0.x] Adjusted model definition in GeoDjango tutorial. The example uses the world border data set, where one of the entries has a null field value. Backport of 6bc9283751d51cab474d1bf6883a3b40cce32d4b from master --- docs/ref/contrib/gis/tutorial.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ref/contrib/gis/tutorial.txt b/docs/ref/contrib/gis/tutorial.txt index 8cf95622f49a..39d2b6787991 100644 --- a/docs/ref/contrib/gis/tutorial.txt +++ b/docs/ref/contrib/gis/tutorial.txt @@ -203,7 +203,7 @@ model to represent this data:: name = models.CharField(max_length=50) area = models.IntegerField() pop2005 = models.IntegerField('Population 2005') - fips = models.CharField('FIPS Code', max_length=2) + fips = models.CharField('FIPS Code', max_length=2, null=True) iso2 = models.CharField('2 Digit ISO', max_length=2) iso3 = models.CharField('3 Digit ISO', max_length=3) un = models.IntegerField('United Nations Code') From 453a5bf3024ed385f95f2f9a5378d8fc03baffc2 Mon Sep 17 00:00:00 2001 From: Simon Charette Date: Tue, 23 Jun 2020 23:43:22 -0400 Subject: [PATCH 17/88] [3.0.x] Fixed #31735 -- Fixed migrations crash on namespaced inline FK addition on PostgreSQL. The namespace of the constraint must be included when making the constraint immediate. Regression in 22ce5d0031bd795ade081394043833e82046016c. Thanks Rodrigo Estevao for the report. Backport of 2e8941b6f90e65ffad3f07083b8de59e8ed29767 from master --- django/db/backends/base/schema.py | 2 ++ django/db/backends/postgresql/schema.py | 2 +- docs/releases/3.0.8.txt | 4 ++++ tests/schema/tests.py | 29 +++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/django/db/backends/base/schema.py b/django/db/backends/base/schema.py index 98afbcc05a8b..5d2ee4eb04eb 100644 --- a/django/db/backends/base/schema.py +++ b/django/db/backends/base/schema.py @@ -461,8 +461,10 @@ def add_field(self, model, field): if self.sql_create_column_inline_fk: to_table = field.remote_field.model._meta.db_table to_column = field.remote_field.model._meta.get_field(field.remote_field.field_name).column + namespace, _ = split_identifier(model._meta.db_table) definition += " " + self.sql_create_column_inline_fk % { 'name': self._fk_constraint_name(model, field, constraint_suffix), + 'namespace': '%s.' % self.quote_name(namespace) if namespace else '', 'column': self.quote_name(field.column), 'to_table': self.quote_name(to_table), 'to_column': self.quote_name(to_column), diff --git a/django/db/backends/postgresql/schema.py b/django/db/backends/postgresql/schema.py index 62e0bf95fddf..233eb10eb39f 100644 --- a/django/db/backends/postgresql/schema.py +++ b/django/db/backends/postgresql/schema.py @@ -23,7 +23,7 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): # transaction. sql_create_column_inline_fk = ( 'CONSTRAINT %(name)s REFERENCES %(to_table)s(%(to_column)s)%(deferrable)s' - '; SET CONSTRAINTS %(name)s IMMEDIATE' + '; SET CONSTRAINTS %(namespace)s%(name)s IMMEDIATE' ) # Setting the constraint to IMMEDIATE runs any deferred checks to allow # dropping it in the same transaction. diff --git a/docs/releases/3.0.8.txt b/docs/releases/3.0.8.txt index 6bb3e39de741..f23208f9ae47 100644 --- a/docs/releases/3.0.8.txt +++ b/docs/releases/3.0.8.txt @@ -18,3 +18,7 @@ Bugfixes * Reallowed, following a regression in Django 3.0, non-expressions having a ``filterable`` attribute to be used as the right-hand side in queryset filters (:ticket:`31664`). + +* Fixed a regression in Django 3.0.2 that caused a migration crash on + PostgreSQL when adding a foreign key to a model with a namespaced + ``db_table`` (:ticket:`31735`). diff --git a/tests/schema/tests.py b/tests/schema/tests.py index 074f2521975c..ad66f556501d 100644 --- a/tests/schema/tests.py +++ b/tests/schema/tests.py @@ -2999,6 +2999,35 @@ class Meta: student = Student.objects.create(name='Some man') doc.students.add(student) + @isolate_apps('schema') + @unittest.skipUnless(connection.vendor == 'postgresql', 'PostgreSQL specific db_table syntax.') + def test_namespaced_db_table_foreign_key_reference(self): + with connection.cursor() as cursor: + cursor.execute('CREATE SCHEMA django_schema_tests') + + def delete_schema(): + with connection.cursor() as cursor: + cursor.execute('DROP SCHEMA django_schema_tests CASCADE') + + self.addCleanup(delete_schema) + + class Author(Model): + class Meta: + app_label = 'schema' + + class Book(Model): + class Meta: + app_label = 'schema' + db_table = '"django_schema_tests"."schema_book"' + + author = ForeignKey(Author, CASCADE) + author.set_attributes_from_name('author') + + with connection.schema_editor() as editor: + editor.create_model(Author) + editor.create_model(Book) + editor.add_field(Book, author) + def test_rename_table_renames_deferred_sql_references(self): atomic_rename = connection.features.supports_atomic_references_rename with connection.schema_editor(atomic=atomic_rename) as editor: From af2e95b0fae0437c1cc6c08adfe4ce9c68991a50 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 24 Jun 2020 11:41:10 +0200 Subject: [PATCH 18/88] [3.0.x] Refs #31493 -- Replaced var with const/let in documentation JS. Backport of 2afa61e7d99b2ff2656dc64b6e28db88baf786a4 from master Co-authored-by: Carlton Gibson --- .../contributing/writing-code/javascript.txt | 6 ++-- docs/ref/csrf.txt | 14 ++++---- docs/ref/templates/builtins.txt | 2 +- docs/topics/i18n/translation.txt | 35 ++++++++++++------- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/docs/internals/contributing/writing-code/javascript.txt b/docs/internals/contributing/writing-code/javascript.txt index 09017af63a33..07bea197a26a 100644 --- a/docs/internals/contributing/writing-code/javascript.txt +++ b/docs/internals/contributing/writing-code/javascript.txt @@ -83,13 +83,13 @@ Django's JavaScript tests use `QUnit`_. Here is an example test module: QUnit.module('magicTricks', { beforeEach: function() { - var $ = django.jQuery; + const $ = django.jQuery; $('#qunit-fixture').append(''); } }); QUnit.test('removeOnClick removes button on click', function(assert) { - var $ = django.jQuery; + const $ = django.jQuery; removeOnClick('.button'); assert.equal($('.button').length, 1); $('.button').click(); @@ -97,7 +97,7 @@ Django's JavaScript tests use `QUnit`_. Here is an example test module: }); QUnit.test('copyOnClick adds button on click', function(assert) { - var $ = django.jQuery; + const $ = django.jQuery; copyOnClick('.button'); assert.equal($('.button').length, 1); $('.button').click(); diff --git a/docs/ref/csrf.txt b/docs/ref/csrf.txt index 20a8ddb433d9..0e4423248d23 100644 --- a/docs/ref/csrf.txt +++ b/docs/ref/csrf.txt @@ -85,11 +85,11 @@ You can acquire the token like this: .. code-block:: javascript function getCookie(name) { - var cookieValue = null; + let cookieValue = null; if (document.cookie && document.cookie !== '') { - var cookies = document.cookie.split(';'); - for (var i = 0; i < cookies.length; i++) { - var cookie = cookies[i].trim(); + const cookies = document.cookie.split(';'); + for (let i = 0; i < cookies.length; i++) { + const cookie = cookies[i].trim(); // Does this cookie string begin with the name we want? if (cookie.substring(0, name.length + 1) === (name + '=')) { cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); @@ -99,14 +99,14 @@ You can acquire the token like this: } return cookieValue; } - var csrftoken = getCookie('csrftoken'); + const csrftoken = getCookie('csrftoken'); The above code could be simplified by using the `JavaScript Cookie library `_ to replace ``getCookie``: .. code-block:: javascript - var csrftoken = Cookies.get('csrftoken'); + const csrftoken = Cookies.get('csrftoken'); .. note:: @@ -138,7 +138,7 @@ and read the token from the DOM with JavaScript: {% csrf_token %} Setting the token on the AJAX request diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index 387ad09a83df..ae62727266b5 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -1816,7 +1816,7 @@ The resulting data can be accessed in JavaScript like this: .. code-block:: javascript - var value = JSON.parse(document.getElementById('hello-data').textContent); + const value = JSON.parse(document.getElementById('hello-data').textContent); XSS attacks are mitigated by escaping the characters "<", ">" and "&". For example if ``value`` is ``{'hello': 'world&'}``, the output is: diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 53f2f9920500..441085b4974f 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -1067,9 +1067,12 @@ interface within your Python code:: The ``ngettext`` function provides an interface to pluralize words and phrases:: - var object_count = 1 // or 0, or 2, or 3, ... - s = ngettext('literal for the singular case', - 'literal for the plural case', object_count); + const objectCount = 1 // or 0, or 2, or 3, ... + const string = ngettext( + 'literal for the singular case', + 'literal for the plural case', + objectCount + ); ``interpolate`` ~~~~~~~~~~~~~~~ @@ -1083,23 +1086,29 @@ function supports both positional and named interpolation: corresponding ``fmt`` placeholders in the same order they appear. For example:: - fmts = ngettext('There is %s object. Remaining: %s', - 'There are %s objects. Remaining: %s', 11); - s = interpolate(fmts, [11, 20]); - // s is 'There are 11 objects. Remaining: 20' + const formats = ngettext( + 'There is %s object. Remaining: %s', + 'There are %s objects. Remaining: %s', + 11 + ); + const string = interpolate(formats, [11, 20]); + // string is 'There are 11 objects. Remaining: 20' * Named interpolation: This mode is selected by passing the optional boolean ``named`` parameter as ``true``. ``obj`` contains a JavaScript object or associative array. For example:: - d = { - count: 10, - total: 50 + const data = { + count: 10, + total: 50 }; - fmts = ngettext('Total: %(total)s, there is %(count)s object', - 'there are %(count)s of a total of %(total)s objects', d.count); - s = interpolate(fmts, d, true); + const formats = ngettext( + 'Total: %(total)s, there is %(count)s object', + 'there are %(count)s of a total of %(total)s objects', + data.count + ); + const string = interpolate(formats, data, true); You shouldn't go over the top with string interpolation, though: this is still JavaScript, so the code has to make repeated regular-expression substitutions. From 9b9083981add14ddbb1328a62c5000ecba364027 Mon Sep 17 00:00:00 2001 From: Steven Pousty Date: Tue, 16 Jun 2020 17:35:13 -0700 Subject: [PATCH 19/88] [3.0.x] Fixed #31743 -- Doc't that managed=False prevents Django from managing tables modifications. Backport of d2c135da4c75079e45661ec609bd72f27dddf2a9 from master --- docs/ref/models/options.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/ref/models/options.txt b/docs/ref/models/options.txt index 275f077bc842..b0ea93f9c941 100644 --- a/docs/ref/models/options.txt +++ b/docs/ref/models/options.txt @@ -157,11 +157,12 @@ Django quotes column and table names behind the scenes. part of a :djadmin:`flush` management command. That is, Django *manages* the database tables' lifecycles. - If ``False``, no database table creation or deletion operations will be - performed for this model. This is useful if the model represents an existing - table or a database view that has been created by some other means. This is - the *only* difference when ``managed=False``. All other aspects of - model handling are exactly the same as normal. This includes + If ``False``, no database table creation, modification, or deletion + operations will be performed for this model. This is useful if the model + represents an existing table or a database view that has been created by + some other means. This is the *only* difference when ``managed=False``. All + other aspects of model handling are exactly the same as normal. This + includes #. Adding an automatic primary key field to the model if you don't declare it. To avoid confusion for later code readers, it's From d1ff7c50e3a9815804818ef8f76ad3c16d3f1da4 Mon Sep 17 00:00:00 2001 From: Ad Timmering <8476375+awtimmering@users.noreply.github.com> Date: Mon, 29 Jun 2020 14:51:43 +0900 Subject: [PATCH 20/88] [3.0.x] Fixed #30807 -- Fixed TestArchive.test_extract_file_permissions() when umask is 0o000. Fixed test that checks permissions on files extracted from archives with no permissions set, to not assume a default umask of 0o002. Test regression in c95d063e776e849cf1a0bf616c654165cb89c706. Backport of ec5aa2161d8015a3fe57dcbbfe14200cd18f0a16 from master --- tests/utils_tests/test_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils_tests/test_archive.py b/tests/utils_tests/test_archive.py index dfbef8ab184a..dc7c4b4ebd9a 100644 --- a/tests/utils_tests/test_archive.py +++ b/tests/utils_tests/test_archive.py @@ -44,4 +44,4 @@ def test_extract_file_permissions(self): self.assertEqual(os.stat(filepath).st_mode & mask, 0o775) # A file is readable even if permission data is missing. filepath = os.path.join(tmpdir, 'no_permissions') - self.assertEqual(os.stat(filepath).st_mode & mask, 0o664 & ~umask) + self.assertEqual(os.stat(filepath).st_mode & mask, 0o666 & ~umask) From 21e8f9f7c96c97e42a301e53acbfd6cb6f1a7227 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 30 Jun 2020 09:50:15 +0200 Subject: [PATCH 21/88] [3.0.x] Fixed #31751 -- Fixed database introspection with cx_Oracle 8. Backport of 615e32162ff646db3456b90fb4eaaecc33dd3e4e from master --- .../gis/db/backends/oracle/introspection.py | 9 +++- django/db/backends/oracle/base.py | 4 ++ django/db/backends/oracle/introspection.py | 51 +++++++++++++------ docs/releases/3.0.8.txt | 2 + 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/django/contrib/gis/db/backends/oracle/introspection.py b/django/contrib/gis/db/backends/oracle/introspection.py index 5e71c84fdd07..80899b33d7d9 100644 --- a/django/contrib/gis/db/backends/oracle/introspection.py +++ b/django/contrib/gis/db/backends/oracle/introspection.py @@ -1,14 +1,19 @@ import cx_Oracle from django.db.backends.oracle.introspection import DatabaseIntrospection +from django.utils.functional import cached_property class OracleIntrospection(DatabaseIntrospection): # Associating any OBJECTVAR instances with GeometryField. Of course, # this won't work right on Oracle objects that aren't MDSYS.SDO_GEOMETRY, # but it is the only object type supported within Django anyways. - data_types_reverse = DatabaseIntrospection.data_types_reverse.copy() - data_types_reverse[cx_Oracle.OBJECT] = 'GeometryField' + @cached_property + def data_types_reverse(self): + return { + **super().data_types_reverse, + cx_Oracle.OBJECT: 'GeometryField', + } def get_geometry_type(self, table_name, description): with self.connection.cursor() as cursor: diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index 86650a894ea5..ec993bbcfd49 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -310,6 +310,10 @@ def is_usable(self): else: return True + @cached_property + def cx_oracle_version(self): + return tuple(int(x) for x in Database.version.split('.')) + @cached_property def oracle_version(self): with self.temporary_connection(): diff --git a/django/db/backends/oracle/introspection.py b/django/db/backends/oracle/introspection.py index 2322ae0b5d6a..1ad0c9e8d7ad 100644 --- a/django/db/backends/oracle/introspection.py +++ b/django/db/backends/oracle/introspection.py @@ -6,29 +6,48 @@ from django.db.backends.base.introspection import ( BaseDatabaseIntrospection, FieldInfo as BaseFieldInfo, TableInfo, ) +from django.utils.functional import cached_property FieldInfo = namedtuple('FieldInfo', BaseFieldInfo._fields + ('is_autofield',)) class DatabaseIntrospection(BaseDatabaseIntrospection): - # Maps type objects to Django Field types. - data_types_reverse = { - cx_Oracle.BLOB: 'BinaryField', - cx_Oracle.CLOB: 'TextField', - cx_Oracle.DATETIME: 'DateField', - cx_Oracle.FIXED_CHAR: 'CharField', - cx_Oracle.FIXED_NCHAR: 'CharField', - cx_Oracle.INTERVAL: 'DurationField', - cx_Oracle.NATIVE_FLOAT: 'FloatField', - cx_Oracle.NCHAR: 'CharField', - cx_Oracle.NCLOB: 'TextField', - cx_Oracle.NUMBER: 'DecimalField', - cx_Oracle.STRING: 'CharField', - cx_Oracle.TIMESTAMP: 'DateTimeField', - } - cache_bust_counter = 1 + # Maps type objects to Django Field types. + @cached_property + def data_types_reverse(self): + if self.connection.cx_oracle_version < (8,): + return { + cx_Oracle.BLOB: 'BinaryField', + cx_Oracle.CLOB: 'TextField', + cx_Oracle.DATETIME: 'DateField', + cx_Oracle.FIXED_CHAR: 'CharField', + cx_Oracle.FIXED_NCHAR: 'CharField', + cx_Oracle.INTERVAL: 'DurationField', + cx_Oracle.NATIVE_FLOAT: 'FloatField', + cx_Oracle.NCHAR: 'CharField', + cx_Oracle.NCLOB: 'TextField', + cx_Oracle.NUMBER: 'DecimalField', + cx_Oracle.STRING: 'CharField', + cx_Oracle.TIMESTAMP: 'DateTimeField', + } + else: + return { + cx_Oracle.DB_TYPE_DATE: 'DateField', + cx_Oracle.DB_TYPE_BINARY_DOUBLE: 'FloatField', + cx_Oracle.DB_TYPE_BLOB: 'BinaryField', + cx_Oracle.DB_TYPE_CHAR: 'CharField', + cx_Oracle.DB_TYPE_CLOB: 'TextField', + cx_Oracle.DB_TYPE_INTERVAL_DS: 'DurationField', + cx_Oracle.DB_TYPE_NCHAR: 'CharField', + cx_Oracle.DB_TYPE_NCLOB: 'TextField', + cx_Oracle.DB_TYPE_NVARCHAR: 'CharField', + cx_Oracle.DB_TYPE_NUMBER: 'DecimalField', + cx_Oracle.DB_TYPE_TIMESTAMP: 'DateTimeField', + cx_Oracle.DB_TYPE_VARCHAR: 'CharField', + } + def get_field_type(self, data_type, description): if data_type == cx_Oracle.NUMBER: precision, scale = description[4:6] diff --git a/docs/releases/3.0.8.txt b/docs/releases/3.0.8.txt index f23208f9ae47..d702d757646a 100644 --- a/docs/releases/3.0.8.txt +++ b/docs/releases/3.0.8.txt @@ -22,3 +22,5 @@ Bugfixes * Fixed a regression in Django 3.0.2 that caused a migration crash on PostgreSQL when adding a foreign key to a model with a namespaced ``db_table`` (:ticket:`31735`). + +* Added compatibility for ``cx_Oracle`` 8 (:ticket:`31751`). From 7d133e81e827269bf031f0cfa51bcc5e1841e1ff Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 1 Jul 2020 06:16:32 +0200 Subject: [PATCH 22/88] [3.0.x] Added release date for 2.2.14 and 3.0.8. Backport of 0f3aecf581b50215820455eb2f6a19a1b3b3ef8b from master --- docs/releases/2.2.14.txt | 2 +- docs/releases/3.0.8.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/releases/2.2.14.txt b/docs/releases/2.2.14.txt index 0c7232fe08b7..38683cf30198 100644 --- a/docs/releases/2.2.14.txt +++ b/docs/releases/2.2.14.txt @@ -2,7 +2,7 @@ Django 2.2.14 release notes =========================== -*Expected July 1, 2020* +*July 1, 2020* Django 2.2.14 fixes a bug in 2.2.13. diff --git a/docs/releases/3.0.8.txt b/docs/releases/3.0.8.txt index d702d757646a..f8fc8ab96141 100644 --- a/docs/releases/3.0.8.txt +++ b/docs/releases/3.0.8.txt @@ -2,7 +2,7 @@ Django 3.0.8 release notes ========================== -*Expected July 1, 2020* +*July 1, 2020* Django 3.0.8 fixes several bugs in 3.0.7. From 47c5666ae7c1fc54ad87de817e926e1acff5792a Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 1 Jul 2020 06:25:03 +0200 Subject: [PATCH 23/88] [3.0.x] Bumped version for 3.0.8 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index b567a794347a..46651882000e 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 8, 'alpha', 0) +VERSION = (3, 0, 8, 'final', 0) __version__ = get_version(VERSION) From bb0b9a779f2ad98f552ac06ac1865e16f8fdb003 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 1 Jul 2020 06:30:38 +0200 Subject: [PATCH 24/88] [3.0.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 46651882000e..0d40dc097726 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 8, 'final', 0) +VERSION = (3, 0, 9, 'alpha', 0) __version__ = get_version(VERSION) From 5a15e3e3788cf321ce5bdf0a874906229afd3b47 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 1 Jul 2020 07:00:43 +0200 Subject: [PATCH 25/88] [3.0.x] Added stub release notes for 3.0.9. Backport of c2a835703f706583542e9dae82749ac3b92819f8 from master --- docs/releases/3.0.9.txt | 12 ++++++++++++ docs/releases/index.txt | 1 + 2 files changed, 13 insertions(+) create mode 100644 docs/releases/3.0.9.txt diff --git a/docs/releases/3.0.9.txt b/docs/releases/3.0.9.txt new file mode 100644 index 000000000000..adc0f30b100b --- /dev/null +++ b/docs/releases/3.0.9.txt @@ -0,0 +1,12 @@ +========================== +Django 3.0.9 release notes +========================== + +*Expected August 3, 2020* + +Django 3.0.9 fixes several bugs in 3.0.8. + +Bugfixes +======== + +* ... diff --git a/docs/releases/index.txt b/docs/releases/index.txt index af020844e8fe..434789de1f88 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.0.9 3.0.8 3.0.7 3.0.6 From 9e4f55757dcdb171c515d65a4d8e9ff5dc33a8f8 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Thu, 2 Jul 2020 11:18:50 +0200 Subject: [PATCH 26/88] [3.0.x] Refs #6903 -- Adjusted ModelAdmin.preserve_filters docs. Backport of b142bd4a1b83b77c5c81a8cf5a80a63608f96ad4 from master --- docs/ref/contrib/admin/index.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ref/contrib/admin/index.txt b/docs/ref/contrib/admin/index.txt index e68466a8c67a..01184a43745f 100644 --- a/docs/ref/contrib/admin/index.txt +++ b/docs/ref/contrib/admin/index.txt @@ -1088,9 +1088,9 @@ subclass:: .. attribute:: ModelAdmin.preserve_filters - The admin now preserves filters on the list view after creating, editing - or deleting an object. You can restore the previous behavior of clearing - filters by setting this attribute to ``False``. + By default, applied filters are preserved on the list view after creating, + editing, or deleting an object. You can have filters cleared by setting + this attribute to ``False``. .. attribute:: ModelAdmin.radio_fields From 8f750bc295c8a8d3415cc35f4ae0f6b5bfe77bf2 Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 2 Jul 2020 18:06:28 +0100 Subject: [PATCH 27/88] [3.0.x] Fixed #30945 -- Doc'd plural equations changes in 2.2. release notes. Backport of 392036be29b759204cbc4033072672acacabf3f7 from master --- docs/releases/2.2.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/releases/2.2.txt b/docs/releases/2.2.txt index b3a815fbb14a..d6c9c19fc650 100644 --- a/docs/releases/2.2.txt +++ b/docs/releases/2.2.txt @@ -475,6 +475,14 @@ Miscellaneous * Providing an integer in the ``key`` argument of the :meth:`.cache.delete` or :meth:`.cache.get` now raises :exc:`ValueError`. +* Plural equations for some languages are changed, because the latest versions + from Transifex are incorporated. + + .. note:: + + The ability to handle ``.po`` files containing different plural equations + for the same language was added in Django 2.2.12. + .. _deprecated-features-2.2: Features deprecated in 2.2 From 9141841cca505b9f6b605834dc5adb3e5269fe72 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 4 Jul 2020 09:41:40 -0400 Subject: [PATCH 28/88] [3.0.x] Fixed #29308 -- Clarified how assertQuerysetEqual()'s transform works. Backport of 659a73bc0a2df9be856e23fcfc6cc66d0d1a35fd from master --- docs/topics/testing/tools.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/topics/testing/tools.txt b/docs/topics/testing/tools.txt index 57f2e6b7727f..4f63ab903155 100644 --- a/docs/topics/testing/tools.txt +++ b/docs/topics/testing/tools.txt @@ -1687,10 +1687,11 @@ your test suite. Asserts that a queryset ``qs`` returns a particular list of values ``values``. - The comparison of the contents of ``qs`` and ``values`` is performed using - the function ``transform``; by default, this means that the ``repr()`` of - each value is compared. Any other callable can be used if ``repr()`` doesn't - provide a unique or helpful comparison. + The comparison of the contents of ``qs`` and ``values`` is performed by + applying ``transform`` to ``qs``. By default, this means that the + ``repr()`` of each value in ``qs`` is compared to the ``values``. Any other + callable can be used if ``repr()`` doesn't provide a unique or helpful + comparison. By default, the comparison is also ordering dependent. If ``qs`` doesn't provide an implicit ordering, you can set the ``ordered`` parameter to From 3ca2361d70b003019e4135413bdcc800bcbf99e5 Mon Sep 17 00:00:00 2001 From: Jason Held Date: Tue, 30 Jun 2020 22:54:39 -0400 Subject: [PATCH 29/88] [3.0.x] Fixed #24816 -- Clarified docs about preventing duplicate signals. Backport of 639142e24d41c5e5a508cb1280f32fd7ff159cca from master --- docs/topics/signals.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/topics/signals.txt b/docs/topics/signals.txt index ee097f9faa46..33b6a1b3ae32 100644 --- a/docs/topics/signals.txt +++ b/docs/topics/signals.txt @@ -188,7 +188,11 @@ Preventing duplicate signals In some circumstances, the code connecting receivers to signals may run multiple times. This can cause your receiver function to be registered more -than once, and thus called multiple times for a single signal event. +than once, and thus called as many times for a signal event. For example, the +:meth:`~django.apps.AppConfig.ready` method may be executed more than once +during testing. More generally, this occurs everywhere your project imports the +module where you define the signals, because signal registration runs as many +times as it is imported. If this behavior is problematic (such as when using signals to send an email whenever a model is saved), pass a unique identifier as From b66588abe25fb54cbf1202b774cca8b27c546cbc Mon Sep 17 00:00:00 2001 From: Tim Park Date: Sun, 5 Jul 2020 21:30:00 -0700 Subject: [PATCH 30/88] [3.0.x] Fixed #31502 -- Documented Model._state.db and Model._state.adding Backport of 697e59d5cf81e6c7e4a06ca98d6e3e16cea486dc from master --- docs/ref/models/instances.txt | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index aef2e0e1ddc7..b2f5aaa4a88e 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -81,7 +81,7 @@ fields are present, then ``values`` are guaranteed to be in the order to each of the missing fields. In addition to creating the new model, the ``from_db()`` method must set the -``adding`` and ``db`` flags in the new instance's ``_state`` attribute. +``adding`` and ``db`` flags in the new instance's :attr:`~Model._state` attribute. Below is an example showing how to record the initial values of fields that are loaded from the database:: @@ -854,3 +854,21 @@ Other attributes class to identify the class of object that could not be found and to allow you to catch a particular model class with ``try/except``. The exception is a subclass of :exc:`django.core.exceptions.ObjectDoesNotExist`. + +``_state`` +---------- + +.. attribute:: Model._state + + The ``_state`` attribute refers to a ``ModelState`` object that tracks + the lifecycle of the model instance. + + The ``ModelState`` object has two attributes: ``adding``, a flag which is + ``True`` if the model has not been saved to the database yet, and ``db``, + a string referring to the database alias the instance was loaded from or + saved to. + + Newly instantiated instances have ``adding=True`` and ``db=None``, + since they are yet to be saved. Instances fetched from a ``QuerySet`` + will have ``adding=False`` and ``db`` set to the alias of the associated + database. From 502349ce77a2bbf86ec33891c37f2ee9e087d520 Mon Sep 17 00:00:00 2001 From: Tim Park <51100862+timpark0807@users.noreply.github.com> Date: Wed, 8 Jul 2020 01:40:33 -0700 Subject: [PATCH 31/88] [3.0.x] Fixed #31739 -- Documented dependency between HttpRequest stream IO methods and body. Backport of 060576b0abac460d72714e300aa709d1e7a87dd7 from master --- docs/ref/request-response.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/ref/request-response.txt b/docs/ref/request-response.txt index e03bc56f8979..619ade110d0c 100644 --- a/docs/ref/request-response.txt +++ b/docs/ref/request-response.txt @@ -43,8 +43,10 @@ All attributes should be considered read-only, unless stated otherwise. XML payload etc. For processing conventional form data, use :attr:`HttpRequest.POST`. - You can also read from an ``HttpRequest`` using a file-like interface. See - :meth:`HttpRequest.read()`. + You can also read from an ``HttpRequest`` using a file-like interface with + :meth:`HttpRequest.read` or :meth:`HttpRequest.readline`. Accessing + the ``body`` attribute *after* reading the request with either of these I/O + stream methods will produce a ``RawPostDataException``. .. attribute:: HttpRequest.path From bb47c1dde4fa91a44d1e8729add2f2f26b90c937 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Wed, 8 Jul 2020 22:29:21 +0200 Subject: [PATCH 32/88] [3.0.x] Removed unsupported third-party database backends from docs. Backport of 1d8256719eecb724476e6cb8d63cfebf6ba1a3cc from master --- docs/ref/databases.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt index 16a4c2c52680..1e761d263734 100644 --- a/docs/ref/databases.txt +++ b/docs/ref/databases.txt @@ -1037,9 +1037,7 @@ by 3rd parties that allow you to use other databases with Django: * `CockroachDB`_ * `Firebird`_ -* `IBM DB2`_ * `Microsoft SQL Server`_ -* `ODBC`_ The Django versions and ORM features supported by these unofficial backends vary considerably. Queries regarding the specific capabilities of these @@ -1048,6 +1046,4 @@ the support channels provided by each 3rd party project. .. _CockroachDB: https://pypi.org/project/django-cockroachdb/ .. _Firebird: https://pypi.org/project/django-firebird/ -.. _IBM DB2: https://pypi.org/project/ibm_db_django/ .. _Microsoft SQL Server: https://pypi.org/project/django-mssql-backend/ -.. _ODBC: https://pypi.org/project/django-pyodbc/ From 7808a04fe5d84412fce63645c3e773186ab6b67f Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Wed, 1 Jul 2020 11:46:21 -0300 Subject: [PATCH 33/88] [3.0.x] Corrected custom model fields how-to. get_prep_value() method is complementary of from_db_value(). Follow up to e9103402c0fa873aea58a6a11dba510cd308cb84. Backport of 52a0a03671437fc4c1be1eef431685a16aef8b43 from master --- docs/howto/custom-model-fields.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/howto/custom-model-fields.txt b/docs/howto/custom-model-fields.txt index 8a8da037f3ee..9064ea11a53a 100644 --- a/docs/howto/custom-model-fields.txt +++ b/docs/howto/custom-model-fields.txt @@ -541,8 +541,8 @@ Converting Python objects to query values ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Since using a database requires conversion in both ways, if you override -:meth:`~Field.to_python` you also have to override :meth:`~Field.get_prep_value` -to convert Python objects back to query values. +:meth:`~Field.from_db_value` you also have to override +:meth:`~Field.get_prep_value` to convert Python objects back to query values. For example:: From b44e2d62c0993589726914b4edf8f6063abac257 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Fri, 10 Jul 2020 09:20:18 +0200 Subject: [PATCH 34/88] [3.0.x] Refs #30676 -- Added pdb argument to DiscoverRunner docs. Backport of 2c43840dfba42ed02574a270d826fda08e4b50d1 from master --- docs/topics/testing/advanced.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/topics/testing/advanced.txt b/docs/topics/testing/advanced.txt index cdff8889ee5e..f49177fe5969 100644 --- a/docs/topics/testing/advanced.txt +++ b/docs/topics/testing/advanced.txt @@ -499,7 +499,7 @@ behavior. This class defines the ``run_tests()`` entry point, plus a selection of other methods that are used to by ``run_tests()`` to set up, execute and tear down the test suite. -.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, test_name_patterns=None, **kwargs) +.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, test_name_patterns=None, pdb=False, **kwargs) ``DiscoverRunner`` will search for tests in any file matching ``pattern``. @@ -541,6 +541,9 @@ execute and tear down the test suite. ``test_name_patterns`` can be used to specify a set of patterns for filtering test methods and classes by their names. + If ``pdb`` is ``True``, a debugger (``pdb`` or ``ipdb``) will be spawned at + each test error or failure. + Django may, from time to time, extend the capabilities of the test runner by adding new arguments. The ``**kwargs`` declaration allows for this expansion. If you subclass ``DiscoverRunner`` or write your own test @@ -551,6 +554,10 @@ execute and tear down the test suite. custom arguments by calling ``parser.add_argument()`` inside the method, so that the :djadmin:`test` command will be able to use those arguments. + .. versionadded:: 3.0 + + The ``pdb`` argument was added. + Attributes ~~~~~~~~~~ From 96b04f53c56d912b491cd06a3ec776e1195c6308 Mon Sep 17 00:00:00 2001 From: Caio Ariede Date: Mon, 18 Nov 2019 10:53:30 -0300 Subject: [PATCH 35/88] [3.0.x] Fixed #21528 -- Added note about filtering form field's queryset based on instance to admin docs. Backport of d38c34119e91a533c797098f150abe99b5ee2fd8 from master --- docs/ref/contrib/admin/index.txt | 12 ++++++++++++ docs/ref/forms/fields.txt | 2 ++ 2 files changed, 14 insertions(+) diff --git a/docs/ref/contrib/admin/index.txt b/docs/ref/contrib/admin/index.txt index 01184a43745f..0b0085b8c0c3 100644 --- a/docs/ref/contrib/admin/index.txt +++ b/docs/ref/contrib/admin/index.txt @@ -1782,6 +1782,18 @@ templates used by the :class:`ModelAdmin` views: This uses the ``HttpRequest`` instance to filter the ``Car`` foreign key field to only display the cars owned by the ``User`` instance. + For more complex filters, you can use ``ModelForm.__init__()`` method to + filter based on an ``instance`` of your model (see + :ref:`fields-which-handle-relationships`). For example:: + + class CountryAdminForm(forms.ModelForm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['capital'].queryset = self.instance.cities.all() + + class CountryAdmin(admin.ModelAdmin): + form = CountryAdminForm + .. method:: ModelAdmin.formfield_for_manytomany(db_field, request, **kwargs) Like the ``formfield_for_foreignkey`` method, the diff --git a/docs/ref/forms/fields.txt b/docs/ref/forms/fields.txt index 1d08e2d069fd..5f53becce83b 100644 --- a/docs/ref/forms/fields.txt +++ b/docs/ref/forms/fields.txt @@ -1098,6 +1098,8 @@ Slightly complex built-in ``Field`` classes If no ``input_time_formats`` argument is provided, the default input formats for :class:`TimeField` are used. +.. _fields-which-handle-relationships: + Fields which handle relationships ================================= From 419a78300f7cd27611196e1e464d50fd0385ff27 Mon Sep 17 00:00:00 2001 From: Eric Theise Date: Wed, 15 Jul 2020 00:02:51 -0700 Subject: [PATCH 36/88] [3.0.x] Fixed typo in docs/ref/contrib/postgres/forms.txt. Backport of d08e6f55e3a986a8d4b3a58431d9615c7bc81eaa from master --- docs/ref/contrib/postgres/forms.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ref/contrib/postgres/forms.txt b/docs/ref/contrib/postgres/forms.txt index bb78e5bc0571..62af57538660 100644 --- a/docs/ref/contrib/postgres/forms.txt +++ b/docs/ref/contrib/postgres/forms.txt @@ -255,4 +255,4 @@ Widgets Takes a single "compressed" value of a field, for example a :class:`~django.contrib.postgres.fields.DateRangeField`, - and returns a tuple representing and lower and upper bound. + and returns a tuple representing a lower and upper bound. From 331324ecce1330dce3dbd1713203cb9a42854ad7 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Thu, 16 Jul 2020 09:30:15 +0200 Subject: [PATCH 37/88] [3.0.x] Fixed #31790 -- Fixed setting SameSite cookies flag in HttpResponse.delete_cookie(). Cookies with the "SameSite" flag set to None and without the "secure" flag will be soon rejected by latest browser versions. This affects sessions and messages cookies. Backport of 240cbb63bf9965c63d7a3cc9032f91410f414d46 from master. --- django/contrib/messages/storage/cookie.py | 6 +++++- django/contrib/sessions/middleware.py | 1 + django/http/response.py | 4 ++-- docs/ref/request-response.txt | 6 +++++- docs/releases/2.2.15.txt | 13 +++++++++++++ docs/releases/3.0.9.txt | 3 ++- docs/releases/index.txt | 1 + tests/messages_tests/test_cookie.py | 5 +++++ tests/responses/test_cookie.py | 6 ++++++ tests/sessions_tests/tests.py | 6 ++++-- 10 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 docs/releases/2.2.15.txt diff --git a/django/contrib/messages/storage/cookie.py b/django/contrib/messages/storage/cookie.py index 9e0c93e436f3..057d573d3fa8 100644 --- a/django/contrib/messages/storage/cookie.py +++ b/django/contrib/messages/storage/cookie.py @@ -89,7 +89,11 @@ def _update_cookie(self, encoded_data, response): samesite=settings.SESSION_COOKIE_SAMESITE, ) else: - response.delete_cookie(self.cookie_name, domain=settings.SESSION_COOKIE_DOMAIN) + response.delete_cookie( + self.cookie_name, + domain=settings.SESSION_COOKIE_DOMAIN, + samesite=settings.SESSION_COOKIE_SAMESITE, + ) def _store(self, messages, response, remove_oldest=True, *args, **kwargs): """ diff --git a/django/contrib/sessions/middleware.py b/django/contrib/sessions/middleware.py index d36be4eca8bd..7a0f9e030b2f 100644 --- a/django/contrib/sessions/middleware.py +++ b/django/contrib/sessions/middleware.py @@ -38,6 +38,7 @@ def process_response(self, request, response): settings.SESSION_COOKIE_NAME, path=settings.SESSION_COOKIE_PATH, domain=settings.SESSION_COOKIE_DOMAIN, + samesite=settings.SESSION_COOKIE_SAMESITE, ) patch_vary_headers(response, ('Cookie',)) else: diff --git a/django/http/response.py b/django/http/response.py index c33feb97c43b..d98c040d9124 100644 --- a/django/http/response.py +++ b/django/http/response.py @@ -209,13 +209,13 @@ def set_signed_cookie(self, key, value, salt='', **kwargs): value = signing.get_cookie_signer(salt=key + salt).sign(value) return self.set_cookie(key, value, **kwargs) - def delete_cookie(self, key, path='/', domain=None): + def delete_cookie(self, key, path='/', domain=None, samesite=None): # Most browsers ignore the Set-Cookie header if the cookie name starts # with __Host- or __Secure- and the cookie doesn't use the secure flag. secure = key.startswith(('__Secure-', '__Host-')) self.set_cookie( key, max_age=0, path=path, domain=domain, secure=secure, - expires='Thu, 01 Jan 1970 00:00:00 GMT', + expires='Thu, 01 Jan 1970 00:00:00 GMT', samesite=samesite, ) # Common methods used by subclasses diff --git a/docs/ref/request-response.txt b/docs/ref/request-response.txt index 619ade110d0c..8eb578d50598 100644 --- a/docs/ref/request-response.txt +++ b/docs/ref/request-response.txt @@ -858,7 +858,7 @@ Methods you will need to remember to pass it to the corresponding :meth:`HttpRequest.get_signed_cookie` call. -.. method:: HttpResponse.delete_cookie(key, path='/', domain=None) +.. method:: HttpResponse.delete_cookie(key, path='/', domain=None, samesite=None) Deletes the cookie with the given key. Fails silently if the key doesn't exist. @@ -867,6 +867,10 @@ Methods values you used in ``set_cookie()`` -- otherwise the cookie may not be deleted. + .. versionchanged:: 2.2.15 + + The ``samesite`` argument was added. + .. method:: HttpResponse.close() This method is called at the end of the request directly by the WSGI diff --git a/docs/releases/2.2.15.txt b/docs/releases/2.2.15.txt new file mode 100644 index 000000000000..df2696202908 --- /dev/null +++ b/docs/releases/2.2.15.txt @@ -0,0 +1,13 @@ +=========================== +Django 2.2.15 release notes +=========================== + +*Expected August 3, 2020* + +Django 2.2.15 fixes a bug in 2.2.14. + +Bugfixes +======== + +* Allowed setting the ``SameSite`` cookie flag in + :meth:`.HttpResponse.delete_cookie` (:ticket:`31790`). diff --git a/docs/releases/3.0.9.txt b/docs/releases/3.0.9.txt index adc0f30b100b..36bcf4546f6e 100644 --- a/docs/releases/3.0.9.txt +++ b/docs/releases/3.0.9.txt @@ -9,4 +9,5 @@ Django 3.0.9 fixes several bugs in 3.0.8. Bugfixes ======== -* ... +* Allowed setting the ``SameSite`` cookie flag in + :meth:`.HttpResponse.delete_cookie` (:ticket:`31790`). diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 434789de1f88..6b9674005fe3 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -41,6 +41,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 2.2.15 2.2.14 2.2.13 2.2.12 diff --git a/tests/messages_tests/test_cookie.py b/tests/messages_tests/test_cookie.py index 211d33f04c54..7456e03a70fc 100644 --- a/tests/messages_tests/test_cookie.py +++ b/tests/messages_tests/test_cookie.py @@ -1,5 +1,6 @@ import json +from django.conf import settings from django.contrib.messages import constants from django.contrib.messages.storage.base import Message from django.contrib.messages.storage.cookie import ( @@ -85,6 +86,10 @@ def test_cookie_setings(self): self.assertEqual(response.cookies['messages'].value, '') self.assertEqual(response.cookies['messages']['domain'], '.example.com') self.assertEqual(response.cookies['messages']['expires'], 'Thu, 01 Jan 1970 00:00:00 GMT') + self.assertEqual( + response.cookies['messages']['samesite'], + settings.SESSION_COOKIE_SAMESITE, + ) def test_get_bad_cookie(self): request = self.get_request() diff --git a/tests/responses/test_cookie.py b/tests/responses/test_cookie.py index a46d910f3482..3470e9bf28be 100644 --- a/tests/responses/test_cookie.py +++ b/tests/responses/test_cookie.py @@ -102,6 +102,7 @@ def test_default(self): self.assertEqual(cookie['path'], '/') self.assertEqual(cookie['secure'], '') self.assertEqual(cookie['domain'], '') + self.assertEqual(cookie['samesite'], '') def test_delete_cookie_secure_prefix(self): """ @@ -115,3 +116,8 @@ def test_delete_cookie_secure_prefix(self): cookie_name = '__%s-c' % prefix response.delete_cookie(cookie_name) self.assertEqual(response.cookies[cookie_name]['secure'], True) + + def test_delete_cookie_samesite(self): + response = HttpResponse() + response.delete_cookie('c', samesite='lax') + self.assertEqual(response.cookies['c']['samesite'], 'lax') diff --git a/tests/sessions_tests/tests.py b/tests/sessions_tests/tests.py index 24e4e0c81b5b..3254013d501a 100644 --- a/tests/sessions_tests/tests.py +++ b/tests/sessions_tests/tests.py @@ -755,8 +755,9 @@ def test_session_delete_on_end(self): # Set-Cookie: sessionid=; expires=Thu, 01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/ self.assertEqual( 'Set-Cookie: {}=""; expires=Thu, 01 Jan 1970 00:00:00 GMT; ' - 'Max-Age=0; Path=/'.format( + 'Max-Age=0; Path=/; SameSite={}'.format( settings.SESSION_COOKIE_NAME, + settings.SESSION_COOKIE_SAMESITE, ), str(response.cookies[settings.SESSION_COOKIE_NAME]) ) @@ -787,8 +788,9 @@ def test_session_delete_on_end_with_custom_domain_and_path(self): # Path=/example/ self.assertEqual( 'Set-Cookie: {}=""; Domain=.example.local; expires=Thu, ' - '01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/example/'.format( + '01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/example/; SameSite={}'.format( settings.SESSION_COOKIE_NAME, + settings.SESSION_COOKIE_SAMESITE, ), str(response.cookies[settings.SESSION_COOKIE_NAME]) ) From 051e6f58ea0260a99e3b96e5d2ad5d39f51e4a7b Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 15 Jul 2020 10:27:46 +0100 Subject: [PATCH 38/88] [3.0.x] Refs #31502 -- Made minor edits to Model._state docs. Backport of 5ef6f626347f9ea13915f3cf5b8b045c5b42b102 from master --- docs/ref/signals.txt | 8 ++++---- docs/topics/db/multi-db.txt | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/ref/signals.txt b/docs/ref/signals.txt index dc8c53ac2d20..a4ed248769d1 100644 --- a/docs/ref/signals.txt +++ b/docs/ref/signals.txt @@ -101,10 +101,10 @@ Arguments sent with this signal: .. note:: - ``instance._state`` isn't set before sending the ``post_init`` signal, - so ``_state`` attributes always have their default values. For example, - ``_state.db`` is ``None`` and cannot be used to check an ``instance`` - database. + :attr:`instance._state ` isn't set + before sending the ``post_init`` signal, so ``_state`` attributes + always have their default values. For example, ``_state.db`` is + ``None``. .. warning:: diff --git a/docs/topics/db/multi-db.txt b/docs/topics/db/multi-db.txt index be660cfcc2de..afd6e28accc2 100644 --- a/docs/topics/db/multi-db.txt +++ b/docs/topics/db/multi-db.txt @@ -239,9 +239,10 @@ database usage. Whenever a query needs to know which database to use, it calls the master router, providing a model and a hint (if available). Django then tries each router in turn until a database suggestion can be found. If no suggestion can be found, it tries the -current ``_state.db`` of the hint instance. If a hint instance wasn't -provided, or the instance doesn't currently have database state, the -master router will allocate the ``default`` database. +current :attr:`instance._state.db ` of the hint +instance. If a hint instance wasn't provided, or :attr:`instance._state.db +` is ``None``, the master router will allocate +the ``default`` database. An example ---------- From 5bd7c18306c265dcc03baac4fe0131d5626a6b9f Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Fri, 10 Jul 2020 14:27:31 +0100 Subject: [PATCH 39/88] [3.0.x] Improved ManyToManyField.through docs. Backport of e7fa8aff432a90b6df9914d63aad239164b6b4d4 from master --- docs/ref/models/fields.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index 2fad2d64b7bd..4acf47c18894 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -1713,6 +1713,13 @@ that control how the relationship functions. :ref:`extra data with a many-to-many relationship `. + .. note:: + + If you don't want multiple associations between the same instances, add + a :class:`~django.db.models.UniqueConstraint` including the from and to + fields. Django's automatically generated many-to-many tables include + such a constraint. + .. note:: Recursive relationships using an intermediary model and defined as @@ -1746,7 +1753,9 @@ that control how the relationship functions. points (i.e. the target model instance). This class can be used to query associated records for a given model - instance like a normal model. + instance like a normal model:: + + Model.m2mfield.through.objects.all() .. attribute:: ManyToManyField.through_fields From ccc088f8ced67a9ea57a6ee1e00964f2cf6baffd Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Wed, 15 Jul 2020 07:30:15 +0200 Subject: [PATCH 40/88] [3.0.x] Fixed #31784 -- Fixed crash when sending emails on Python 3.6.11+, 3.7.8+, and 3.8.4+. Fixed sending emails crash on email addresses with display names longer then 75 chars on Python 3.6.11+, 3.7.8+, and 3.8.4+. Wrapped display names were passed to email.headerregistry.Address() what caused raising an exception because address parts cannot contain CR or LF. See https://bugs.python.org/issue39073 Co-Authored-By: Mariusz Felisiak Backport of 96a3ea39ef0790dbc413dde0a3e19f6a769356a2 from master --- django/core/mail/message.py | 16 +++++++++--- docs/releases/2.2.15.txt | 5 +++- docs/releases/3.0.9.txt | 3 +++ tests/mail/tests.py | 50 ++++++++++++++++++++++++++++++++----- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/django/core/mail/message.py b/django/core/mail/message.py index 607eb4af0b85..963542cd6239 100644 --- a/django/core/mail/message.py +++ b/django/core/mail/message.py @@ -10,7 +10,7 @@ from email.mime.message import MIMEMessage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.utils import formatdate, getaddresses, make_msgid +from email.utils import formataddr, formatdate, getaddresses, make_msgid from io import BytesIO, StringIO from pathlib import Path @@ -96,16 +96,24 @@ def sanitize_address(addr, encoding): nm, address = addr localpart, domain = address.rsplit('@', 1) - nm = Header(nm, encoding).encode() + address_parts = nm + localpart + domain + if '\n' in address_parts or '\r' in address_parts: + raise ValueError('Invalid address; address parts cannot contain newlines.') + # Avoid UTF-8 encode, if it's possible. + try: + nm.encode('ascii') + nm = Header(nm).encode() + except UnicodeEncodeError: + nm = Header(nm, encoding).encode() try: localpart.encode('ascii') except UnicodeEncodeError: localpart = Header(localpart, encoding).encode() domain = punycode(domain) - parsed_address = Address(nm, username=localpart, domain=domain) - return str(parsed_address) + parsed_address = Address(username=localpart, domain=domain) + return formataddr((nm, parsed_address.addr_spec)) class MIMEMixin: diff --git a/docs/releases/2.2.15.txt b/docs/releases/2.2.15.txt index df2696202908..d8ed58b596cb 100644 --- a/docs/releases/2.2.15.txt +++ b/docs/releases/2.2.15.txt @@ -4,10 +4,13 @@ Django 2.2.15 release notes *Expected August 3, 2020* -Django 2.2.15 fixes a bug in 2.2.14. +Django 2.2.15 fixes two bugs in 2.2.14. Bugfixes ======== * Allowed setting the ``SameSite`` cookie flag in :meth:`.HttpResponse.delete_cookie` (:ticket:`31790`). + +* Fixed crash when sending emails to addresses with display names longer than + 75 chars on Python 3.6.11+, 3.7.8+, and 3.8.4+ (:ticket:`31784`). diff --git a/docs/releases/3.0.9.txt b/docs/releases/3.0.9.txt index 36bcf4546f6e..65310384ec3c 100644 --- a/docs/releases/3.0.9.txt +++ b/docs/releases/3.0.9.txt @@ -11,3 +11,6 @@ Bugfixes * Allowed setting the ``SameSite`` cookie flag in :meth:`.HttpResponse.delete_cookie` (:ticket:`31790`). + +* Fixed crash when sending emails to addresses with display names longer than + 75 chars on Python 3.6.11+, 3.7.8+, and 3.8.4+ (:ticket:`31784`). diff --git a/tests/mail/tests.py b/tests/mail/tests.py index 6de819965af6..accdba5e4a13 100644 --- a/tests/mail/tests.py +++ b/tests/mail/tests.py @@ -722,14 +722,14 @@ def test_sanitize_address(self): ( ('A name', 'to@example.com'), 'utf-8', - '=?utf-8?q?A_name?= ', + 'A name ', ), ('localpartonly', 'ascii', 'localpartonly'), # ASCII addresses with display names. ('A name ', 'ascii', 'A name '), - ('A name ', 'utf-8', '=?utf-8?q?A_name?= '), + ('A name ', 'utf-8', 'A name '), ('"A name" ', 'ascii', 'A name '), - ('"A name" ', 'utf-8', '=?utf-8?q?A_name?= '), + ('"A name" ', 'utf-8', 'A name '), # Unicode addresses (supported per RFC-6532). ('tó@example.com', 'utf-8', '=?utf-8?b?dMOz?=@example.com'), ('to@éxample.com', 'utf-8', 'to@xn--xample-9ua.com'), @@ -748,20 +748,45 @@ def test_sanitize_address(self): ( 'To Example ', 'utf-8', - '=?utf-8?q?To_Example?= ', + 'To Example ', ), # Addresses with two @ signs. ('"to@other.com"@example.com', 'utf-8', r'"to@other.com"@example.com'), ( '"to@other.com" ', 'utf-8', - '=?utf-8?q?to=40other=2Ecom?= ', + '"to@other.com" ', ), ( ('To Example', 'to@other.com@example.com'), 'utf-8', - '=?utf-8?q?To_Example?= <"to@other.com"@example.com>', + 'To Example <"to@other.com"@example.com>', + ), + # Addresses with long unicode display names. + ( + 'Tó Example very long' * 4 + ' ', + 'utf-8', + '=?utf-8?q?T=C3=B3_Example_very_longT=C3=B3_Example_very_longT' + '=C3=B3_Example_?=\n' + ' =?utf-8?q?very_longT=C3=B3_Example_very_long?= ' + '', + ), + ( + ('Tó Example very long' * 4, 'to@example.com'), + 'utf-8', + '=?utf-8?q?T=C3=B3_Example_very_longT=C3=B3_Example_very_longT' + '=C3=B3_Example_?=\n' + ' =?utf-8?q?very_longT=C3=B3_Example_very_long?= ' + '', ), + # Address with long display name and unicode domain. + ( + ('To Example very long' * 4, 'to@exampl€.com'), + 'utf-8', + 'To Example very longTo Example very longTo Example very longT' + 'o Example very\n' + ' long ' + ) ): with self.subTest(email_address=email_address, encoding=encoding): self.assertEqual(sanitize_address(email_address, encoding), expected_result) @@ -781,6 +806,19 @@ def test_sanitize_address_invalid(self): with self.assertRaises(ValueError): sanitize_address(email_address, encoding='utf-8') + def test_sanitize_address_header_injection(self): + msg = 'Invalid address; address parts cannot contain newlines.' + tests = [ + 'Name\nInjection ', + ('Name\nInjection', 'to@xample.com'), + 'Name ', + ('Name', 'to\ninjection@example.com'), + ] + for email_address in tests: + with self.subTest(email_address=email_address): + with self.assertRaisesMessage(ValueError, msg): + sanitize_address(email_address, encoding='utf-8') + @requires_tz_support class MailTimeZoneTests(SimpleTestCase): From bcb511f9d8e0fc2cee9715e839df6ca05f42aab9 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Mon, 20 Jul 2020 11:54:55 +0100 Subject: [PATCH 41/88] [3.0.x] Improved description of USE_THOUSAND_SEPARATOR setting. Backport of 80f92177eb2a175579f4a6907ef5a358863bddca from master --- docs/ref/settings.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 6396a9b3eee4..96b2b974ab5a 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -2711,11 +2711,10 @@ See also :setting:`LANGUAGE_CODE`, :setting:`USE_I18N` and :setting:`USE_TZ`. Default: ``False`` A boolean that specifies whether to display numbers using a thousand separator. -When :setting:`USE_L10N` is set to ``True`` and if this is also set to -``True``, Django will use the values of :setting:`THOUSAND_SEPARATOR` and -:setting:`NUMBER_GROUPING` to format numbers unless the locale already has an -existing thousands separator. If there is a thousands separator in the locale -format, it will have higher precedence and will be applied instead. +When set to ``True`` and :setting:`USE_L10N` is also ``True``, Django will +format numbers using the :setting:`NUMBER_GROUPING` and +:setting:`THOUSAND_SEPARATOR` settings. These settings may also be dictated by +the locale, which takes precedence. See also :setting:`DECIMAL_SEPARATOR`, :setting:`NUMBER_GROUPING` and :setting:`THOUSAND_SEPARATOR`. From 0a3c1272f2213ccd2b36081c8d587b3f33237a5d Mon Sep 17 00:00:00 2001 From: David Chorpash Date: Mon, 20 Jul 2020 10:15:53 +0200 Subject: [PATCH 42/88] [3.0.x] Refs #31720 -- Added examples to BoolAnd() and BoolOr() documentation. Backport of a2e621b14e85836362b7fc0e6b1bf7d7ff98e42b from master --- docs/ref/contrib/postgres/aggregates.txt | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/ref/contrib/postgres/aggregates.txt b/docs/ref/contrib/postgres/aggregates.txt index c70b6ec3b316..46d56d77773e 100644 --- a/docs/ref/contrib/postgres/aggregates.txt +++ b/docs/ref/contrib/postgres/aggregates.txt @@ -77,6 +77,20 @@ General-purpose aggregation functions Returns ``True``, if all input values are true, ``None`` if all values are null or if there are no values, otherwise ``False`` . + Usage example:: + + class Comment(models.Model): + body = models.TextField() + published = models.BooleanField() + rank = models.IntegerField() + + >>> from django.db.models import BooleanField, Q + >>> from django.contrib.postgres.aggregates import BoolAnd + >>> Comment.objects.aggregate(booland=BoolAnd('published')) + {'booland': False} + >>> Comment.objects.aggregate(booland=BoolAnd(Q(rank__lt=100), output_field=BooleanField())) + {'booland': True} + ``BoolOr`` ---------- @@ -85,6 +99,20 @@ General-purpose aggregation functions Returns ``True`` if at least one input value is true, ``None`` if all values are null or if there are no values, otherwise ``False``. + Usage example:: + + class Comment(models.Model): + body = models.TextField() + published = models.BooleanField() + rank = models.IntegerField() + + >>> from django.db.models import BooleanField, Q + >>> from django.contrib.postgres.aggregates import BoolOr + >>> Comment.objects.aggregate(boolor=BoolOr('published')) + {'boolor': True} + >>> Comment.objects.aggregate(boolor=BoolOr(Q(rank__gt=2), output_field=BooleanField())) + {'boolor': False} + ``JSONBAgg`` ------------ From 1a81d34e70bdffbb563e053c3178ab885436c00c Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 22 Jul 2020 10:57:38 +0200 Subject: [PATCH 43/88] [3.0.x] Fixed #31797 -- Skipped schema tests on specific MariaDB versions. test_alter_not_unique_field_to_primary_key() test is affected by https://jira.mariadb.org/browse/MDEV-19598 on MariaDB 10.4.4 to 10.5.1. test_alter_pk_with_self_referential_field() test is affected by https://jira.mariadb.org/browse/MDEV-22775 on MariaDB 10.4 series from 10.4.13. Backport of c071c408d72330b422c124a6bdd85a68acae9566 from master --- tests/schema/tests.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/schema/tests.py b/tests/schema/tests.py index ad66f556501d..d8680501bc29 100644 --- a/tests/schema/tests.py +++ b/tests/schema/tests.py @@ -2,7 +2,7 @@ import itertools import unittest from copy import copy -from unittest import mock +from unittest import mock, skipIf from django.core.management.color import no_style from django.db import ( @@ -694,6 +694,12 @@ class Meta: editor.alter_field(Foo, old_field, new_field, strict=True) Foo.objects.create() + @skipIf( + connection.vendor == 'mysql' and + connection.mysql_is_mariadb and + (10, 4, 3) < connection.mysql_version < (10, 5, 2), + 'https://jira.mariadb.org/browse/MDEV-19598', + ) def test_alter_not_unique_field_to_primary_key(self): # Create the table. with connection.schema_editor() as editor: @@ -2879,6 +2885,12 @@ def test_alter_field_add_index_to_integerfield(self): editor.alter_field(Author, new_field, old_field, strict=True) self.assertEqual(self.get_constraints_for_column(Author, 'weight'), []) + @skipIf( + connection.vendor == 'mysql' and + connection.mysql_is_mariadb and + (10, 4, 12) < connection.mysql_version < (10, 5), + 'https://jira.mariadb.org/browse/MDEV-22775', + ) def test_alter_pk_with_self_referential_field(self): """ Changing the primary key field name of a model with a self-referential From dcb27ead84a91e8d38a183cfc2f7640141eb40f8 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Wed, 22 Jul 2020 12:49:56 +0200 Subject: [PATCH 44/88] [3.0.x] Fixed #31805 -- Fixed SchemaTests.tearDown() when table names are case-insensitive. Backport of fd53db842c35c994dbd54196dd38a908f3676b1a from master --- tests/schema/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/schema/tests.py b/tests/schema/tests.py index d8680501bc29..9a84969447f9 100644 --- a/tests/schema/tests.py +++ b/tests/schema/tests.py @@ -94,8 +94,12 @@ def delete_tables(self): with connection.schema_editor() as editor: connection.disable_constraint_checking() table_names = connection.introspection.table_names() + if connection.features.ignores_table_name_case: + table_names = [table_name.lower() for table_name in table_names] for model in itertools.chain(SchemaTests.models, self.local_models): tbl = converter(model._meta.db_table) + if connection.features.ignores_table_name_case: + tbl = tbl.lower() if tbl in table_names: editor.delete_model(model) table_names.remove(tbl) From f026d761d63ecd10b937b8009b0ad59673f772b5 Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 22 Jul 2020 11:11:58 +0200 Subject: [PATCH 45/88] [3.0.x] Refs #24763 -- Moved DoesNotExist to Model class docs. Backport of b5f0efa19c82d274082bcde8a8acae5038667614 from master --- docs/ref/models/class.txt | 14 ++++++++++++++ docs/ref/models/instances.txt | 14 -------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/ref/models/class.txt b/docs/ref/models/class.txt index 0183a82093b2..c41052788eb1 100644 --- a/docs/ref/models/class.txt +++ b/docs/ref/models/class.txt @@ -11,6 +11,20 @@ reference guides `. Attributes ========== +``DoesNotExist`` +---------------- + +.. exception:: Model.DoesNotExist + + This exception is raised by the ORM when an expected object is not found. + For example, :meth:`.QuerySet.get` will raise it when no object is found + for the given lookups. + + Django provides a ``DoesNotExist`` exception as an attribute of each model + class to identify the class of object that could not be found, allowing you + to catch exceptions for a particular model class. The exception is a + subclass of :exc:`django.core.exceptions.ObjectDoesNotExist`. + ``objects`` ----------- diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index b2f5aaa4a88e..b439098674fe 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -841,20 +841,6 @@ duplicated. That also means you cannot use those methods on unsaved objects. Other attributes ================ -``DoesNotExist`` ----------------- - -.. exception:: Model.DoesNotExist - - This exception is raised by the ORM in a couple places, for example by - :meth:`QuerySet.get() ` when an object - is not found for the given query parameters. - - Django provides a ``DoesNotExist`` exception as an attribute of each model - class to identify the class of object that could not be found and to allow - you to catch a particular model class with ``try/except``. The exception is - a subclass of :exc:`django.core.exceptions.ObjectDoesNotExist`. - ``_state`` ---------- From 76b7b3a77821841ead8bf053ff34753c128900ff Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Wed, 15 Jul 2020 10:32:59 +0100 Subject: [PATCH 46/88] [3.0.x] Doc'd Model.MultipleObjectsReturned docs and improved documentation related with models exceptions. Backport of bc4fea92b296a7eacbd5f89263ca67515feeb53f from master --- docs/ref/exceptions.txt | 20 +++++++-------- docs/ref/models/class.txt | 14 ++++++++++ docs/ref/models/fields.txt | 10 +++++--- docs/ref/models/querysets.txt | 48 ++++++++++++++++++++--------------- 4 files changed, 58 insertions(+), 34 deletions(-) diff --git a/docs/ref/exceptions.txt b/docs/ref/exceptions.txt index 83869d6b4247..8f276a993c71 100644 --- a/docs/ref/exceptions.txt +++ b/docs/ref/exceptions.txt @@ -26,12 +26,12 @@ Django core exception classes are defined in ``django.core.exceptions``. .. exception:: ObjectDoesNotExist - The base class for :exc:`~django.db.models.Model.DoesNotExist` exceptions; - a ``try/except`` for ``ObjectDoesNotExist`` will catch + The base class for :exc:`Model.DoesNotExist + ` exceptions. A ``try/except`` for + ``ObjectDoesNotExist`` will catch :exc:`~django.db.models.Model.DoesNotExist` exceptions for all models. - See :meth:`~django.db.models.query.QuerySet.get()` for further information - on :exc:`ObjectDoesNotExist` and :exc:`~django.db.models.Model.DoesNotExist`. + See :meth:`~django.db.models.query.QuerySet.get()`. ``EmptyResultSet`` ------------------ @@ -56,13 +56,13 @@ Django core exception classes are defined in ``django.core.exceptions``. .. exception:: MultipleObjectsReturned - The :exc:`MultipleObjectsReturned` exception is raised by a query if only - one object is expected, but multiple objects are returned. A base version - of this exception is provided in :mod:`django.core.exceptions`; each model - class contains a subclassed version that can be used to identify the - specific object type that has returned multiple objects. + The base class for :exc:`Model.MultipleObjectsReturned + ` exceptions. A + ``try/except`` for ``MultipleObjectsReturned`` will catch + :exc:`~django.db.models.Model.MultipleObjectsReturned` exceptions for all + models. - See :meth:`~django.db.models.query.QuerySet.get()` for further information. + See :meth:`~django.db.models.query.QuerySet.get()`. ``SuspiciousOperation`` ----------------------- diff --git a/docs/ref/models/class.txt b/docs/ref/models/class.txt index c41052788eb1..81414973a939 100644 --- a/docs/ref/models/class.txt +++ b/docs/ref/models/class.txt @@ -25,6 +25,20 @@ Attributes to catch exceptions for a particular model class. The exception is a subclass of :exc:`django.core.exceptions.ObjectDoesNotExist`. +``MultipleObjectsReturned`` +--------------------------- + +.. exception:: Model.MultipleObjectsReturned + + This exception is raised by :meth:`.QuerySet.get` when multiple objects are + found for the given lookups. + + Django provides a ``MultipleObjectsReturned`` exception as an attribute of + each model class to identify the class of object for which multiple objects + were found, allowing you to catch exceptions for a particular model class. + The exception is a subclass of + :exc:`django.core.exceptions.MultipleObjectsReturned`. + ``objects`` ----------- diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index 4acf47c18894..03b713704baf 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -1894,14 +1894,16 @@ your resulting ``User`` model will have the following attributes:: >>> hasattr(user, 'supervisor_of') True -A ``DoesNotExist`` exception is raised when accessing the reverse relationship -if an entry in the related table doesn't exist. For example, if a user doesn't -have a supervisor designated by ``MySpecialUser``:: +A ``RelatedObjectDoesNotExist`` exception is raised when accessing the reverse +relationship if an entry in the related table doesn't exist. This is a subclass +of the target model's :exc:`Model.DoesNotExist +` exception. For example, if a user +doesn't have a supervisor designated by ``MySpecialUser``:: >>> user.supervisor_of Traceback (most recent call last): ... - DoesNotExist: User matching query does not exist. + RelatedObjectDoesNotExist: User has no supervisor_of. .. _onetoone-arguments: diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index 4d41d0c390f1..6f0b74ec92f0 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -1823,34 +1823,42 @@ they query the database each time they're called. .. method:: get(**kwargs) Returns the object matching the given lookup parameters, which should be in -the format described in `Field lookups`_. +the format described in `Field lookups`_. You should use lookups that are +guaranteed unique, such as the primary key or fields in a unique constraint. +For example:: + + Entry.objects.get(id=1) + Entry.objects.get(blog=blog, entry_number=1) + +If you expect a queryset to already return one row, you can use ``get()`` +without any arguments to return the object for that row:: + + Entry.objects.filter(pk=1).get() -``get()`` raises :exc:`~django.core.exceptions.MultipleObjectsReturned` if more -than one object was found. The -:exc:`~django.core.exceptions.MultipleObjectsReturned` exception is an -attribute of the model class. +If ``get()`` doesn't find any object, it raises a :exc:`Model.DoesNotExist +` exception:: -``get()`` raises a :exc:`~django.db.models.Model.DoesNotExist` exception if an -object wasn't found for the given parameters. This exception is an attribute -of the model class. Example:: + Entry.objects.get(id=-999) # raises Entry.DoesNotExist - Entry.objects.get(id='foo') # raises Entry.DoesNotExist +If ``get()`` finds more than one object, it raises a +:exc:`Model.MultipleObjectsReturned +` exception:: -The :exc:`~django.db.models.Model.DoesNotExist` exception inherits from -:exc:`django.core.exceptions.ObjectDoesNotExist`, so you can target multiple -:exc:`~django.db.models.Model.DoesNotExist` exceptions. Example:: + Entry.objects.get(name='A Duplicated Name') # raises Entry.MultipleObjectsReturned + +Both these exception classes are attributes of the model class, and specific to +that model. If you want to handle such exceptions from several ``get()`` calls +for different models, you can use their generic base classes. For example, you +can use :exc:`django.core.exceptions.ObjectDoesNotExist` to handle +:exc:`~django.db.models.Model.DoesNotExist` exceptions from multiple models:: from django.core.exceptions import ObjectDoesNotExist + try: - e = Entry.objects.get(id=3) - b = Blog.objects.get(id=1) + blog = Blog.objects.get(id=1) + entry = Entry.objects.get(blog=blog, entry_number=1) except ObjectDoesNotExist: - print("Either the entry or blog doesn't exist.") - -If you expect a queryset to return one row, you can use ``get()`` without any -arguments to return the object for that row:: - - entry = Entry.objects.filter(...).exclude(...).get() + print("Either the blog or entry doesn't exist.") ``create()`` ~~~~~~~~~~~~ From f4b7996e9b05a6158482da937723700872429847 Mon Sep 17 00:00:00 2001 From: Buk Bukowski <38087302+bukowa@users.noreply.github.com> Date: Wed, 22 Jul 2020 22:38:27 +0200 Subject: [PATCH 47/88] [3.0.x] Fixed #31814 -- Fixed typo in docs/ref/settings.txt. Backport of f65454801bfa13fc043fee0aca8f49af41380683 from master --- docs/ref/settings.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 96b2b974ab5a..18fba8ac328f 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -246,7 +246,7 @@ See the :ref:`cache documentation ` for more information. ``CACHE_MIDDLEWARE_ALIAS`` -------------------------- -Default: ``default`` +Default: ``'default'`` The cache connection to use for the :ref:`cache middleware `. From b732afe30a66ac9ae13c3a43e582fb7cb6315ee4 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Wed, 22 Jul 2020 08:10:05 -0300 Subject: [PATCH 48/88] [3.0.x] Fixed typo in docs/ref/models/querysets.txt. Backport of 51e536178cba9489b9d759f69f72f442af16ba32 from master --- docs/ref/models/querysets.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index 6f0b74ec92f0..c72c7885d0b3 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -851,8 +851,8 @@ duplicate values, use the ``all=True`` argument. of the type of the first ``QuerySet`` even if the arguments are ``QuerySet``\s of other models. Passing different models works as long as the ``SELECT`` list is the same in all ``QuerySet``\s (at least the types, the names don't matter -as long as the types in the same order). In such cases, you must use the column -names from the first ``QuerySet`` in ``QuerySet`` methods applied to the +as long as the types are in the same order). In such cases, you must use the +column names from the first ``QuerySet`` in ``QuerySet`` methods applied to the resulting ``QuerySet``. For example:: >>> qs1 = Author.objects.values_list('name') From 6fc4445cff9a181317d6e1718b077032bbfea8a8 Mon Sep 17 00:00:00 2001 From: LincolnPuzey Date: Thu, 23 Jul 2020 19:12:58 +0800 Subject: [PATCH 49/88] [3.0.x] Fixed #31816 -- Corrected the expected content type in StreamingHttpResponse docs. Backport of d75436109694c286d9af48ae94ca39759d080214 from master --- docs/ref/request-response.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ref/request-response.txt b/docs/ref/request-response.txt index 8eb578d50598..99878f6ab97c 100644 --- a/docs/ref/request-response.txt +++ b/docs/ref/request-response.txt @@ -1084,7 +1084,7 @@ The :class:`StreamingHttpResponse` is not a subclass of :class:`HttpResponse`, because it features a slightly different API. However, it is almost identical, with the following notable differences: -* It should be given an iterator that yields strings as content. +* It should be given an iterator that yields bytestrings as content. * You cannot access its content, except by iterating the response object itself. This should only occur when the response is returned to the client. From 08063f0122ce58ecb00ee94f0ca78a42df0b0af2 Mon Sep 17 00:00:00 2001 From: Harpreet Sharma Date: Thu, 23 Jul 2020 22:18:39 +0530 Subject: [PATCH 50/88] [3.0.x] Fixed #31821 -- Removed outdated note in FILE_UPLOAD_PERMISSIONS docs. Follow up to 22aab8662f0368b63f91f2526bdd0532524bc0fe Backport of 248d03fbe932b0844c628e56dafba334f9e028e4 from master --- docs/ref/settings.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 18fba8ac328f..cfabdf7d3979 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1489,10 +1489,9 @@ The numeric mode (i.e. ``0o644``) to set newly uploaded files to. For more information about what these modes mean, see the documentation for :func:`os.chmod`. -If this isn't given or is ``None``, you'll get operating-system -dependent behavior. On most platforms, temporary files will have a mode -of ``0o600``, and files saved from memory will be saved using the -system's standard umask. +If ``None``, you'll get operating-system dependent behavior. On most platforms, +temporary files will have a mode of ``0o600``, and files saved from memory will +be saved using the system's standard umask. For security reasons, these permissions aren't applied to the temporary files that are stored in :setting:`FILE_UPLOAD_TEMP_DIR`. From e0397e11b560a502933fbffd84ff069b4dcbd31b Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 28 Jul 2020 12:07:18 +0200 Subject: [PATCH 51/88] [3.0.x] Refs #30165 -- Removed leftover 'u' prefix. Backport of bac5777bff8e8d8189193438b5af52f158a3f2a4 from master --- docs/topics/i18n/translation.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index 441085b4974f..34098a627ee3 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -447,7 +447,7 @@ helper function described next. Lazy translations and plural ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When using lazy translation for a plural string (``[u]n[p]gettext_lazy``), you +When using lazy translation for a plural string (``n[p]gettext_lazy``), you generally don't know the ``number`` argument at the time of the string definition. Therefore, you are authorized to pass a key name instead of an integer as the ``number`` argument. Then ``number`` will be looked up in the From 8f8d4b3d526a577004860a6a84302c84ca078308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Bartosi=C5=84ski?= Date: Wed, 29 Jul 2020 12:53:11 +0200 Subject: [PATCH 52/88] [3.0.x] Corrected admin.register() signature in docs. Backport of eb215da363e6cf0e8f3405db3c4392398c8777cb from master --- docs/ref/contrib/admin/index.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ref/contrib/admin/index.txt b/docs/ref/contrib/admin/index.txt index 0b0085b8c0c3..14187891d9c2 100644 --- a/docs/ref/contrib/admin/index.txt +++ b/docs/ref/contrib/admin/index.txt @@ -109,7 +109,7 @@ Other topics The ``register`` decorator -------------------------- -.. function:: register(*models, site=django.admin.sites.site) +.. function:: register(*models, site=django.contrib.admin.sites.site) There is also a decorator for registering your ``ModelAdmin`` classes:: From d70ed9439d65a5c5dff3f2b7c3a37a0c175f2f88 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Fri, 31 Jul 2020 07:38:01 +0200 Subject: [PATCH 53/88] [3.0.x] Corrected signing.dumps()/loads() signatures in docs. Backport of 8703680ebee47bfa9e912a30a0509798500bf42a from master --- docs/topics/signing.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/signing.txt b/docs/topics/signing.txt index cd2b4ef2e8aa..c60930c1e5c1 100644 --- a/docs/topics/signing.txt +++ b/docs/topics/signing.txt @@ -167,12 +167,12 @@ and tuples) if you pass in a tuple, you will get a list from >>> signing.loads(value) ['a', 'b', 'c'] -.. function:: dumps(obj, key=None, salt='django.core.signing', compress=False) +.. function:: dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer, compress=False) Returns URL-safe, sha1 signed base64 compressed JSON string. Serialized object is signed using :class:`~TimestampSigner`. -.. function:: loads(string, key=None, salt='django.core.signing', max_age=None) +.. function:: loads(string, key=None, salt='django.core.signing', serializer=JSONSerializer, max_age=None) Reverse of ``dumps()``, raises ``BadSignature`` if signature fails. Checks ``max_age`` (in seconds) if given. From b1ae5d015b869ffd41bfcf3d76cea9dc165ba49c Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 3 Aug 2020 08:52:28 +0200 Subject: [PATCH 54/88] [3.0.x] Added release date for 2.2.15 and 3.0.9. Backport of b68b8cb89abb35ff2152175ea540619ec384b1f4 from master --- docs/releases/2.2.15.txt | 2 +- docs/releases/3.0.9.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/releases/2.2.15.txt b/docs/releases/2.2.15.txt index d8ed58b596cb..c36d746d5db9 100644 --- a/docs/releases/2.2.15.txt +++ b/docs/releases/2.2.15.txt @@ -2,7 +2,7 @@ Django 2.2.15 release notes =========================== -*Expected August 3, 2020* +*August 3, 2020* Django 2.2.15 fixes two bugs in 2.2.14. diff --git a/docs/releases/3.0.9.txt b/docs/releases/3.0.9.txt index 65310384ec3c..7317ead794f5 100644 --- a/docs/releases/3.0.9.txt +++ b/docs/releases/3.0.9.txt @@ -2,7 +2,7 @@ Django 3.0.9 release notes ========================== -*Expected August 3, 2020* +*August 3, 2020* Django 3.0.9 fixes several bugs in 3.0.8. From 39716e4578e9563cbba2fa754795901d0e25501f Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 3 Aug 2020 09:02:29 +0200 Subject: [PATCH 55/88] [3.0.x] Bumped version for 3.0.9 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 0d40dc097726..360aaa71d638 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 9, 'alpha', 0) +VERSION = (3, 0, 9, 'final', 0) __version__ = get_version(VERSION) From 8f8edec6c438bc0a915e07a25e5764bbfa296b47 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 3 Aug 2020 09:06:21 +0200 Subject: [PATCH 56/88] [3.0.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 360aaa71d638..31218bce6529 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 9, 'final', 0) +VERSION = (3, 0, 10, 'alpha', 0) __version__ = get_version(VERSION) From 9f74a248039974be2e0571ed034a3c9a735a3b60 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 11 Aug 2020 09:44:31 +0200 Subject: [PATCH 57/88] [3.0.x] Added stub release notes for 2.2.16 and 3.0.10. Backport of 8a5683b6b2aede38edcff070686ed1fce470dec5 from master --- docs/releases/2.2.16.txt | 12 ++++++++++++ docs/releases/3.0.10.txt | 12 ++++++++++++ docs/releases/index.txt | 2 ++ 3 files changed, 26 insertions(+) create mode 100644 docs/releases/2.2.16.txt create mode 100644 docs/releases/3.0.10.txt diff --git a/docs/releases/2.2.16.txt b/docs/releases/2.2.16.txt new file mode 100644 index 000000000000..7e80406e9dc7 --- /dev/null +++ b/docs/releases/2.2.16.txt @@ -0,0 +1,12 @@ +=========================== +Django 2.2.16 release notes +=========================== + +*Expected September 1, 2020* + +Django 2.2.16 fixes a data loss bug in 2.2.15. + +Bugfixes +======== + +* ... diff --git a/docs/releases/3.0.10.txt b/docs/releases/3.0.10.txt new file mode 100644 index 000000000000..66c49c5dffce --- /dev/null +++ b/docs/releases/3.0.10.txt @@ -0,0 +1,12 @@ +=========================== +Django 3.0.10 release notes +=========================== + +*Expected September 1, 2020* + +Django 3.0.10 fixes a data loss bug in 3.0.9. + +Bugfixes +======== + +* ... diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 6b9674005fe3..4b2f3af8baf9 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.0.10 3.0.9 3.0.8 3.0.7 @@ -41,6 +42,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 2.2.16 2.2.15 2.2.14 2.2.13 From 784ed4ada112753c35dd7c6b85686ad1d935b65d Mon Sep 17 00:00:00 2001 From: Daniel Hillier Date: Sat, 8 Aug 2020 15:17:36 +1000 Subject: [PATCH 58/88] [3.0.x] Fixed #31866 -- Fixed locking proxy models in QuerySet.select_for_update(of=()). Backport of 60626162f76f26d32a38d18151700cb041201fb3 from master --- django/db/models/sql/compiler.py | 6 ++++-- docs/releases/2.2.16.txt | 5 ++++- docs/releases/3.0.10.txt | 5 ++++- tests/select_for_update/models.py | 14 ++++++++++++++ tests/select_for_update/tests.py | 32 ++++++++++++++++++++++++++++++- 5 files changed, 57 insertions(+), 5 deletions(-) diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index ccec6fdc380c..18f8a5120b1b 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -973,7 +973,8 @@ def get_select_for_update_of_arguments(self): the query. """ def _get_parent_klass_info(klass_info): - for parent_model, parent_link in klass_info['model']._meta.parents.items(): + concrete_model = klass_info['model']._meta.concrete_model + for parent_model, parent_link in concrete_model._meta.parents.items(): parent_list = parent_model._meta.get_parent_list() yield { 'model': parent_model, @@ -998,8 +999,9 @@ def _get_first_selected_col_from_model(klass_info): select_fields is filled recursively, so it also contains fields from the parent models. """ + concrete_model = klass_info['model']._meta.concrete_model for select_index in klass_info['select_fields']: - if self.select[select_index][0].target.model == klass_info['model']: + if self.select[select_index][0].target.model == concrete_model: return self.select[select_index][0] def _get_field_choices(): diff --git a/docs/releases/2.2.16.txt b/docs/releases/2.2.16.txt index 7e80406e9dc7..daba6f6f5250 100644 --- a/docs/releases/2.2.16.txt +++ b/docs/releases/2.2.16.txt @@ -9,4 +9,7 @@ Django 2.2.16 fixes a data loss bug in 2.2.15. Bugfixes ======== -* ... +* Fixed a data loss possibility in the + :meth:`~django.db.models.query.QuerySet.select_for_update()`. When using + related fields pointing to a proxy model in the ``of`` argument, the + corresponding model was not locked (:ticket:`31866`). diff --git a/docs/releases/3.0.10.txt b/docs/releases/3.0.10.txt index 66c49c5dffce..2d97851163bf 100644 --- a/docs/releases/3.0.10.txt +++ b/docs/releases/3.0.10.txt @@ -9,4 +9,7 @@ Django 3.0.10 fixes a data loss bug in 3.0.9. Bugfixes ======== -* ... +* Fixed a data loss possibility in the + :meth:`~django.db.models.query.QuerySet.select_for_update()`. When using + related fields pointing to a proxy model in the ``of`` argument, the + corresponding model was not locked (:ticket:`31866`). diff --git a/tests/select_for_update/models.py b/tests/select_for_update/models.py index 305e8cac490b..0a6396bc7066 100644 --- a/tests/select_for_update/models.py +++ b/tests/select_for_update/models.py @@ -23,6 +23,20 @@ class EUCity(models.Model): country = models.ForeignKey(EUCountry, models.CASCADE) +class CountryProxy(Country): + class Meta: + proxy = True + + +class CountryProxyProxy(CountryProxy): + class Meta: + proxy = True + + +class CityCountryProxy(models.Model): + country = models.ForeignKey(CountryProxyProxy, models.CASCADE) + + class Person(models.Model): name = models.CharField(max_length=30) born = models.ForeignKey(City, models.CASCADE, related_name='+') diff --git a/tests/select_for_update/tests.py b/tests/select_for_update/tests.py index 3622a95c11a7..70511b09a123 100644 --- a/tests/select_for_update/tests.py +++ b/tests/select_for_update/tests.py @@ -15,7 +15,9 @@ ) from django.test.utils import CaptureQueriesContext -from .models import City, Country, EUCity, EUCountry, Person, PersonProfile +from .models import ( + City, CityCountryProxy, Country, EUCity, EUCountry, Person, PersonProfile, +) class SelectForUpdateTests(TransactionTestCase): @@ -195,6 +197,21 @@ def test_for_update_sql_multilevel_model_inheritance_ptr_generated_of(self): expected = [connection.ops.quote_name(value) for value in expected] self.assertTrue(self.has_for_update_sql(ctx.captured_queries, of=expected)) + @skipUnlessDBFeature('has_select_for_update_of') + def test_for_update_sql_model_proxy_generated_of(self): + with transaction.atomic(), CaptureQueriesContext(connection) as ctx: + list(CityCountryProxy.objects.select_related( + 'country', + ).select_for_update( + of=('country',), + )) + if connection.features.select_for_update_of_column: + expected = ['select_for_update_country"."entity_ptr_id'] + else: + expected = ['select_for_update_country'] + expected = [connection.ops.quote_name(value) for value in expected] + self.assertTrue(self.has_for_update_sql(ctx.captured_queries, of=expected)) + @skipUnlessDBFeature('has_select_for_update_of') def test_for_update_of_followed_by_values(self): with transaction.atomic(): @@ -353,6 +370,19 @@ def test_model_inheritance_of_argument_raises_error_ptr_in_choices(self): with transaction.atomic(): EUCountry.objects.select_for_update(of=('name',)).get() + @skipUnlessDBFeature('has_select_for_update', 'has_select_for_update_of') + def test_model_proxy_of_argument_raises_error_proxy_field_in_choices(self): + msg = ( + 'Invalid field name(s) given in select_for_update(of=(...)): ' + 'name. Only relational fields followed in the query are allowed. ' + 'Choices are: self, country, country__entity_ptr.' + ) + with self.assertRaisesMessage(FieldError, msg): + with transaction.atomic(): + CityCountryProxy.objects.select_related( + 'country', + ).select_for_update(of=('name',)).get() + @skipUnlessDBFeature('has_select_for_update', 'has_select_for_update_of') def test_reverse_one_to_one_of_arguments(self): """ From c33b6ceff4c3462dec8555e395c6f45faa61c490 Mon Sep 17 00:00:00 2001 From: Gert Burger Date: Fri, 7 Aug 2020 10:02:29 +0200 Subject: [PATCH 59/88] [3.0.x] Fixed #31863 -- Prevented mutating model state by copies of model instances. Regression in bfb746f983aa741afa3709794e70f1e0ab6040b5. Backport of 94ea79be137f3cb30949bf82198e96e094f2650d from master --- django/db/models/base.py | 5 ++++- tests/model_regress/tests.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/django/db/models/base.py b/django/db/models/base.py index 63801da3c299..814e6ee28083 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -543,7 +543,10 @@ def __reduce__(self): def __getstate__(self): """Hook to allow choosing the attributes to pickle.""" - return self.__dict__ + state = self.__dict__.copy() + state['_state'] = copy.copy(state['_state']) + state['_state'].fields_cache = state['_state'].fields_cache.copy() + return state def __setstate__(self, state): msg = None diff --git a/tests/model_regress/tests.py b/tests/model_regress/tests.py index 28eed87008a4..87df240d8153 100644 --- a/tests/model_regress/tests.py +++ b/tests/model_regress/tests.py @@ -1,3 +1,4 @@ +import copy import datetime from operator import attrgetter @@ -256,3 +257,17 @@ def test_model_with_evaluate_method(self): dept = Department.objects.create(pk=1, name='abc') dept.evaluate = 'abc' Worker.objects.filter(department=dept) + + +class ModelFieldsCacheTest(TestCase): + def test_fields_cache_reset_on_copy(self): + department1 = Department.objects.create(id=1, name='department1') + department2 = Department.objects.create(id=2, name='department2') + worker1 = Worker.objects.create(name='worker', department=department1) + worker2 = copy.copy(worker1) + + self.assertEqual(worker2.department, department1) + # Changing related fields doesn't mutate the base object. + worker2.department = department2 + self.assertEqual(worker2.department, department2) + self.assertEqual(worker1.department, department1) From ab5491c7cc46f81be9ca388ae3dcda02c30d013d Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Thu, 13 Aug 2020 16:29:55 +0200 Subject: [PATCH 60/88] [3.0.x] Refs #31863 -- Added release notes for 94ea79be137f3cb30949bf82198e96e094f2650d. Backport of 21768a99f47ee73a2f93405151550ef7c3d9c8a2 from master --- docs/releases/2.2.16.txt | 5 ++++- docs/releases/3.0.10.txt | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/releases/2.2.16.txt b/docs/releases/2.2.16.txt index daba6f6f5250..ce700e5043c7 100644 --- a/docs/releases/2.2.16.txt +++ b/docs/releases/2.2.16.txt @@ -4,7 +4,7 @@ Django 2.2.16 release notes *Expected September 1, 2020* -Django 2.2.16 fixes a data loss bug in 2.2.15. +Django 2.2.16 fixes two data loss bugs in 2.2.15. Bugfixes ======== @@ -13,3 +13,6 @@ Bugfixes :meth:`~django.db.models.query.QuerySet.select_for_update()`. When using related fields pointing to a proxy model in the ``of`` argument, the corresponding model was not locked (:ticket:`31866`). + +* Fixed a data loss possibility, following a regression in Django 2.0, when + copying model instances with a cached fields value (:ticket:`31863`). diff --git a/docs/releases/3.0.10.txt b/docs/releases/3.0.10.txt index 2d97851163bf..9c60435a2807 100644 --- a/docs/releases/3.0.10.txt +++ b/docs/releases/3.0.10.txt @@ -4,7 +4,7 @@ Django 3.0.10 release notes *Expected September 1, 2020* -Django 3.0.10 fixes a data loss bug in 3.0.9. +Django 3.0.10 fixes two data loss bugs in 3.0.9. Bugfixes ======== @@ -13,3 +13,6 @@ Bugfixes :meth:`~django.db.models.query.QuerySet.select_for_update()`. When using related fields pointing to a proxy model in the ``of`` argument, the corresponding model was not locked (:ticket:`31866`). + +* Fixed a data loss possibility, following a regression in Django 2.0, when + copying model instances with a cached fields value (:ticket:`31863`). From db8b935730002f2cff6df957e8adab9561072834 Mon Sep 17 00:00:00 2001 From: Kaustubh <55352418+kc611@users.noreply.github.com> Date: Fri, 21 Aug 2020 13:17:37 +0530 Subject: [PATCH 61/88] [3.0.x] Fixed #31925 -- Fixed typo in docs/releases/3.0.txt. Backport of 3e753d3de33469493b1f0947a2e0152c4000ed40 from master --- docs/releases/3.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releases/3.0.txt b/docs/releases/3.0.txt index 2bfd7d4719c8..284782d80ce4 100644 --- a/docs/releases/3.0.txt +++ b/docs/releases/3.0.txt @@ -513,7 +513,7 @@ In older versions, the :setting:`FILE_UPLOAD_PERMISSIONS` setting defaults to uploaded files having different permissions depending on their size and which upload handler is used. -``FILE_UPLOAD_PERMISSION`` now defaults to ``0o644`` to avoid this +``FILE_UPLOAD_PERMISSIONS`` now defaults to ``0o644`` to avoid this inconsistency. New default values for security settings From 08892bffd275c79ee1f8f67639eb170aaaf1181e Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Fri, 21 Aug 2020 11:44:46 +0200 Subject: [PATCH 62/88] [3.0.x] Fixed CVE-2020-24583, #31921 -- Fixed permissions on intermediate-level static and storage directories on Python 3.7+. Thanks WhiteSage for the report. Backport of ea0febbba531a3ecc8c77b570efbfb68ca7155db from master. --- django/core/files/storage.py | 6 +-- docs/releases/2.2.16.txt | 13 ++++- docs/releases/3.0.10.txt | 13 ++++- tests/file_storage/tests.py | 16 +++--- .../project/documents/nested/css/base.css | 1 + tests/staticfiles_tests/test_storage.py | 52 +++++++++++++------ 6 files changed, 75 insertions(+), 26 deletions(-) create mode 100644 tests/staticfiles_tests/project/documents/nested/css/base.css diff --git a/django/core/files/storage.py b/django/core/files/storage.py index 4c27fce6052e..af70b3fa1584 100644 --- a/django/core/files/storage.py +++ b/django/core/files/storage.py @@ -237,9 +237,9 @@ def _save(self, name, content): directory = os.path.dirname(full_path) try: if self.directory_permissions_mode is not None: - # os.makedirs applies the global umask, so we reset it, - # for consistency with file_permissions_mode behavior. - old_umask = os.umask(0) + # Set the umask because os.makedirs() doesn't apply the "mode" + # argument to intermediate-level directories. + old_umask = os.umask(0o777 & ~self.directory_permissions_mode) try: os.makedirs(directory, self.directory_permissions_mode, exist_ok=True) finally: diff --git a/docs/releases/2.2.16.txt b/docs/releases/2.2.16.txt index ce700e5043c7..f0c3ec894aab 100644 --- a/docs/releases/2.2.16.txt +++ b/docs/releases/2.2.16.txt @@ -4,7 +4,18 @@ Django 2.2.16 release notes *Expected September 1, 2020* -Django 2.2.16 fixes two data loss bugs in 2.2.15. +Django 2.2.16 fixes a security issue and two data loss bugs in 2.2.15. + +CVE-2020-24583: Incorrect permissions on intermediate-level directories on Python 3.7+ +====================================================================================== + +On Python 3.7+, :setting:`FILE_UPLOAD_DIRECTORY_PERMISSIONS` mode was not +applied to intermediate-level directories created in the process of uploading +files and to intermediate-level collected static directories when using the +:djadmin:`collectstatic` management command. + +You should review and manually fix permissions on existing intermediate-level +directories. Bugfixes ======== diff --git a/docs/releases/3.0.10.txt b/docs/releases/3.0.10.txt index 9c60435a2807..4238a9fd7154 100644 --- a/docs/releases/3.0.10.txt +++ b/docs/releases/3.0.10.txt @@ -4,7 +4,18 @@ Django 3.0.10 release notes *Expected September 1, 2020* -Django 3.0.10 fixes two data loss bugs in 3.0.9. +Django 3.0.10 fixes a security issue and two data loss bugs in 3.0.9. + +CVE-2020-24583: Incorrect permissions on intermediate-level directories on Python 3.7+ +====================================================================================== + +On Python 3.7+, :setting:`FILE_UPLOAD_DIRECTORY_PERMISSIONS` mode was not +applied to intermediate-level directories created in the process of uploading +files and to intermediate-level collected static directories when using the +:djadmin:`collectstatic` management command. + +You should review and manually fix permissions on existing intermediate-level +directories. Bugfixes ======== diff --git a/tests/file_storage/tests.py b/tests/file_storage/tests.py index 1c4176014c51..e2a1d06b5dc2 100644 --- a/tests/file_storage/tests.py +++ b/tests/file_storage/tests.py @@ -7,6 +7,7 @@ import unittest from datetime import datetime, timedelta from io import StringIO +from pathlib import Path from urllib.request import urlopen from django.core.cache import cache @@ -910,16 +911,19 @@ def test_file_upload_default_permissions(self): @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o765) def test_file_upload_directory_permissions(self): self.storage = FileSystemStorage(self.storage_dir) - name = self.storage.save("the_directory/the_file", ContentFile("data")) - dir_mode = os.stat(os.path.dirname(self.storage.path(name)))[0] & 0o777 - self.assertEqual(dir_mode, 0o765) + name = self.storage.save('the_directory/subdir/the_file', ContentFile('data')) + file_path = Path(self.storage.path(name)) + self.assertEqual(file_path.parent.stat().st_mode & 0o777, 0o765) + self.assertEqual(file_path.parent.parent.stat().st_mode & 0o777, 0o765) @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=None) def test_file_upload_directory_default_permissions(self): self.storage = FileSystemStorage(self.storage_dir) - name = self.storage.save("the_directory/the_file", ContentFile("data")) - dir_mode = os.stat(os.path.dirname(self.storage.path(name)))[0] & 0o777 - self.assertEqual(dir_mode, 0o777 & ~self.umask) + name = self.storage.save('the_directory/subdir/the_file', ContentFile('data')) + file_path = Path(self.storage.path(name)) + expected_mode = 0o777 & ~self.umask + self.assertEqual(file_path.parent.stat().st_mode & 0o777, expected_mode) + self.assertEqual(file_path.parent.parent.stat().st_mode & 0o777, expected_mode) class FileStoragePathParsing(SimpleTestCase): diff --git a/tests/staticfiles_tests/project/documents/nested/css/base.css b/tests/staticfiles_tests/project/documents/nested/css/base.css new file mode 100644 index 000000000000..06041ca25f1e --- /dev/null +++ b/tests/staticfiles_tests/project/documents/nested/css/base.css @@ -0,0 +1 @@ +html {height: 100%;} diff --git a/tests/staticfiles_tests/test_storage.py b/tests/staticfiles_tests/test_storage.py index 93bd60446a42..063fbb38e453 100644 --- a/tests/staticfiles_tests/test_storage.py +++ b/tests/staticfiles_tests/test_storage.py @@ -4,6 +4,7 @@ import tempfile import unittest from io import StringIO +from pathlib import Path from unittest import mock from django.conf import settings @@ -530,12 +531,19 @@ def run_collectstatic(self, **kwargs): ) def test_collect_static_files_permissions(self): call_command('collectstatic', **self.command_params) - test_file = os.path.join(settings.STATIC_ROOT, "test.txt") - test_dir = os.path.join(settings.STATIC_ROOT, "subdir") - file_mode = os.stat(test_file)[0] & 0o777 - dir_mode = os.stat(test_dir)[0] & 0o777 + static_root = Path(settings.STATIC_ROOT) + test_file = static_root / 'test.txt' + file_mode = test_file.stat().st_mode & 0o777 self.assertEqual(file_mode, 0o655) - self.assertEqual(dir_mode, 0o765) + tests = [ + static_root / 'subdir', + static_root / 'nested', + static_root / 'nested' / 'css', + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o765) @override_settings( FILE_UPLOAD_PERMISSIONS=None, @@ -543,12 +551,19 @@ def test_collect_static_files_permissions(self): ) def test_collect_static_files_default_permissions(self): call_command('collectstatic', **self.command_params) - test_file = os.path.join(settings.STATIC_ROOT, "test.txt") - test_dir = os.path.join(settings.STATIC_ROOT, "subdir") - file_mode = os.stat(test_file)[0] & 0o777 - dir_mode = os.stat(test_dir)[0] & 0o777 + static_root = Path(settings.STATIC_ROOT) + test_file = static_root / 'test.txt' + file_mode = test_file.stat().st_mode & 0o777 self.assertEqual(file_mode, 0o666 & ~self.umask) - self.assertEqual(dir_mode, 0o777 & ~self.umask) + tests = [ + static_root / 'subdir', + static_root / 'nested', + static_root / 'nested' / 'css', + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o777 & ~self.umask) @override_settings( FILE_UPLOAD_PERMISSIONS=0o655, @@ -557,12 +572,19 @@ def test_collect_static_files_default_permissions(self): ) def test_collect_static_files_subclass_of_static_storage(self): call_command('collectstatic', **self.command_params) - test_file = os.path.join(settings.STATIC_ROOT, "test.txt") - test_dir = os.path.join(settings.STATIC_ROOT, "subdir") - file_mode = os.stat(test_file)[0] & 0o777 - dir_mode = os.stat(test_dir)[0] & 0o777 + static_root = Path(settings.STATIC_ROOT) + test_file = static_root / 'test.txt' + file_mode = test_file.stat().st_mode & 0o777 self.assertEqual(file_mode, 0o640) - self.assertEqual(dir_mode, 0o740) + tests = [ + static_root / 'subdir', + static_root / 'nested', + static_root / 'nested' / 'css', + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o740) @override_settings( From cdb367c92a0ba72ddc0cbd13ff42b0e6df709554 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Fri, 21 Aug 2020 12:43:45 +0200 Subject: [PATCH 63/88] [3.0.x] Fixed CVE-2020-24584 -- Fixed permission escalation in intermediate-level directories of the file system cache on Python 3.7+. Backport of f56b57976133129b0b351a38bba4ac882badabf0 from master. --- django/core/cache/backends/filebased.py | 8 +++++++- docs/releases/2.2.16.txt | 9 ++++++++- docs/releases/3.0.10.txt | 9 ++++++++- tests/cache/tests.py | 26 ++++++++++++++++++++++++- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py index 48b8df90abd9..64cc175b1835 100644 --- a/django/core/cache/backends/filebased.py +++ b/django/core/cache/backends/filebased.py @@ -113,7 +113,13 @@ def _cull(self): self._delete(fname) def _createdir(self): - os.makedirs(self._dir, 0o700, exist_ok=True) + # Set the umask because os.makedirs() doesn't apply the "mode" argument + # to intermediate-level directories. + old_umask = os.umask(0o077) + try: + os.makedirs(self._dir, 0o700, exist_ok=True) + finally: + os.umask(old_umask) def _key_to_file(self, key, version=None): """ diff --git a/docs/releases/2.2.16.txt b/docs/releases/2.2.16.txt index f0c3ec894aab..f531871d1ae5 100644 --- a/docs/releases/2.2.16.txt +++ b/docs/releases/2.2.16.txt @@ -4,7 +4,7 @@ Django 2.2.16 release notes *Expected September 1, 2020* -Django 2.2.16 fixes a security issue and two data loss bugs in 2.2.15. +Django 2.2.16 fixes two security issues and two data loss bugs in 2.2.15. CVE-2020-24583: Incorrect permissions on intermediate-level directories on Python 3.7+ ====================================================================================== @@ -17,6 +17,13 @@ files and to intermediate-level collected static directories when using the You should review and manually fix permissions on existing intermediate-level directories. +CVE-2020-24584: Permission escalation in intermediate-level directories of the file system cache on Python 3.7+ +=============================================================================================================== + +On Python 3.7+, the intermediate-level directories of the file system cache had +the system's standard umask rather than ``0o077`` (no group or others +permissions). + Bugfixes ======== diff --git a/docs/releases/3.0.10.txt b/docs/releases/3.0.10.txt index 4238a9fd7154..ebc67d472535 100644 --- a/docs/releases/3.0.10.txt +++ b/docs/releases/3.0.10.txt @@ -4,7 +4,7 @@ Django 3.0.10 release notes *Expected September 1, 2020* -Django 3.0.10 fixes a security issue and two data loss bugs in 3.0.9. +Django 3.0.10 fixes two security issues and two data loss bugs in 3.0.9. CVE-2020-24583: Incorrect permissions on intermediate-level directories on Python 3.7+ ====================================================================================== @@ -17,6 +17,13 @@ files and to intermediate-level collected static directories when using the You should review and manually fix permissions on existing intermediate-level directories. +CVE-2020-24584: Permission escalation in intermediate-level directories of the file system cache on Python 3.7+ +=============================================================================================================== + +On Python 3.7+, the intermediate-level directories of the file system cache had +the system's standard umask rather than ``0o077`` (no group or others +permissions). + Bugfixes ======== diff --git a/tests/cache/tests.py b/tests/cache/tests.py index d8ea67687577..c0f90b832637 100644 --- a/tests/cache/tests.py +++ b/tests/cache/tests.py @@ -6,11 +6,13 @@ import pickle import re import shutil +import sys import tempfile import threading import time import unittest -from unittest import mock +from pathlib import Path +from unittest import mock, skipIf from django.conf import settings from django.core import management, signals @@ -1443,6 +1445,28 @@ def test_get_ignores_enoent(self): # Returns the default instead of erroring. self.assertEqual(cache.get('foo', 'baz'), 'baz') + @skipIf( + sys.platform == 'win32', + 'Windows only partially supports umasks and chmod.', + ) + def test_cache_dir_permissions(self): + os.rmdir(self.dirname) + dir_path = Path(self.dirname) / 'nested' / 'filebasedcache' + for cache_params in settings.CACHES.values(): + cache_params['LOCATION'] = dir_path + setting_changed.send(self.__class__, setting='CACHES', enter=False) + cache.set('foo', 'bar') + self.assertIs(dir_path.exists(), True) + tests = [ + dir_path, + dir_path.parent, + dir_path.parent.parent, + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o700) + def test_get_does_not_ignore_non_filenotfound_exceptions(self): with mock.patch('builtins.open', side_effect=OSError): with self.assertRaises(OSError): From 79e6eb3853882924078d8e6557470404d756f5db Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Tue, 1 Sep 2020 09:56:42 +0200 Subject: [PATCH 64/88] [3.0.x] Added release date for 3.0.10, and 2.2.16. Backport of 976e2b7420c0f7e3060a13792b97511a9aad31d7 from master --- docs/releases/2.2.16.txt | 2 +- docs/releases/3.0.10.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/releases/2.2.16.txt b/docs/releases/2.2.16.txt index f531871d1ae5..31231fb0655d 100644 --- a/docs/releases/2.2.16.txt +++ b/docs/releases/2.2.16.txt @@ -2,7 +2,7 @@ Django 2.2.16 release notes =========================== -*Expected September 1, 2020* +*September 1, 2020* Django 2.2.16 fixes two security issues and two data loss bugs in 2.2.15. diff --git a/docs/releases/3.0.10.txt b/docs/releases/3.0.10.txt index ebc67d472535..0583f8d5aa5d 100644 --- a/docs/releases/3.0.10.txt +++ b/docs/releases/3.0.10.txt @@ -2,7 +2,7 @@ Django 3.0.10 release notes =========================== -*Expected September 1, 2020* +*September 1, 2020* Django 3.0.10 fixes two security issues and two data loss bugs in 3.0.9. From a208020ecd0aa568cb5be6ef8b76701224ebe1a2 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Tue, 1 Sep 2020 10:26:20 +0200 Subject: [PATCH 65/88] [3.0.x] Bumped version for 3.0.10 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 31218bce6529..aaacd7702ebf 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 10, 'alpha', 0) +VERSION = (3, 0, 10, 'final', 0) __version__ = get_version(VERSION) From 26323dbcf4b4bf37d4d7b49ac17c98679c0e4587 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Tue, 1 Sep 2020 10:29:42 +0200 Subject: [PATCH 66/88] [3.0.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index aaacd7702ebf..f2237f0b7384 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 10, 'final', 0) +VERSION = (3, 0, 11, 'alpha', 0) __version__ = get_version(VERSION) From 1734484f1265804aca5e84299d03d013e59dacc5 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Tue, 1 Sep 2020 11:32:57 +0200 Subject: [PATCH 67/88] [3.0.x] Added CVE-2020-24583 & CVE-2020-24584 to security archive. Backport of d5b526bf78a9e5d9760e0c0f7647622bf47782fe from master --- docs/releases/security.txt | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index d896974e72e8..3d659bc1ea42 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -1106,3 +1106,31 @@ Versions affected * Django 3.0 :commit:`(patch) <1f2dd37f6fcefdd10ed44cb233b2e62b520afb38>` * Django 2.2 :commit:`(patch) <6d61860b22875f358fac83d903dc629897934815>` + +September 1, 2020 - :cve:`2020-24583` +------------------------------------- + +Incorrect permissions on intermediate-level directories on Python 3.7+. `Full +description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.1 :commit:`(patch) <934430d22aa5d90c2ba33495ff69a6a1d997d584>` +* Django 3.0 :commit:`(patch) <08892bffd275c79ee1f8f67639eb170aaaf1181e>` +* Django 2.2 :commit:`(patch) <375657a71c889c588f723469bd868bd1d40c369f>` + +September 1, 2020 - :cve:`2020-24584` +------------------------------------- + +Permission escalation in intermediate-level directories of the file system +cache on Python 3.7+. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.1 :commit:`(patch) <2b099caa5923afa8cfb5f1e8c0d56b6e0e81915b>` +* Django 3.0 :commit:`(patch) ` +* Django 2.2 :commit:`(patch) ` From 9deb850e234ea52ac230916b6edd5d4889449e55 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 6 Oct 2020 11:25:04 +0200 Subject: [PATCH 68/88] [3.0.x] Skipped GetImageDimensionsTests.test_webp when WEBP is not installed. Bumped minimum Pillow version to 4.2.0 in test requirements. Backport of fce389af7cf95151118c9fc7cafd777a31f94946 from master --- docs/internals/contributing/writing-code/unit-tests.txt | 2 +- tests/files/tests.py | 5 ++++- tests/requirements/py3.txt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index efc862dc76e4..ad7c5b494e93 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -277,7 +277,7 @@ dependencies: * geoip2_ * jinja2_ 2.7+ * numpy_ -* Pillow_ +* Pillow_ 4.2.0+ * PyYAML_ * pytz_ (required) * pywatchman_ diff --git a/tests/files/tests.py b/tests/files/tests.py index 1c005dde5779..d90b79efe353 100644 --- a/tests/files/tests.py +++ b/tests/files/tests.py @@ -17,9 +17,11 @@ ) try: - from PIL import Image + from PIL import Image, features + HAS_WEBP = features.check('webp') except ImportError: Image = None + HAS_WEBP = False else: from django.core.files import images @@ -343,6 +345,7 @@ def test_valid_image(self): size = images.get_image_dimensions(fh) self.assertEqual(size, (None, None)) + @unittest.skipUnless(HAS_WEBP, 'WEBP not installed') def test_webp(self): img_path = os.path.join(os.path.dirname(__file__), 'test.webp') with open(img_path, 'rb') as fh: diff --git a/tests/requirements/py3.txt b/tests/requirements/py3.txt index fd677713f880..8b8f2a6f33da 100644 --- a/tests/requirements/py3.txt +++ b/tests/requirements/py3.txt @@ -5,7 +5,7 @@ docutils geoip2 jinja2 >= 2.9.2 numpy -Pillow != 5.4.0 +Pillow >=4.2.0, != 5.4.0 # pylibmc/libmemcached can't be built on Windows. pylibmc; sys.platform != 'win32' python-memcached >= 1.59 From 301bca9394cd977bc085c5cc50bd0ef9f6af52d7 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 13 Oct 2020 08:35:01 +0200 Subject: [PATCH 69/88] [3.0.x] Refs #31040 -- Doc'd Python 3.9 compatibility. Backport of e18156b6c35908f2a4026287b5225a6a4da8af1a from master. --- docs/faq/install.txt | 4 ++-- docs/releases/2.2.17.txt | 7 +++++++ docs/releases/2.2.txt | 6 +++--- docs/releases/3.0.11.txt | 7 +++++++ docs/releases/3.0.txt | 4 ++-- docs/releases/index.txt | 2 ++ setup.py | 1 + tox.ini | 2 +- 8 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 docs/releases/2.2.17.txt create mode 100644 docs/releases/3.0.11.txt diff --git a/docs/faq/install.txt b/docs/faq/install.txt index fc8f2ec7dcb3..972fb73cd963 100644 --- a/docs/faq/install.txt +++ b/docs/faq/install.txt @@ -53,8 +53,8 @@ Django version Python versions 1.11 2.7, 3.4, 3.5, 3.6, 3.7 (added in 1.11.17) 2.0 3.4, 3.5, 3.6, 3.7 2.1 3.5, 3.6, 3.7 -2.2 3.5, 3.6, 3.7, 3.8 (added in 2.2.8) -3.0 3.6, 3.7, 3.8 +2.2 3.5, 3.6, 3.7, 3.8 (added in 2.2.8), 3.9 (added in 2.2.17) +3.0 3.6, 3.7, 3.8, 3.9 (added in 3.0.11) ============== =============== For each version of Python, only the latest micro release (A.B.C) is officially diff --git a/docs/releases/2.2.17.txt b/docs/releases/2.2.17.txt new file mode 100644 index 000000000000..d574de6574b0 --- /dev/null +++ b/docs/releases/2.2.17.txt @@ -0,0 +1,7 @@ +=========================== +Django 2.2.17 release notes +=========================== + +*Expected November 2, 2020* + +Django 2.2.17 adds compatibility with Python 3.9. diff --git a/docs/releases/2.2.txt b/docs/releases/2.2.txt index d6c9c19fc650..7f9822c8cbb2 100644 --- a/docs/releases/2.2.txt +++ b/docs/releases/2.2.txt @@ -23,9 +23,9 @@ end in April 2020. Python compatibility ==================== -Django 2.2 supports Python 3.5, 3.6, 3.7, and 3.8 (as of 2.2.8). We -**highly recommend** and only officially support the latest release of each -series. +Django 2.2 supports Python 3.5, 3.6, 3.7, 3.8 (as of 2.2.8), and 3.9 (as of +2.2.17). We **highly recommend** and only officially support the latest release +of each series. .. _whats-new-2.2: diff --git a/docs/releases/3.0.11.txt b/docs/releases/3.0.11.txt new file mode 100644 index 000000000000..a5009feb5119 --- /dev/null +++ b/docs/releases/3.0.11.txt @@ -0,0 +1,7 @@ +=========================== +Django 3.0.11 release notes +=========================== + +*Expected November 2, 2020* + +Django 3.0.11 adds compatibility with Python 3.9. diff --git a/docs/releases/3.0.txt b/docs/releases/3.0.txt index 284782d80ce4..1c39980a91ed 100644 --- a/docs/releases/3.0.txt +++ b/docs/releases/3.0.txt @@ -19,8 +19,8 @@ project. Python compatibility ==================== -Django 3.0 supports Python 3.6, 3.7, and 3.8. We **highly recommend** and only -officially support the latest release of each series. +Django 3.0 supports Python 3.6, 3.7, 3.8, and 3.9 (as of 3.0.11). We **highly +recommend** and only officially support the latest release of each series. The Django 2.2.x series is the last to support Python 3.5. diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 4b2f3af8baf9..b08af9fa515d 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.0.11 3.0.10 3.0.9 3.0.8 @@ -42,6 +43,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 2.2.17 2.2.16 2.2.15 2.2.14 diff --git a/setup.py b/setup.py index eb133926bafe..2c6170fe5b86 100644 --- a/setup.py +++ b/setup.py @@ -101,6 +101,7 @@ def read(fname): 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3 :: Only', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', diff --git a/tox.ini b/tox.ini index 9e0bdf7accd6..15617d39c515 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ passenv = DJANGO_SETTINGS_MODULE PYTHONPATH HOME DISPLAY setenv = PYTHONDONTWRITEBYTECODE=1 deps = - py{3,36,37,38}: -rtests/requirements/py3.txt + py{3,36,37,38,39}: -rtests/requirements/py3.txt postgres: -rtests/requirements/postgres.txt mysql: -rtests/requirements/mysql.txt oracle: -rtests/requirements/oracle.txt From 72a17c919bf9071ca34b00cd72b0084e5c6da8f6 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 17 Oct 2020 16:17:00 +0200 Subject: [PATCH 70/88] [3.0.x] Fixed #32110 -- Doc'd and tested enumerations for ChoiceField.choices. Backport of 7f85498eef1d8fcc52e4fb70df8041f5452d405a from master --- docs/ref/forms/fields.txt | 12 ++++++------ tests/forms_tests/field_tests/test_choicefield.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/ref/forms/fields.txt b/docs/ref/forms/fields.txt index 5f53becce83b..89b9c697f110 100644 --- a/docs/ref/forms/fields.txt +++ b/docs/ref/forms/fields.txt @@ -408,12 +408,12 @@ For each field, we describe the default widget used if you don't specify .. attribute:: choices Either an :term:`iterable` of 2-tuples to use as choices for this - field, or a callable that returns such an iterable. This argument - accepts the same formats as the ``choices`` argument to a model field. - See the :ref:`model field reference documentation on choices - ` for more details. If the argument is a callable, it is - evaluated each time the field's form is initialized. Defaults to an - empty list. + field, :ref:`enumeration ` choices, or a + callable that returns such an iterable. This argument accepts the same + formats as the ``choices`` argument to a model field. See the + :ref:`model field reference documentation on choices ` + for more details. If the argument is a callable, it is evaluated each + time the field's form is initialized. Defaults to an empty list. ``TypedChoiceField`` -------------------- diff --git a/tests/forms_tests/field_tests/test_choicefield.py b/tests/forms_tests/field_tests/test_choicefield.py index 465cfd83a862..c27d19210855 100644 --- a/tests/forms_tests/field_tests/test_choicefield.py +++ b/tests/forms_tests/field_tests/test_choicefield.py @@ -1,3 +1,4 @@ +from django.db import models from django.forms import ChoiceField, Form, ValidationError from django.test import SimpleTestCase @@ -86,3 +87,14 @@ def test_choicefield_disabled(self): '' ) + + def test_choicefield_enumeration(self): + class FirstNames(models.TextChoices): + JOHN = 'J', 'John' + PAUL = 'P', 'Paul' + + f = ChoiceField(choices=FirstNames.choices) + self.assertEqual(f.clean('J'), 'J') + msg = "'Select a valid choice. 3 is not one of the available choices.'" + with self.assertRaisesMessage(ValidationError, msg): + f.clean('3') From b0a6798de59151163b94d1c0ccf0d010a3d46ac2 Mon Sep 17 00:00:00 2001 From: Christian Klus Date: Tue, 27 Oct 2020 13:13:10 -0500 Subject: [PATCH 71/88] [3.0.x] Fixed #32152 -- Fixed grouping by subquery aliases. Regression in 42c08ee46539ef44f8658ebb1cbefb408e0d03fe. Thanks Simon Charette for the review. Backport of 4ac2d4fa42e1659f328c35b6b8d4761b3419c11a from master --- django/db/models/sql/query.py | 4 +++- docs/releases/3.0.11.txt | 9 ++++++++- tests/annotations/tests.py | 21 ++++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index a4f079f988eb..9504105807d6 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -2134,8 +2134,10 @@ def set_values(self, fields): field_names.append(f) self.set_extra_mask(extra_names) self.set_annotation_mask(annotation_names) + selected = frozenset(field_names + extra_names + annotation_names) else: field_names = [f.attname for f in self.model._meta.concrete_fields] + selected = frozenset(field_names) # Selected annotations must be known before setting the GROUP BY # clause. if self.group_by is True: @@ -2149,7 +2151,7 @@ def set_values(self, fields): # the selected fields anymore. group_by = [] for expr in self.group_by: - if isinstance(expr, Ref) and expr.refs not in field_names: + if isinstance(expr, Ref) and expr.refs not in selected: expr = self.annotations[expr.refs] group_by.append(expr) self.group_by = tuple(group_by) diff --git a/docs/releases/3.0.11.txt b/docs/releases/3.0.11.txt index a5009feb5119..1dcf37a5178d 100644 --- a/docs/releases/3.0.11.txt +++ b/docs/releases/3.0.11.txt @@ -4,4 +4,11 @@ Django 3.0.11 release notes *Expected November 2, 2020* -Django 3.0.11 adds compatibility with Python 3.9. +Django 3.0.11 fixes a regression in 3.0.7 and adds compatibility with Python +3.9. + +Bugfixes +======== + +* Fixed a regression in Django 3.0.7 that didn't use ``Subquery()`` aliases in + the ``GROUP BY`` clause (:ticket:`32152`). diff --git a/tests/annotations/tests.py b/tests/annotations/tests.py index bbd35fbd4fae..2ed3474f5d6b 100644 --- a/tests/annotations/tests.py +++ b/tests/annotations/tests.py @@ -10,7 +10,7 @@ Q, Subquery, Sum, Value, When, ) from django.db.models.expressions import RawSQL -from django.db.models.functions import Length, Lower +from django.db.models.functions import ExtractYear, Length, Lower from django.test import TestCase, skipUnlessDBFeature from .models import ( @@ -635,6 +635,25 @@ def test_annotation_exists_aggregate_values_chaining(self): datetime.date(2008, 11, 3), ]) + @skipUnlessDBFeature('supports_subqueries_in_group_by') + def test_annotation_subquery_and_aggregate_values_chaining(self): + qs = Book.objects.annotate( + pub_year=ExtractYear('pubdate') + ).values('pub_year').annotate( + top_rating=Subquery( + Book.objects.filter( + pubdate__year=OuterRef('pub_year') + ).order_by('-rating').values('rating')[:1] + ), + total_pages=Sum('pages'), + ).values('pub_year', 'total_pages', 'top_rating') + self.assertCountEqual(qs, [ + {'pub_year': 1991, 'top_rating': 5.0, 'total_pages': 946}, + {'pub_year': 1995, 'top_rating': 4.0, 'total_pages': 1132}, + {'pub_year': 2007, 'top_rating': 4.5, 'total_pages': 447}, + {'pub_year': 2008, 'top_rating': 4.0, 'total_pages': 1178}, + ]) + @skipIf( connection.vendor == 'mysql', 'GROUP BY optimization does not work properly when ONLY_FULL_GROUP_BY ' From c3b8a62f63bc48ba8784b6ad452a88a95f221354 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Mon, 2 Nov 2020 08:35:24 +0100 Subject: [PATCH 72/88] [3.0.x] Set release date for 3.0.11 and 2.2.17. Backport of 7fc07b9b2ba0c5c62a8840325d21b414a099fda0 from master --- docs/releases/2.2.17.txt | 2 +- docs/releases/3.0.11.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/releases/2.2.17.txt b/docs/releases/2.2.17.txt index d574de6574b0..4bea2eaed43a 100644 --- a/docs/releases/2.2.17.txt +++ b/docs/releases/2.2.17.txt @@ -2,6 +2,6 @@ Django 2.2.17 release notes =========================== -*Expected November 2, 2020* +*November 2, 2020* Django 2.2.17 adds compatibility with Python 3.9. diff --git a/docs/releases/3.0.11.txt b/docs/releases/3.0.11.txt index 1dcf37a5178d..a5a45b2ab710 100644 --- a/docs/releases/3.0.11.txt +++ b/docs/releases/3.0.11.txt @@ -2,7 +2,7 @@ Django 3.0.11 release notes =========================== -*Expected November 2, 2020* +*November 2, 2020* Django 3.0.11 fixes a regression in 3.0.7 and adds compatibility with Python 3.9. From f6778c256f7f372b3ae1ee945a562990919ef13e Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Mon, 2 Nov 2020 08:59:21 +0100 Subject: [PATCH 73/88] [3.0.x] Bumped version for 3.0.11 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index f2237f0b7384..b537ceb12e66 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 11, 'alpha', 0) +VERSION = (3, 0, 11, 'final', 0) __version__ = get_version(VERSION) From 008d49caa2cc0f1fdd3f13d047c4f13a5e811b39 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Mon, 2 Nov 2020 09:02:53 +0100 Subject: [PATCH 74/88] [3.0.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index b537ceb12e66..114355eaa338 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 11, 'final', 0) +VERSION = (3, 0, 12, 'alpha', 0) __version__ = get_version(VERSION) From c506639b425ca876b2d9a633fc6410e917d45605 Mon Sep 17 00:00:00 2001 From: Max Smolens Date: Tue, 6 Oct 2020 17:58:52 -0400 Subject: [PATCH 75/88] [3.0.x] Fixed #31850 -- Fixed BasicExtractorTests.test_extraction_warning with xgettext 0.21+. "format string with unnamed arguments cannot be properly localized" warning is not raised in xgettext 0.21+. This patch uses a message that causes an xgettext warning regardless of the version. Backport of 07a30f561661efae1691ff45d10ec6014b395b58 from master --- AUTHORS | 1 + tests/i18n/commands/code.sample | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 2431fe61e14c..fd965302bba4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -600,6 +600,7 @@ answer newbie questions, and generally made Django that much better: mattycakes@gmail.com Max Burstein Max Derkachev + Max Smolens Maxime Lorant Maxime Turcotte Maximilian Merz diff --git a/tests/i18n/commands/code.sample b/tests/i18n/commands/code.sample index a5f1520ecba5..2c305a3a1dcf 100644 --- a/tests/i18n/commands/code.sample +++ b/tests/i18n/commands/code.sample @@ -1,4 +1,4 @@ from django.utils.translation import gettext -# This will generate an xgettext warning -my_string = gettext("This string contain two placeholders: %s and %s" % ('a', 'b')) +# This will generate an xgettext "Empty msgid" warning. +my_string = gettext('') From f13bedf102a8e65272d8b10e82eecdaab092a470 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sat, 2 Jan 2021 06:49:00 -0500 Subject: [PATCH 76/88] [3.0.x] Updated CVE URL. Backport of 656b331b13e08e82bbf0b88d39080c5b1a02109c from master --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 972cc029d7da..08ae1854a542 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -96,7 +96,7 @@ def django_release(): extlinks = { 'commit': ('https://github.com/django/django/commit/%s', ''), - 'cve': ('https://nvd.nist.gov/view/vuln/detail?vulnId=%s', 'CVE-'), + 'cve': ('https://nvd.nist.gov/vuln/detail/CVE-%s', 'CVE-'), # A file or directory. GitHub redirects from blob to tree if needed. 'source': ('https://github.com/django/django/blob/master/%s', ''), 'ticket': ('https://code.djangoproject.com/ticket/%s', '#'), From 74ca3cac3272b3a9df6a351095fe4117d0cf2608 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Fri, 29 Jan 2021 11:00:12 +0100 Subject: [PATCH 77/88] [3.0.x] Fixed GeoIPTest.test04_city() failure with the latest GeoIP2 database. Backport of 135c800fe6138d7818501a384c0ebbdc5442762c from master --- tests/gis_tests/test_geoip2.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/gis_tests/test_geoip2.py b/tests/gis_tests/test_geoip2.py index 294d875ac810..91bba2e8ff81 100644 --- a/tests/gis_tests/test_geoip2.py +++ b/tests/gis_tests/test_geoip2.py @@ -20,8 +20,8 @@ "GeoIP is required along with the GEOIP_PATH setting." ) class GeoIPTest(SimpleTestCase): - addr = '75.41.39.1' - fqdn = 'tmc.edu' + addr = '129.237.192.1' + fqdn = 'ku.edu' def test01_init(self): "GeoIP initialization." @@ -105,7 +105,7 @@ def test03_country(self, gethostbyname): @mock.patch('socket.gethostbyname') def test04_city(self, gethostbyname): "GeoIP city querying methods." - gethostbyname.return_value = '75.41.39.1' + gethostbyname.return_value = '129.237.192.1' g = GeoIP2(country='') for query in (self.fqdn, self.addr): @@ -130,8 +130,8 @@ def test04_city(self, gethostbyname): self.assertEqual('NA', d['continent_code']) self.assertEqual('North America', d['continent_name']) self.assertEqual('US', d['country_code']) - self.assertEqual('Dallas', d['city']) - self.assertEqual('TX', d['region']) + self.assertEqual('Lawrence', d['city']) + self.assertEqual('KS', d['region']) self.assertEqual('America/Chicago', d['time_zone']) self.assertFalse(d['is_in_european_union']) geom = g.geos(query) From 52e409ed17287e9aabda847b6afe58be2fa9f86a Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Fri, 22 Jan 2021 12:23:18 +0100 Subject: [PATCH 78/88] [3.0.x] Fixed CVE-2021-3281 -- Fixed potential directory-traversal via archive.extract(). Thanks Florian Apolloner, Shai Berger, and Simon Charette for reviews. Thanks Wang Baohua for the report. Backport of 05413afa8c18cdb978fcdf470e09f7a12b234a23 from master. --- django/utils/archive.py | 17 +++++++++++--- docs/releases/2.2.18.txt | 15 +++++++++++++ docs/releases/3.0.12.txt | 15 +++++++++++++ docs/releases/index.txt | 2 ++ tests/utils_tests/test_archive.py | 21 ++++++++++++++++++ .../traversal_archives/traversal.tar | Bin 0 -> 10240 bytes .../traversal_archives/traversal_absolute.tar | Bin 0 -> 10240 bytes .../traversal_archives/traversal_disk_win.tar | Bin 0 -> 10240 bytes .../traversal_archives/traversal_disk_win.zip | Bin 0 -> 312 bytes 9 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 docs/releases/2.2.18.txt create mode 100644 docs/releases/3.0.12.txt create mode 100644 tests/utils_tests/traversal_archives/traversal.tar create mode 100644 tests/utils_tests/traversal_archives/traversal_absolute.tar create mode 100644 tests/utils_tests/traversal_archives/traversal_disk_win.tar create mode 100644 tests/utils_tests/traversal_archives/traversal_disk_win.zip diff --git a/django/utils/archive.py b/django/utils/archive.py index 235809f2ad0e..d5a0cf044657 100644 --- a/django/utils/archive.py +++ b/django/utils/archive.py @@ -27,6 +27,8 @@ import tarfile import zipfile +from django.core.exceptions import SuspiciousOperation + class ArchiveException(Exception): """ @@ -133,6 +135,13 @@ def has_leading_dir(self, paths): return False return True + def target_filename(self, to_path, name): + target_path = os.path.abspath(to_path) + filename = os.path.abspath(os.path.join(target_path, name)) + if not filename.startswith(target_path): + raise SuspiciousOperation("Archive contains invalid path: '%s'" % name) + return filename + def extract(self): raise NotImplementedError('subclasses of BaseArchive must provide an extract() method') @@ -155,7 +164,7 @@ def extract(self, to_path): name = member.name if leading: name = self.split_leading_dir(name)[1] - filename = os.path.join(to_path, name) + filename = self.target_filename(to_path, name) if member.isdir(): if filename: os.makedirs(filename, exist_ok=True) @@ -198,8 +207,10 @@ def extract(self, to_path): info = self._archive.getinfo(name) if leading: name = self.split_leading_dir(name)[1] - filename = os.path.join(to_path, name) - if filename.endswith(('/', '\\')): + if not name: + continue + filename = self.target_filename(to_path, name) + if name.endswith(('/', '\\')): # A directory os.makedirs(filename, exist_ok=True) else: diff --git a/docs/releases/2.2.18.txt b/docs/releases/2.2.18.txt new file mode 100644 index 000000000000..45df4fb83c9f --- /dev/null +++ b/docs/releases/2.2.18.txt @@ -0,0 +1,15 @@ +=========================== +Django 2.2.18 release notes +=========================== + +*February 1, 2021* + +Django 2.2.18 fixes a security issue with severity "low" in 2.2.17. + +CVE-2021-3281: Potential directory-traversal via ``archive.extract()`` +====================================================================== + +The ``django.utils.archive.extract()`` function, used by +:option:`startapp --template` and :option:`startproject --template`, allowed +directory-traversal via an archive with absolute paths or relative paths with +dot segments. diff --git a/docs/releases/3.0.12.txt b/docs/releases/3.0.12.txt new file mode 100644 index 000000000000..20d9459feae2 --- /dev/null +++ b/docs/releases/3.0.12.txt @@ -0,0 +1,15 @@ +=========================== +Django 3.0.12 release notes +=========================== + +*February 1, 2021* + +Django 3.0.12 fixes a security issue with severity "low" in 3.0.11. + +CVE-2021-3281: Potential directory-traversal via ``archive.extract()`` +====================================================================== + +The ``django.utils.archive.extract()`` function, used by +:option:`startapp --template` and :option:`startproject --template`, allowed +directory-traversal via an archive with absolute paths or relative paths with +dot segments. diff --git a/docs/releases/index.txt b/docs/releases/index.txt index b08af9fa515d..49281fa9b532 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.0.12 3.0.11 3.0.10 3.0.9 @@ -43,6 +44,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 2.2.18 2.2.17 2.2.16 2.2.15 diff --git a/tests/utils_tests/test_archive.py b/tests/utils_tests/test_archive.py index dc7c4b4ebd9a..8fdf3ec4459b 100644 --- a/tests/utils_tests/test_archive.py +++ b/tests/utils_tests/test_archive.py @@ -4,6 +4,8 @@ import tempfile import unittest +from django.core.exceptions import SuspiciousOperation +from django.test import SimpleTestCase from django.utils import archive @@ -45,3 +47,22 @@ def test_extract_file_permissions(self): # A file is readable even if permission data is missing. filepath = os.path.join(tmpdir, 'no_permissions') self.assertEqual(os.stat(filepath).st_mode & mask, 0o666 & ~umask) + + +class TestArchiveInvalid(SimpleTestCase): + def test_extract_function_traversal(self): + archives_dir = os.path.join(os.path.dirname(__file__), 'traversal_archives') + tests = [ + ('traversal.tar', '..'), + ('traversal_absolute.tar', '/tmp/evil.py'), + ] + if sys.platform == 'win32': + tests += [ + ('traversal_disk_win.tar', 'd:evil.py'), + ('traversal_disk_win.zip', 'd:evil.py'), + ] + msg = "Archive contains invalid path: '%s'" + for entry, invalid_path in tests: + with self.subTest(entry), tempfile.TemporaryDirectory() as tmpdir: + with self.assertRaisesMessage(SuspiciousOperation, msg % invalid_path): + archive.extract(os.path.join(archives_dir, entry), tmpdir) diff --git a/tests/utils_tests/traversal_archives/traversal.tar b/tests/utils_tests/traversal_archives/traversal.tar new file mode 100644 index 0000000000000000000000000000000000000000..07eede517a56bdcfd67d561015eb8764ab7bfd82 GIT binary patch literal 10240 zcmeIwO$x&x5QgD7iYG`MiF%$cA_M{{{Ug17BSpy1qAM4n?`(!RM26=xO;g)6t<)^L zRE%DhrqSDV&!b$Towd$8)u>`sI~T?HnoE_tpZCY*W{jkM9Ok*49{aEP@sCrSq;LNH zYZ#mByUxlv;1IfX?&TfiQLbJ7F1jGb)>&tJ>!0Vp#o!A81Q0*~0R#|0009ILKmY** g5I_I{1Q0*~0R#|0009ILKmY**5I_I{1kNPz0VrH4j{pDw literal 0 HcmV?d00001 diff --git a/tests/utils_tests/traversal_archives/traversal_absolute.tar b/tests/utils_tests/traversal_archives/traversal_absolute.tar new file mode 100644 index 0000000000000000000000000000000000000000..231566b0699d48145cb6fa5c0c792c2569417a19 GIT binary patch literal 10240 zcmeIwO$x#=5QgD7N={%WX@1V*qJ=`GMXS`?+X!Md#Z^G@olOEmlHvK%Pg5h6OSeiX z$hO!Nv|Mv5msqdrf{R|0sI|_uVnXG)p4VS5%kgZC^xZhD>;8+M`uiupy3;JDx#@1h zc$n|C2F(G-=*!$+{~)(=z4Q0&mcNTOiqTpFmG}O6{v`!_1Q0*~0R#|0009ILKmY** h5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q7Twfd`<_D7ydv literal 0 HcmV?d00001 diff --git a/tests/utils_tests/traversal_archives/traversal_disk_win.tar b/tests/utils_tests/traversal_archives/traversal_disk_win.tar new file mode 100644 index 0000000000000000000000000000000000000000..97f0b95501c1a67b3f04f8d8736de172af68eb57 GIT binary patch literal 10240 zcmeIwTMEK35QX7cN>-qg*lEDs2)0ltRJ>7lZ&DCiq4*R~{O2WZDEmANS3@ z>o=!Ip-pqabzRMSfBgqJ%JbHLh-Tun&_0W6|GfW&1rG=yfB*srAbxYJLGBBsF;!4N{;?fFk21b?_%nS@*A^@ju zUa0OAtJJd09KC`{xG^B3FpObjl4Hi@3<;=%1Q`B0f|w9Tu|gb0f=Rp#Fq0S Date: Mon, 1 Feb 2021 09:40:04 +0100 Subject: [PATCH 79/88] [3.0.x] Bumped version for 3.0.12 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 114355eaa338..6c309230c6a6 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 12, 'alpha', 0) +VERSION = (3, 0, 12, 'final', 0) __version__ = get_version(VERSION) From bfe24c380355c753b9d6b38a841d8f0e0491b571 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 1 Feb 2021 09:48:38 +0100 Subject: [PATCH 80/88] [3.0.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 6c309230c6a6..c9b2f52a0197 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 12, 'final', 0) +VERSION = (3, 0, 13, 'alpha', 0) __version__ = get_version(VERSION) From 0194f0be31b92c8e66636859df618611fb2f4d18 Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Mon, 1 Feb 2021 10:21:30 +0100 Subject: [PATCH 81/88] [3.0.x] Added CVE-2021-3281 to security archive. Backport of f749148d62ece28d208ab66b109f858215ba090a from master --- docs/releases/security.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index 3d659bc1ea42..e82c4be41ee8 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -1134,3 +1134,16 @@ Versions affected * Django 3.1 :commit:`(patch) <2b099caa5923afa8cfb5f1e8c0d56b6e0e81915b>` * Django 3.0 :commit:`(patch) ` * Django 2.2 :commit:`(patch) ` + +February 1, 2021 - :cve:`2021-3281` +----------------------------------- + +Potential directory-traversal via ``archive.extract()``. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.1 :commit:`(patch) <02e6592835b4559909aa3aaaf67988fef435f624>` +* Django 3.0 :commit:`(patch) <52e409ed17287e9aabda847b6afe58be2fa9f86a>` +* Django 2.2 :commit:`(patch) <21e7622dec1f8612c85c2fc37fe8efbfd3311e37>` From ad36388406195ecddb319fea862298e6c8d27d3b Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Tue, 16 Feb 2021 10:08:39 +0000 Subject: [PATCH 82/88] [3.0.x] Added documentation extlink for bugs.python.org. Backport of d02d60eb0f032c9395199fb73c6cd29ee9bb2646 from master --- docs/conf.py | 1 + docs/releases/1.5.1.txt | 6 ++---- docs/releases/1.6.11.txt | 6 +++--- docs/releases/1.7.7.txt | 6 +++--- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 08ae1854a542..525970262983 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -95,6 +95,7 @@ def django_release(): django_next_version = '3.1' extlinks = { + 'bpo': ('https://bugs.python.org/issue%s', 'bpo-'), 'commit': ('https://github.com/django/django/commit/%s', ''), 'cve': ('https://nvd.nist.gov/vuln/detail/CVE-%s', 'CVE-'), # A file or directory. GitHub redirects from blob to tree if needed. diff --git a/docs/releases/1.5.1.txt b/docs/releases/1.5.1.txt index cc961ac304bf..66d78997626a 100644 --- a/docs/releases/1.5.1.txt +++ b/docs/releases/1.5.1.txt @@ -10,10 +10,8 @@ compatible with Django 1.5, but includes a handful of fixes. The biggest fix is for a memory leak introduced in Django 1.5. Under certain circumstances, repeated iteration over querysets could leak memory - sometimes quite a bit of it. If you'd like more information, the details are in -:ticket:`our ticket tracker <19895>` (and in `a related issue`__ in Python -itself). - -__ https://bugs.python.org/issue17468 +:ticket:`our ticket tracker <19895>` (and in :bpo:`a related issue <17468>` in +Python itself). If you've noticed memory problems under Django 1.5, upgrading to 1.5.1 should fix those issues. diff --git a/docs/releases/1.6.11.txt b/docs/releases/1.6.11.txt index 8cf81f89bfdb..1bf2bf89110b 100644 --- a/docs/releases/1.6.11.txt +++ b/docs/releases/1.6.11.txt @@ -13,9 +13,9 @@ Last year :func:`~django.utils.html.strip_tags` was changed to work iteratively. The problem is that the size of the input it's processing can increase on each iteration which results in an infinite loop in ``strip_tags()``. This issue only affects versions of Python that haven't -received `a bugfix in HTMLParser `_; namely -Python < 2.7.7 and 3.3.5. Some operating system vendors have also backported -the fix for the Python bug into their packages of earlier versions. +received :bpo:`a bugfix in HTMLParser <20288>`; namely Python < 2.7.7 and +3.3.5. Some operating system vendors have also backported the fix for the +Python bug into their packages of earlier versions. To remedy this issue, ``strip_tags()`` will now return the original input if it detects the length of the string it's processing increases. Remember that diff --git a/docs/releases/1.7.7.txt b/docs/releases/1.7.7.txt index f20ee127bcc5..bfd54563a1ee 100644 --- a/docs/releases/1.7.7.txt +++ b/docs/releases/1.7.7.txt @@ -13,9 +13,9 @@ Last year :func:`~django.utils.html.strip_tags` was changed to work iteratively. The problem is that the size of the input it's processing can increase on each iteration which results in an infinite loop in ``strip_tags()``. This issue only affects versions of Python that haven't -received `a bugfix in HTMLParser `_; namely -Python < 2.7.7 and 3.3.5. Some operating system vendors have also backported -the fix for the Python bug into their packages of earlier versions. +received :bpo:`a bugfix in HTMLParser <20288>`; namely Python < 2.7.7 and +3.3.5. Some operating system vendors have also backported the fix for the +Python bug into their packages of earlier versions. To remedy this issue, ``strip_tags()`` will now return the original input if it detects the length of the string it's processing increases. Remember that From 326a926beef869d3341bc9ef737887f0449b6b71 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Tue, 16 Feb 2021 10:14:17 +0000 Subject: [PATCH 83/88] [3.0.x] Fixed CVE-2021-23336 -- Fixed web cache poisoning via django.utils.http.limited_parse_qsl(). --- django/utils/http.py | 2 +- docs/releases/2.2.19.txt | 16 +++++++ docs/releases/3.0.13.txt | 16 +++++++ docs/releases/index.txt | 2 + tests/handlers/test_exception.py | 2 +- tests/requests/test_data_upload_settings.py | 8 ++-- tests/utils_tests/test_http.py | 52 +++++++++++++++++++-- 7 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 docs/releases/2.2.19.txt create mode 100644 docs/releases/3.0.13.txt diff --git a/django/utils/http.py b/django/utils/http.py index ff2f08ac1e84..c8f2c49a821a 100644 --- a/django/utils/http.py +++ b/django/utils/http.py @@ -41,7 +41,7 @@ RFC3986_GENDELIMS = ":/?#[]@" RFC3986_SUBDELIMS = "!$&'()*+,;=" -FIELDS_MATCH = re.compile('[&;]') +FIELDS_MATCH = re.compile('&') @keep_lazy_text diff --git a/docs/releases/2.2.19.txt b/docs/releases/2.2.19.txt new file mode 100644 index 000000000000..feaffd996cac --- /dev/null +++ b/docs/releases/2.2.19.txt @@ -0,0 +1,16 @@ +=========================== +Django 2.2.19 release notes +=========================== + +*February 19, 2021* + +Django 2.2.19 fixes a security issue in 2.2.18. + +CVE-2021-23336: Web cache poisoning via ``django.utils.http.limited_parse_qsl()`` +================================================================================= + +Django contains a copy of :func:`urllib.parse.parse_qsl` which was added to +backport some security fixes. A further security fix has been issued recently +such that ``parse_qsl()`` no longer allows using ``;`` as a query parameter +separator by default. Django now includes this fix. See :bpo:`42967` for +further details. diff --git a/docs/releases/3.0.13.txt b/docs/releases/3.0.13.txt new file mode 100644 index 000000000000..c78b8a04fd15 --- /dev/null +++ b/docs/releases/3.0.13.txt @@ -0,0 +1,16 @@ +=========================== +Django 3.0.13 release notes +=========================== + +*February 19, 2021* + +Django 3.0.13 fixes a security issue in 3.0.12. + +CVE-2021-23336: Web cache poisoning via ``django.utils.http.limited_parse_qsl()`` +================================================================================= + +Django contains a copy of :func:`urllib.parse.parse_qsl` which was added to +backport some security fixes. A further security fix has been issued recently +such that ``parse_qsl()`` no longer allows using ``;`` as a query parameter +separator by default. Django now includes this fix. See :bpo:`42967` for +further details. diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 49281fa9b532..aee1d4223159 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.0.13 3.0.12 3.0.11 3.0.10 @@ -44,6 +45,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 2.2.19 2.2.18 2.2.17 2.2.16 diff --git a/tests/handlers/test_exception.py b/tests/handlers/test_exception.py index 7afd4acc6b08..0c1e76399045 100644 --- a/tests/handlers/test_exception.py +++ b/tests/handlers/test_exception.py @@ -6,7 +6,7 @@ class ExceptionHandlerTests(SimpleTestCase): def get_suspicious_environ(self): - payload = FakePayload('a=1&a=2;a=3\r\n') + payload = FakePayload('a=1&a=2&a=3\r\n') return { 'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': 'application/x-www-form-urlencoded', diff --git a/tests/requests/test_data_upload_settings.py b/tests/requests/test_data_upload_settings.py index f60f1850ea25..44897cc9fa97 100644 --- a/tests/requests/test_data_upload_settings.py +++ b/tests/requests/test_data_upload_settings.py @@ -11,7 +11,7 @@ class DataUploadMaxMemorySizeFormPostTests(SimpleTestCase): def setUp(self): - payload = FakePayload('a=1&a=2;a=3\r\n') + payload = FakePayload('a=1&a=2&a=3\r\n') self.request = WSGIRequest({ 'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': 'application/x-www-form-urlencoded', @@ -117,7 +117,7 @@ def test_get_max_fields_exceeded(self): request = WSGIRequest({ 'REQUEST_METHOD': 'GET', 'wsgi.input': BytesIO(b''), - 'QUERY_STRING': 'a=1&a=2;a=3', + 'QUERY_STRING': 'a=1&a=2&a=3', }) request.GET['a'] @@ -126,7 +126,7 @@ def test_get_max_fields_not_exceeded(self): request = WSGIRequest({ 'REQUEST_METHOD': 'GET', 'wsgi.input': BytesIO(b''), - 'QUERY_STRING': 'a=1&a=2;a=3', + 'QUERY_STRING': 'a=1&a=2&a=3', }) request.GET['a'] @@ -168,7 +168,7 @@ def test_no_limit(self): class DataUploadMaxNumberOfFieldsFormPost(SimpleTestCase): def setUp(self): - payload = FakePayload("\r\n".join(['a=1&a=2;a=3', ''])) + payload = FakePayload("\r\n".join(['a=1&a=2&a=3', ''])) self.request = WSGIRequest({ 'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': 'application/x-www-form-urlencoded', diff --git a/tests/utils_tests/test_http.py b/tests/utils_tests/test_http.py index aa9f194a8a53..0ec14a681961 100644 --- a/tests/utils_tests/test_http.py +++ b/tests/utils_tests/test_http.py @@ -3,14 +3,16 @@ from datetime import datetime from unittest import mock +from django.core.exceptions import TooManyFieldsSent from django.test import SimpleTestCase, ignore_warnings from django.utils.datastructures import MultiValueDict from django.utils.deprecation import RemovedInDjango40Warning from django.utils.http import ( base36_to_int, escape_leading_slashes, http_date, int_to_base36, - is_safe_url, is_same_domain, parse_etags, parse_http_date, quote_etag, - url_has_allowed_host_and_scheme, urlencode, urlquote, urlquote_plus, - urlsafe_base64_decode, urlsafe_base64_encode, urlunquote, urlunquote_plus, + is_safe_url, is_same_domain, limited_parse_qsl, parse_etags, + parse_http_date, quote_etag, url_has_allowed_host_and_scheme, urlencode, + urlquote, urlquote_plus, urlsafe_base64_decode, urlsafe_base64_encode, + urlunquote, urlunquote_plus, ) @@ -359,3 +361,47 @@ def test(self): for url, expected in tests: with self.subTest(url=url): self.assertEqual(escape_leading_slashes(url), expected) + + +# Backport of unit tests for urllib.parse.parse_qsl() from Python 3.8.8. +# Copyright (C) 2021 Python Software Foundation (see LICENSE.python). +class ParseQSLBackportTests(unittest.TestCase): + def test_parse_qsl(self): + tests = [ + ('', []), + ('&', []), + ('&&', []), + ('=', [('', '')]), + ('=a', [('', 'a')]), + ('a', [('a', '')]), + ('a=', [('a', '')]), + ('&a=b', [('a', 'b')]), + ('a=a+b&b=b+c', [('a', 'a b'), ('b', 'b c')]), + ('a=1&a=2', [('a', '1'), ('a', '2')]), + (';a=b', [(';a', 'b')]), + ('a=a+b;b=b+c', [('a', 'a b;b=b c')]), + ] + for original, expected in tests: + with self.subTest(original): + result = limited_parse_qsl(original, keep_blank_values=True) + self.assertEqual(result, expected, 'Error parsing %r' % original) + expect_without_blanks = [v for v in expected if len(v[1])] + result = limited_parse_qsl(original, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks, 'Error parsing %r' % original) + + def test_parse_qsl_encoding(self): + result = limited_parse_qsl('key=\u0141%E9', encoding='latin-1') + self.assertEqual(result, [('key', '\u0141\xE9')]) + result = limited_parse_qsl('key=\u0141%C3%A9', encoding='utf-8') + self.assertEqual(result, [('key', '\u0141\xE9')]) + result = limited_parse_qsl('key=\u0141%C3%A9', encoding='ascii') + self.assertEqual(result, [('key', '\u0141\ufffd\ufffd')]) + result = limited_parse_qsl('key=\u0141%E9-', encoding='ascii') + self.assertEqual(result, [('key', '\u0141\ufffd-')]) + result = limited_parse_qsl('key=\u0141%E9-', encoding='ascii', errors='ignore') + self.assertEqual(result, [('key', '\u0141-')]) + + def test_parse_qsl_field_limit(self): + with self.assertRaises(TooManyFieldsSent): + limited_parse_qsl('&'.join(['a=a'] * 11), fields_limit=10) + limited_parse_qsl('&'.join(['a=a'] * 10), fields_limit=10) From 04a9b7df3fe627d03547a01330a20dfcbf2e8bd5 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Fri, 19 Feb 2021 09:42:24 +0100 Subject: [PATCH 84/88] [3.0.x] Bumped version for 3.0.13 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index c9b2f52a0197..af734b9e557f 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 13, 'alpha', 0) +VERSION = (3, 0, 13, 'final', 0) __version__ = get_version(VERSION) From cb7e3ff4a1d4132609751d0dee40e93c8cbda1c3 Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Fri, 19 Feb 2021 09:43:26 +0100 Subject: [PATCH 85/88] [3.0.x] Post-release version bump. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index af734b9e557f..6544631302f7 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 13, 'final', 0) +VERSION = (3, 0, 14, 'alpha', 0) __version__ = get_version(VERSION) From 232d5f61e6afd9cd6f10a47ddb4375f86818717e Mon Sep 17 00:00:00 2001 From: Carlton Gibson Date: Fri, 19 Feb 2021 11:06:46 +0100 Subject: [PATCH 86/88] [3.0.x] Added CVE-2021-23336 to security archive. Backport of ab58f072502e86dfe21b2bd5cccdc5e94dce8d26 from master --- docs/releases/security.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/releases/security.txt b/docs/releases/security.txt index e82c4be41ee8..10f871d563fd 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -1147,3 +1147,18 @@ Versions affected * Django 3.1 :commit:`(patch) <02e6592835b4559909aa3aaaf67988fef435f624>` * Django 3.0 :commit:`(patch) <52e409ed17287e9aabda847b6afe58be2fa9f86a>` * Django 2.2 :commit:`(patch) <21e7622dec1f8612c85c2fc37fe8efbfd3311e37>` + +February 19, 2021 - :cve:`2021-23336` +------------------------------------- + +Web cache poisoning via ``django.utils.http.limited_parse_qsl()``. `Full +description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) ` +* Django 3.1 :commit:`(patch) <8f6d431b08cbb418d9144b976e7b972546607851>` +* Django 3.0 :commit:`(patch) <326a926beef869d3341bc9ef737887f0449b6b71>` +* Django 2.2 :commit:`(patch) ` From e7fba62248f604c76da4f23dcf1db4a57b0808ea Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 16 Mar 2021 10:19:00 +0100 Subject: [PATCH 87/88] [3.0.x] Fixed CVE-2021-28658 -- Fixed potential directory-traversal via uploaded files. Thanks Claude Paroz for the initial patch. Thanks Dennis Brinkrolf for the report. Backport of d4d800ca1addc4141e03c5440a849bb64d1582cd from main. --- django/http/multipartparser.py | 13 +++-- docs/releases/2.2.20.txt | 15 ++++++ docs/releases/3.0.14.txt | 15 ++++++ docs/releases/index.txt | 2 + tests/file_uploads/tests.py | 84 +++++++++++++++++++++++------ tests/file_uploads/uploadhandler.py | 31 +++++++++++ tests/file_uploads/urls.py | 1 + tests/file_uploads/views.py | 12 ++++- 8 files changed, 152 insertions(+), 21 deletions(-) create mode 100644 docs/releases/2.2.20.txt create mode 100644 docs/releases/3.0.14.txt diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index fd8fce8b4d46..db1b5ce8b956 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -9,6 +9,7 @@ import cgi import collections import html +import os from urllib.parse import unquote from django.conf import settings @@ -209,7 +210,7 @@ def parse(self): file_name = disposition.get('filename') if file_name: file_name = force_str(file_name, encoding, errors='replace') - file_name = self.IE_sanitize(html.unescape(file_name)) + file_name = self.sanitize_file_name(file_name) if not file_name: continue @@ -297,9 +298,13 @@ def handle_file_complete(self, old_field_name, counters): self._files.appendlist(force_str(old_field_name, self._encoding, errors='replace'), file_obj) break - def IE_sanitize(self, filename): - """Cleanup filename from Internet Explorer full paths.""" - return filename and filename[filename.rfind("\\") + 1:].strip() + def sanitize_file_name(self, file_name): + file_name = html.unescape(file_name) + # Cleanup Windows-style path separators. + file_name = file_name[file_name.rfind('\\') + 1:].strip() + return os.path.basename(file_name) + + IE_sanitize = sanitize_file_name def _close_files(self): # Free up all file handles. diff --git a/docs/releases/2.2.20.txt b/docs/releases/2.2.20.txt new file mode 100644 index 000000000000..a67c51502181 --- /dev/null +++ b/docs/releases/2.2.20.txt @@ -0,0 +1,15 @@ +=========================== +Django 2.2.20 release notes +=========================== + +*April 6, 2021* + +Django 2.2.20 fixes a security issue with severity "low" in 2.2.19. + +CVE-2021-28658: Potential directory-traversal via uploaded files +================================================================ + +``MultiPartParser`` allowed directory-traversal via uploaded files with +suitably crafted file names. + +Built-in upload handlers were not affected by this vulnerability. diff --git a/docs/releases/3.0.14.txt b/docs/releases/3.0.14.txt new file mode 100644 index 000000000000..c32442874523 --- /dev/null +++ b/docs/releases/3.0.14.txt @@ -0,0 +1,15 @@ +=========================== +Django 3.0.14 release notes +=========================== + +*April 6, 2021* + +Django 3.0.14 fixes a security issue with severity "low" in 3.0.13. + +CVE-2021-28658: Potential directory-traversal via uploaded files +================================================================ + +``MultiPartParser`` allowed directory-traversal via uploaded files with +suitably crafted file names. + +Built-in upload handlers were not affected by this vulnerability. diff --git a/docs/releases/index.txt b/docs/releases/index.txt index aee1d4223159..bc30665aea64 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.0.14 3.0.13 3.0.12 3.0.11 @@ -45,6 +46,7 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 2.2.20 2.2.19 2.2.18 2.2.17 diff --git a/tests/file_uploads/tests.py b/tests/file_uploads/tests.py index 7b31d369b2bc..f6ab0d183cf4 100644 --- a/tests/file_uploads/tests.py +++ b/tests/file_uploads/tests.py @@ -22,6 +22,22 @@ MEDIA_ROOT = sys_tempfile.mkdtemp() UPLOAD_TO = os.path.join(MEDIA_ROOT, 'test_upload') +CANDIDATE_TRAVERSAL_FILE_NAMES = [ + '/tmp/hax0rd.txt', # Absolute path, *nix-style. + 'C:\\Windows\\hax0rd.txt', # Absolute path, win-style. + 'C:/Windows/hax0rd.txt', # Absolute path, broken-style. + '\\tmp\\hax0rd.txt', # Absolute path, broken in a different way. + '/tmp\\hax0rd.txt', # Absolute path, broken by mixing. + 'subdir/hax0rd.txt', # Descendant path, *nix-style. + 'subdir\\hax0rd.txt', # Descendant path, win-style. + 'sub/dir\\hax0rd.txt', # Descendant path, mixed. + '../../hax0rd.txt', # Relative path, *nix-style. + '..\\..\\hax0rd.txt', # Relative path, win-style. + '../..\\hax0rd.txt', # Relative path, mixed. + '../hax0rd.txt', # HTML entities. + '../hax0rd.txt', # HTML entities. +] + @override_settings(MEDIA_ROOT=MEDIA_ROOT, ROOT_URLCONF='file_uploads.urls', MIDDLEWARE=[]) class FileUploadTests(TestCase): @@ -204,22 +220,8 @@ def test_dangerous_file_names(self): # a malicious payload with an invalid file name (containing os.sep or # os.pardir). This similar to what an attacker would need to do when # trying such an attack. - scary_file_names = [ - "/tmp/hax0rd.txt", # Absolute path, *nix-style. - "C:\\Windows\\hax0rd.txt", # Absolute path, win-style. - "C:/Windows/hax0rd.txt", # Absolute path, broken-style. - "\\tmp\\hax0rd.txt", # Absolute path, broken in a different way. - "/tmp\\hax0rd.txt", # Absolute path, broken by mixing. - "subdir/hax0rd.txt", # Descendant path, *nix-style. - "subdir\\hax0rd.txt", # Descendant path, win-style. - "sub/dir\\hax0rd.txt", # Descendant path, mixed. - "../../hax0rd.txt", # Relative path, *nix-style. - "..\\..\\hax0rd.txt", # Relative path, win-style. - "../..\\hax0rd.txt" # Relative path, mixed. - ] - payload = client.FakePayload() - for i, name in enumerate(scary_file_names): + for i, name in enumerate(CANDIDATE_TRAVERSAL_FILE_NAMES): payload.write('\r\n'.join([ '--' + client.BOUNDARY, 'Content-Disposition: form-data; name="file%s"; filename="%s"' % (i, name), @@ -239,7 +241,7 @@ def test_dangerous_file_names(self): response = self.client.request(**r) # The filenames should have been sanitized by the time it got to the view. received = response.json() - for i, name in enumerate(scary_file_names): + for i, name in enumerate(CANDIDATE_TRAVERSAL_FILE_NAMES): got = received["file%s" % i] self.assertEqual(got, "hax0rd.txt") @@ -517,6 +519,47 @@ def test_filename_case_preservation(self): # shouldn't differ. self.assertEqual(os.path.basename(obj.testfile.path), 'MiXeD_cAsE.txt') + def test_filename_traversal_upload(self): + os.makedirs(UPLOAD_TO, exist_ok=True) + self.addCleanup(shutil.rmtree, MEDIA_ROOT) + tests = [ + '../test.txt', + '../test.txt', + ] + for file_name in tests: + with self.subTest(file_name=file_name): + payload = client.FakePayload() + payload.write( + '\r\n'.join([ + '--' + client.BOUNDARY, + 'Content-Disposition: form-data; name="my_file"; ' + 'filename="%s";' % file_name, + 'Content-Type: text/plain', + '', + 'file contents.\r\n', + '\r\n--' + client.BOUNDARY + '--\r\n', + ]), + ) + r = { + 'CONTENT_LENGTH': len(payload), + 'CONTENT_TYPE': client.MULTIPART_CONTENT, + 'PATH_INFO': '/upload_traversal/', + 'REQUEST_METHOD': 'POST', + 'wsgi.input': payload, + } + response = self.client.request(**r) + result = response.json() + self.assertEqual(response.status_code, 200) + self.assertEqual(result['file_name'], 'test.txt') + self.assertIs( + os.path.exists(os.path.join(MEDIA_ROOT, 'test.txt')), + False, + ) + self.assertIs( + os.path.exists(os.path.join(UPLOAD_TO, 'test.txt')), + True, + ) + @override_settings(MEDIA_ROOT=MEDIA_ROOT) class DirectoryCreationTests(SimpleTestCase): @@ -586,6 +629,15 @@ def test_bad_type_content_length(self): }, StringIO('x'), [], 'utf-8') self.assertEqual(multipart_parser._content_length, 0) + def test_sanitize_file_name(self): + parser = MultiPartParser({ + 'CONTENT_TYPE': 'multipart/form-data; boundary=_foo', + 'CONTENT_LENGTH': '1' + }, StringIO('x'), [], 'utf-8') + for file_name in CANDIDATE_TRAVERSAL_FILE_NAMES: + with self.subTest(file_name=file_name): + self.assertEqual(parser.sanitize_file_name(file_name), 'hax0rd.txt') + def test_rfc2231_parsing(self): test_data = ( (b"Content-Type: application/x-stuff; title*=us-ascii'en-us'This%20is%20%2A%2A%2Afun%2A%2A%2A", diff --git a/tests/file_uploads/uploadhandler.py b/tests/file_uploads/uploadhandler.py index 7c6199fd16d3..65d70c648c44 100644 --- a/tests/file_uploads/uploadhandler.py +++ b/tests/file_uploads/uploadhandler.py @@ -1,6 +1,8 @@ """ Upload handlers to test the upload API. """ +import os +from tempfile import NamedTemporaryFile from django.core.files.uploadhandler import FileUploadHandler, StopUpload @@ -35,3 +37,32 @@ class ErroringUploadHandler(FileUploadHandler): """A handler that raises an exception.""" def receive_data_chunk(self, raw_data, start): raise CustomUploadError("Oops!") + + +class TraversalUploadHandler(FileUploadHandler): + """A handler with potential directory-traversal vulnerability.""" + def __init__(self, request=None): + from .views import UPLOAD_TO + + super().__init__(request) + self.upload_dir = UPLOAD_TO + + def file_complete(self, file_size): + self.file.seek(0) + self.file.size = file_size + with open(os.path.join(self.upload_dir, self.file_name), 'wb') as fp: + fp.write(self.file.read()) + return self.file + + def new_file( + self, field_name, file_name, content_type, content_length, charset=None, + content_type_extra=None, + ): + super().new_file( + file_name, file_name, content_length, content_length, charset, + content_type_extra, + ) + self.file = NamedTemporaryFile(suffix='.upload', dir=self.upload_dir) + + def receive_data_chunk(self, raw_data, start): + self.file.write(raw_data) diff --git a/tests/file_uploads/urls.py b/tests/file_uploads/urls.py index 3e7985d2f9db..eaac1dae3d4b 100644 --- a/tests/file_uploads/urls.py +++ b/tests/file_uploads/urls.py @@ -4,6 +4,7 @@ urlpatterns = [ path('upload/', views.file_upload_view), + path('upload_traversal/', views.file_upload_traversal_view), path('verify/', views.file_upload_view_verify), path('unicode_name/', views.file_upload_unicode_name), path('echo/', views.file_upload_echo), diff --git a/tests/file_uploads/views.py b/tests/file_uploads/views.py index 36c9fc12a2ba..06c47d18dd7f 100644 --- a/tests/file_uploads/views.py +++ b/tests/file_uploads/views.py @@ -6,7 +6,9 @@ from .models import FileModel from .tests import UNICODE_FILENAME, UPLOAD_TO -from .uploadhandler import ErroringUploadHandler, QuotaUploadHandler +from .uploadhandler import ( + ErroringUploadHandler, QuotaUploadHandler, TraversalUploadHandler, +) def file_upload_view(request): @@ -141,3 +143,11 @@ def file_upload_fd_closing(request, access): if access == 't': request.FILES # Trigger file parsing. return HttpResponse() + + +def file_upload_traversal_view(request): + request.upload_handlers.insert(0, TraversalUploadHandler()) + request.FILES # Trigger file parsing. + return JsonResponse( + {'file_name': request.upload_handlers[0].file_name}, + ) From f52800243545af658c27f83623b5fc72ae6b8dcf Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 6 Apr 2021 08:34:45 +0200 Subject: [PATCH 88/88] [3.0.x] Bumped version for 3.0.14 release. --- django/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/__init__.py b/django/__init__.py index 6544631302f7..c245e9b41023 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 14, 'alpha', 0) +VERSION = (3, 0, 14, 'final', 0) __version__ = get_version(VERSION)